chrome/browser/safe_browsing/safe_browsing_database_unittest.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4 //
   5 // Unit tests for the SafeBrowsing storage system.
   6
   7 #include "base/file_util.h"
   8 #include "base/files/scoped_temp_dir.h"
   9 #include "base/logging.h"
  10 #include "base/message_loop/message_loop.h"
  11 #include "base/sha1.h"
  12 #include "base/time/time.h"
  13 #include "chrome/browser/safe_browsing/safe_browsing_database.h"
  14 #include "chrome/browser/safe_browsing/safe_browsing_store_file.h"
  15 #include "chrome/browser/safe_browsing/safe_browsing_store_unittest_helper.h"
  16 #include "content/public/test/test_browser_thread_bundle.h"
  17 #include "crypto/sha2.h"
  18 #include "net/base/net_util.h"
  19 #include "sql/connection.h"
  20 #include "sql/statement.h"
  21 #include "testing/gtest/include/gtest/gtest.h"
  22 #include "testing/platform_test.h"
  23 #include "url/gurl.h"
  24
  25 using base::Time;
  26
  27 namespace {
  28
  29 SBPrefix Sha256Prefix(const std::string& str) {
  30   SBPrefix prefix;
  31   crypto::SHA256HashString(str, &prefix, sizeof(prefix));
  32   return prefix;
  33 }
  34
  35 SBFullHash Sha256Hash(const std::string& str) {
  36   SBFullHash hash;
  37   crypto::SHA256HashString(str, &hash, sizeof(hash));
  38   return hash;
  39 }
  40
  41 std::string HashedIpPrefix(const std::string& ip_prefix, size_t prefix_size) {
  42   net::IPAddressNumber ip_number;
  43   EXPECT_TRUE(net::ParseIPLiteralToNumber(ip_prefix, &ip_number));
  44   EXPECT_EQ(net::kIPv6AddressSize, ip_number.size());
  45   const std::string hashed_ip_prefix = base::SHA1HashString(
  46       net::IPAddressToPackedString(ip_number));
  47   std::string hash(crypto::kSHA256Length, '\0');
  48   hash.replace(0, hashed_ip_prefix.size(), hashed_ip_prefix);
  49   hash[base::kSHA1Length] = static_cast<char>(prefix_size);
  50   return hash;
  51 }
  52
  53 // Same as InsertAddChunkHostPrefixUrl, but with pre-computed
  54 // prefix values.
  55 void InsertAddChunkHostPrefixValue(SBChunk* chunk,
  56                                    int chunk_number,
  57                                    const SBPrefix& host_prefix,
  58                                    const SBPrefix& url_prefix) {
  59   chunk->chunk_number = chunk_number;
  60   chunk->is_add = true;
  61   SBChunkHost host;
  62   host.host = host_prefix;
  63   host.entry = SBEntry::Create(SBEntry::ADD_PREFIX, 1);
  64   host.entry->set_chunk_id(chunk->chunk_number);
  65   host.entry->SetPrefixAt(0, url_prefix);
  66   chunk->hosts.push_back(host);
  67 }
  68
  69 // A helper function that appends one AddChunkHost to chunk with
  70 // one url for prefix.
  71 void InsertAddChunkHostPrefixUrl(SBChunk* chunk,
  72                                  int chunk_number,
  73                                  const std::string& host_name,
  74                                  const std::string& url) {
  75   InsertAddChunkHostPrefixValue(chunk, chunk_number,
  76                                 Sha256Prefix(host_name),
  77                                 Sha256Prefix(url));
  78 }
  79
  80 // Same as InsertAddChunkHostPrefixUrl, but with full hashes.
  81 void InsertAddChunkHostFullHashes(SBChunk* chunk,
  82                                   int chunk_number,
  83                                   const std::string& host_name,
  84                                   const std::string& url) {
  85   chunk->chunk_number = chunk_number;
  86   chunk->is_add = true;
  87   SBChunkHost host;
  88   host.host = Sha256Prefix(host_name);
  89   host.entry = SBEntry::Create(SBEntry::ADD_FULL_HASH, 1);
  90   host.entry->set_chunk_id(chunk->chunk_number);
  91   host.entry->SetFullHashAt(0, Sha256Hash(url));
  92   chunk->hosts.push_back(host);
  93 }
  94
  95 void InsertAddChunkFullHash(SBChunk* chunk,
  96                             int chunk_number,
  97                             const std::string& ip_str,
  98                             size_t prefix_size) {
  99   const std::string full_hash_str = HashedIpPrefix(ip_str, prefix_size);
 100   EXPECT_EQ(sizeof(SBFullHash), full_hash_str.size());
 101   SBFullHash full_hash;
 102   std::memcpy(&(full_hash.full_hash), full_hash_str.data(), sizeof(SBFullHash));
 103
 104   chunk->chunk_number = chunk_number;
 105   chunk->is_add = true;
 106   SBChunkHost host;
 107   host.host = full_hash.prefix;
 108   host.entry = SBEntry::Create(SBEntry::ADD_FULL_HASH, 1);
 109   host.entry->set_chunk_id(chunk->chunk_number);
 110   host.entry->SetFullHashAt(0, full_hash);
 111   chunk->hosts.push_back(host);
 112 }
 113
 114 // Same as InsertAddChunkHostPrefixUrl, but with two urls for prefixes.
 115 void InsertAddChunkHost2PrefixUrls(SBChunk* chunk,
 116                                    int chunk_number,
 117                                    const std::string& host_name,
 118                                    const std::string& url1,
 119                                    const std::string& url2) {
 120   chunk->chunk_number = chunk_number;
 121   chunk->is_add = true;
 122   SBChunkHost host;
 123   host.host = Sha256Prefix(host_name);
 124   host.entry = SBEntry::Create(SBEntry::ADD_PREFIX, 2);
 125   host.entry->set_chunk_id(chunk->chunk_number);
 126   host.entry->SetPrefixAt(0, Sha256Prefix(url1));
 127   host.entry->SetPrefixAt(1, Sha256Prefix(url2));
 128   chunk->hosts.push_back(host);
 129 }
 130
 131 // Same as InsertAddChunkHost2PrefixUrls, but with full hashes.
 132 void InsertAddChunkHost2FullHashes(SBChunk* chunk,
 133                                    int chunk_number,
 134                                    const std::string& host_name,
 135                                    const std::string& url1,
 136                                    const std::string& url2) {
 137   chunk->chunk_number = chunk_number;
 138   chunk->is_add = true;
 139   SBChunkHost host;
 140   host.host = Sha256Prefix(host_name);
 141   host.entry = SBEntry::Create(SBEntry::ADD_FULL_HASH, 2);
 142   host.entry->set_chunk_id(chunk->chunk_number);
 143   host.entry->SetFullHashAt(0, Sha256Hash(url1));
 144   host.entry->SetFullHashAt(1, Sha256Hash(url2));
 145   chunk->hosts.push_back(host);
 146 }
 147
 148 // Same as InsertSubChunkHostPrefixUrl, but with pre-computed
 149 // prefix values.
 150 void InsertSubChunkHostPrefixValue(SBChunk* chunk,
 151                                    int chunk_number,
 152                                    int chunk_id_to_sub,
 153                                    const SBPrefix& host_prefix,
 154                                    const SBPrefix& url_prefix) {
 155   chunk->chunk_number = chunk_number;
 156   chunk->is_add = false;
 157   SBChunkHost host;
 158   host.host = host_prefix;
 159   host.entry = SBEntry::Create(SBEntry::SUB_PREFIX, 1);
 160   host.entry->set_chunk_id(chunk->chunk_number);
 161   host.entry->SetChunkIdAtPrefix(0, chunk_id_to_sub);
 162   host.entry->SetPrefixAt(0, url_prefix);
 163   chunk->hosts.push_back(host);
 164 }
 165
 166 // A helper function that adds one SubChunkHost to chunk with
 167 // one url for prefix.
 168 void InsertSubChunkHostPrefixUrl(SBChunk* chunk,
 169                                  int chunk_number,
 170                                  int chunk_id_to_sub,
 171                                  const std::string& host_name,
 172                                  const std::string& url) {
 173   InsertSubChunkHostPrefixValue(chunk, chunk_number,
 174                                 chunk_id_to_sub,
 175                                 Sha256Prefix(host_name),
 176                                 Sha256Prefix(url));
 177 }
 178
 179 // Same as InsertSubChunkHostPrefixUrl, but with two urls for prefixes.
 180 void InsertSubChunkHost2PrefixUrls(SBChunk* chunk,
 181                                    int chunk_number,
 182                                    int chunk_id_to_sub,
 183                                    const std::string& host_name,
 184                                    const std::string& url1,
 185                                    const std::string& url2) {
 186   chunk->chunk_number = chunk_number;
 187   chunk->is_add = false;
 188   SBChunkHost host;
 189   host.host = Sha256Prefix(host_name);
 190   host.entry = SBEntry::Create(SBEntry::SUB_PREFIX, 2);
 191   host.entry->set_chunk_id(chunk->chunk_number);
 192   host.entry->SetPrefixAt(0, Sha256Prefix(url1));
 193   host.entry->SetChunkIdAtPrefix(0, chunk_id_to_sub);
 194   host.entry->SetPrefixAt(1, Sha256Prefix(url2));
 195   host.entry->SetChunkIdAtPrefix(1, chunk_id_to_sub);
 196   chunk->hosts.push_back(host);
 197 }
 198
 199 // Same as InsertSubChunkHost2PrefixUrls, but with full hashes.
 200 void InsertSubChunkHostFullHash(SBChunk* chunk,
 201                                 int chunk_number,
 202                                 int chunk_id_to_sub,
 203                                 const std::string& host_name,
 204                                 const std::string& url) {
 205   chunk->chunk_number = chunk_number;
 206   chunk->is_add = false;
 207   SBChunkHost host;
 208   host.host = Sha256Prefix(host_name);
 209   host.entry = SBEntry::Create(SBEntry::SUB_FULL_HASH, 2);
 210   host.entry->set_chunk_id(chunk->chunk_number);
 211   host.entry->SetFullHashAt(0, Sha256Hash(url));
 212   host.entry->SetChunkIdAtPrefix(0, chunk_id_to_sub);
 213   chunk->hosts.push_back(host);
 214 }
 215
 216 // Prevent DCHECK from killing tests.
 217 // TODO(shess): Pawel disputes the use of this, so the test which uses
 218 // it is DISABLED.  http://crbug.com/56448
 219 class ScopedLogMessageIgnorer {
 220  public:
 221   ScopedLogMessageIgnorer() {
 222     logging::SetLogMessageHandler(&LogMessageIgnorer);
 223   }
 224   ~ScopedLogMessageIgnorer() {
 225     // TODO(shess): Would be better to verify whether anyone else
 226     // changed it, and then restore it to the previous value.
 227     logging::SetLogMessageHandler(NULL);
 228   }
 229
 230  private:
 231   static bool LogMessageIgnorer(int severity, const char* file, int line,
 232       size_t message_start, const std::string& str) {
 233     // Intercept FATAL, strip the stack backtrace, and log it without
 234     // the crash part.
 235     if (severity == logging::LOG_FATAL) {
 236       size_t newline = str.find('\n');
 237       if (newline != std::string::npos) {
 238         const std::string msg = str.substr(0, newline + 1);
 239         fprintf(stderr, "%s", msg.c_str());
 240         fflush(stderr);
 241       }
 242       return true;
 243     }
 244
 245     return false;
 246   }
 247 };
 248
 249 }  // namespace
 250
 251 class SafeBrowsingDatabaseTest : public PlatformTest {
 252  public:
 253   virtual void SetUp() {
 254     PlatformTest::SetUp();
 255
 256     // Setup a database in a temporary directory.
 257     ASSERT_TRUE(temp_dir_.CreateUniqueTempDir());
 258     database_.reset(new SafeBrowsingDatabaseNew);
 259     database_filename_ =
 260         temp_dir_.path().AppendASCII("SafeBrowsingTestDatabase");
 261     database_->Init(database_filename_);
 262   }
 263
 264   virtual void TearDown() {
 265     database_.reset();
 266
 267     PlatformTest::TearDown();
 268   }
 269
 270   void GetListsInfo(std::vector<SBListChunkRanges>* lists) {
 271     lists->clear();
 272     EXPECT_TRUE(database_->UpdateStarted(lists));
 273     database_->UpdateFinished(true);
 274   }
 275
 276   // Helper function to do an AddDel or SubDel command.
 277   void DelChunk(const std::string& list,
 278                 int chunk_id,
 279                 bool is_sub_del) {
 280     std::vector<SBChunkDelete> deletes;
 281     SBChunkDelete chunk_delete;
 282     chunk_delete.list_name = list;
 283     chunk_delete.is_sub_del = is_sub_del;
 284     chunk_delete.chunk_del.push_back(ChunkRange(chunk_id));
 285     deletes.push_back(chunk_delete);
 286     database_->DeleteChunks(deletes);
 287   }
 288
 289   void AddDelChunk(const std::string& list, int chunk_id) {
 290     DelChunk(list, chunk_id, false);
 291   }
 292
 293   void SubDelChunk(const std::string& list, int chunk_id) {
 294     DelChunk(list, chunk_id, true);
 295   }
 296
 297   // Utility function for setting up the database for the caching test.
 298   void PopulateDatabaseForCacheTest();
 299
 300   scoped_ptr<SafeBrowsingDatabaseNew> database_;
 301   base::FilePath database_filename_;
 302   base::ScopedTempDir temp_dir_;
 303 };
 304
 305 // Tests retrieving list name information.
 306 TEST_F(SafeBrowsingDatabaseTest, ListNameForBrowse) {
 307   SBChunkList chunks;
 308   SBChunk chunk;
 309
 310   InsertAddChunkHostPrefixUrl(&chunk, 1, "www.evil.com/",
 311                               "www.evil.com/malware.html");
 312   chunks.clear();
 313   chunks.push_back(chunk);
 314   std::vector<SBListChunkRanges> lists;
 315   EXPECT_TRUE(database_->UpdateStarted(&lists));
 316   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
 317
 318   chunk.hosts.clear();
 319   InsertAddChunkHostPrefixUrl(&chunk, 2, "www.foo.com/",
 320                               "www.foo.com/malware.html");
 321   chunks.clear();
 322   chunks.push_back(chunk);
 323   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
 324
 325   chunk.hosts.clear();
 326   InsertAddChunkHostPrefixUrl(&chunk, 3, "www.whatever.com/",
 327                               "www.whatever.com/malware.html");
 328   chunks.clear();
 329   chunks.push_back(chunk);
 330   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
 331   database_->UpdateFinished(true);
 332
 333   GetListsInfo(&lists);
 334   EXPECT_TRUE(lists[0].name == safe_browsing_util::kMalwareList);
 335   EXPECT_EQ(lists[0].adds, "1-3");
 336   EXPECT_TRUE(lists[0].subs.empty());
 337
 338   // Insert a malware sub chunk.
 339   chunk.hosts.clear();
 340   InsertSubChunkHostPrefixUrl(&chunk, 7, 19, "www.subbed.com/",
 341                               "www.subbed.com/noteveil1.html");
 342   chunks.clear();
 343   chunks.push_back(chunk);
 344
 345   EXPECT_TRUE(database_->UpdateStarted(&lists));
 346   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
 347   database_->UpdateFinished(true);
 348
 349   GetListsInfo(&lists);
 350   EXPECT_TRUE(lists[0].name == safe_browsing_util::kMalwareList);
 351   EXPECT_EQ(lists[0].adds, "1-3");
 352   EXPECT_EQ(lists[0].subs, "7");
 353   if (lists.size() == 2) {
 354     // Old style database won't have the second entry since it creates the lists
 355     // when it receives an update containing that list. The filter-based
 356     // database has these values hard coded.
 357     EXPECT_TRUE(lists[1].name == safe_browsing_util::kPhishingList);
 358     EXPECT_TRUE(lists[1].adds.empty());
 359     EXPECT_TRUE(lists[1].subs.empty());
 360   }
 361
 362   // Add a phishing add chunk.
 363   chunk.hosts.clear();
 364   InsertAddChunkHostPrefixUrl(&chunk, 47, "www.evil.com/",
 365                               "www.evil.com/phishing.html");
 366   chunks.clear();
 367   chunks.push_back(chunk);
 368   EXPECT_TRUE(database_->UpdateStarted(&lists));
 369   database_->InsertChunks(safe_browsing_util::kPhishingList, chunks);
 370
 371   // Insert some phishing sub chunks.
 372   chunk.hosts.clear();
 373   InsertSubChunkHostPrefixUrl(&chunk, 200, 1999, "www.phishy.com/",
 374                               "www.phishy.com/notevil1.html");
 375   chunks.clear();
 376   chunks.push_back(chunk);
 377   database_->InsertChunks(safe_browsing_util::kPhishingList, chunks);
 378
 379   chunk.hosts.clear();
 380   InsertSubChunkHostPrefixUrl(&chunk, 201, 1999, "www.phishy2.com/",
 381                               "www.phishy2.com/notevil1.html");
 382   chunks.clear();
 383   chunks.push_back(chunk);
 384   database_->InsertChunks(safe_browsing_util::kPhishingList, chunks);
 385   database_->UpdateFinished(true);
 386
 387   GetListsInfo(&lists);
 388   EXPECT_TRUE(lists[0].name == safe_browsing_util::kMalwareList);
 389   EXPECT_EQ(lists[0].adds, "1-3");
 390   EXPECT_EQ(lists[0].subs, "7");
 391   EXPECT_TRUE(lists[1].name == safe_browsing_util::kPhishingList);
 392   EXPECT_EQ(lists[1].adds, "47");
 393   EXPECT_EQ(lists[1].subs, "200-201");
 394 }
 395
 396 TEST_F(SafeBrowsingDatabaseTest, ListNameForBrowseAndDownload) {
 397   database_.reset();
 398   base::MessageLoop loop;
 399   SafeBrowsingStoreFile* browse_store = new SafeBrowsingStoreFile();
 400   SafeBrowsingStoreFile* download_store = new SafeBrowsingStoreFile();
 401   SafeBrowsingStoreFile* csd_whitelist_store = new SafeBrowsingStoreFile();
 402   SafeBrowsingStoreFile* download_whitelist_store = new SafeBrowsingStoreFile();
 403   SafeBrowsingStoreFile* extension_blacklist_store =
 404       new SafeBrowsingStoreFile();
 405   SafeBrowsingStoreFile* ip_blacklist_store = new SafeBrowsingStoreFile();
 406   database_.reset(new SafeBrowsingDatabaseNew(browse_store,
 407                                               download_store,
 408                                               csd_whitelist_store,
 409                                               download_whitelist_store,
 410                                               extension_blacklist_store,
 411                                               NULL,
 412                                               ip_blacklist_store));
 413   database_->Init(database_filename_);
 414
 415   SBChunkList chunks;
 416   SBChunk chunk;
 417
 418   // Insert malware, phish, binurl and bindownload add chunks.
 419   InsertAddChunkHostPrefixUrl(&chunk, 1, "www.evil.com/",
 420                               "www.evil.com/malware.html");
 421   chunks.push_back(chunk);
 422   std::vector<SBListChunkRanges> lists;
 423   EXPECT_TRUE(database_->UpdateStarted(&lists));
 424   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
 425
 426   chunk.hosts.clear();
 427   InsertAddChunkHostPrefixUrl(&chunk, 2, "www.foo.com/",
 428                               "www.foo.com/malware.html");
 429   chunks.clear();
 430   chunks.push_back(chunk);
 431   database_->InsertChunks(safe_browsing_util::kPhishingList, chunks);
 432
 433   chunk.hosts.clear();
 434   InsertAddChunkHostPrefixUrl(&chunk, 3, "www.whatever.com/",
 435                               "www.whatever.com/download.html");
 436   chunks.clear();
 437   chunks.push_back(chunk);
 438   database_->InsertChunks(safe_browsing_util::kBinUrlList, chunks);
 439
 440   chunk.hosts.clear();
 441   InsertAddChunkHostPrefixUrl(&chunk, 4, "www.forhash.com/",
 442                               "www.forhash.com/download.html");
 443   chunks.clear();
 444   chunks.push_back(chunk);
 445   database_->InsertChunks(safe_browsing_util::kBinHashList, chunks);
 446
 447   chunk.hosts.clear();
 448   InsertAddChunkHostFullHashes(&chunk, 5, "www.forwhitelist.com/",
 449                                "www.forwhitelist.com/a.html");
 450   chunks.clear();
 451   chunks.push_back(chunk);
 452   database_->InsertChunks(safe_browsing_util::kCsdWhiteList, chunks);
 453
 454   chunk.hosts.clear();
 455   InsertAddChunkHostFullHashes(&chunk, 6, "www.download.com/",
 456                                "www.download.com/");
 457
 458   chunks.clear();
 459   chunks.push_back(chunk);
 460   database_->InsertChunks(safe_browsing_util::kDownloadWhiteList, chunks);
 461
 462   chunk.hosts.clear();
 463   InsertAddChunkHostFullHashes(&chunk, 8,
 464                                "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
 465                                "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
 466
 467   chunks.clear();
 468   chunks.push_back(chunk);
 469   database_->InsertChunks(safe_browsing_util::kExtensionBlacklist, chunks);
 470
 471   chunk.hosts.clear();
 472   InsertAddChunkFullHash(&chunk, 9, "::ffff:192.168.1.0", 120);
 473
 474   chunks.clear();
 475   chunks.push_back(chunk);
 476   database_->InsertChunks(safe_browsing_util::kIPBlacklist, chunks);
 477
 478   database_->UpdateFinished(true);
 479
 480   GetListsInfo(&lists);
 481   ASSERT_EQ(7U, lists.size());
 482   EXPECT_TRUE(lists[0].name == safe_browsing_util::kMalwareList);
 483   EXPECT_EQ(lists[0].adds, "1");
 484   EXPECT_TRUE(lists[0].subs.empty());
 485   EXPECT_TRUE(lists[1].name == safe_browsing_util::kPhishingList);
 486   EXPECT_EQ(lists[1].adds, "2");
 487   EXPECT_TRUE(lists[1].subs.empty());
 488   EXPECT_TRUE(lists[2].name == safe_browsing_util::kBinUrlList);
 489   EXPECT_EQ(lists[2].adds, "3");
 490   EXPECT_TRUE(lists[2].subs.empty());
 491   // kBinHashList is ignored.  (http://crbug.com/108130)
 492   EXPECT_TRUE(lists[3].name == safe_browsing_util::kCsdWhiteList);
 493   EXPECT_EQ(lists[3].adds, "5");
 494   EXPECT_TRUE(lists[3].subs.empty());
 495   EXPECT_TRUE(lists[4].name == safe_browsing_util::kDownloadWhiteList);
 496   EXPECT_EQ(lists[4].adds, "6");
 497   EXPECT_TRUE(lists[4].subs.empty());
 498   EXPECT_TRUE(lists[5].name == safe_browsing_util::kExtensionBlacklist);
 499   EXPECT_EQ(lists[5].adds, "8");
 500   EXPECT_TRUE(lists[5].subs.empty());
 501   EXPECT_TRUE(lists[6].name == safe_browsing_util::kIPBlacklist);
 502   EXPECT_EQ(lists[6].adds, "9");
 503   EXPECT_TRUE(lists[6].subs.empty());
 504
 505   database_.reset();
 506 }
 507
 508 // Checks database reading and writing for browse.
 509 TEST_F(SafeBrowsingDatabaseTest, BrowseDatabase) {
 510   SBChunkList chunks;
 511   SBChunk chunk;
 512
 513   // Add a simple chunk with one hostkey.
 514   InsertAddChunkHost2PrefixUrls(&chunk, 1, "www.evil.com/",
 515                                 "www.evil.com/phishing.html",
 516                                 "www.evil.com/malware.html");
 517   chunks.push_back(chunk);
 518   std::vector<SBListChunkRanges> lists;
 519   EXPECT_TRUE(database_->UpdateStarted(&lists));
 520   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
 521
 522   chunk.hosts.clear();
 523   InsertAddChunkHost2PrefixUrls(&chunk, 2, "www.evil.com/",
 524                                 "www.evil.com/notevil1.html",
 525                                 "www.evil.com/notevil2.html");
 526   InsertAddChunkHost2PrefixUrls(&chunk, 2, "www.good.com/",
 527                                 "www.good.com/good1.html",
 528                                 "www.good.com/good2.html");
 529   chunks.clear();
 530   chunks.push_back(chunk);
 531   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
 532
 533   // and a chunk with an IP-based host
 534   chunk.hosts.clear();
 535   InsertAddChunkHostPrefixUrl(&chunk, 3, "192.168.0.1/",
 536                               "192.168.0.1/malware.html");
 537   chunks.clear();
 538   chunks.push_back(chunk);
 539   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
 540   database_->UpdateFinished(true);
 541
 542   // Make sure they were added correctly.
 543   GetListsInfo(&lists);
 544   EXPECT_TRUE(lists[0].name == safe_browsing_util::kMalwareList);
 545   EXPECT_EQ(lists[0].adds, "1-3");
 546   EXPECT_TRUE(lists[0].subs.empty());
 547
 548   const Time now = Time::Now();
 549   std::vector<SBFullHashResult> full_hashes;
 550   std::vector<SBPrefix> prefix_hits;
 551   std::string matching_list;
 552   EXPECT_TRUE(database_->ContainsBrowseUrl(
 553       GURL("http://www.evil.com/phishing.html"),
 554       &matching_list, &prefix_hits,
 555       &full_hashes, now));
 556   EXPECT_EQ(prefix_hits[0], Sha256Prefix("www.evil.com/phishing.html"));
 557   EXPECT_EQ(prefix_hits.size(), 1U);
 558
 559   EXPECT_TRUE(database_->ContainsBrowseUrl(
 560       GURL("http://www.evil.com/malware.html"),
 561       &matching_list, &prefix_hits,
 562       &full_hashes, now));
 563
 564   EXPECT_TRUE(database_->ContainsBrowseUrl(
 565       GURL("http://www.evil.com/notevil1.html"),
 566       &matching_list, &prefix_hits,
 567       &full_hashes, now));
 568
 569   EXPECT_TRUE(database_->ContainsBrowseUrl(
 570       GURL("http://www.evil.com/notevil2.html"),
 571       &matching_list, &prefix_hits,
 572       &full_hashes, now));
 573
 574   EXPECT_TRUE(database_->ContainsBrowseUrl(
 575       GURL("http://www.good.com/good1.html"),
 576       &matching_list, &prefix_hits,
 577       &full_hashes, now));
 578
 579   EXPECT_TRUE(database_->ContainsBrowseUrl(
 580       GURL("http://www.good.com/good2.html"),
 581       &matching_list, &prefix_hits,
 582       &full_hashes, now));
 583
 584   EXPECT_TRUE(database_->ContainsBrowseUrl(
 585       GURL("http://192.168.0.1/malware.html"),
 586       &matching_list, &prefix_hits,
 587       &full_hashes, now));
 588
 589   EXPECT_FALSE(database_->ContainsBrowseUrl(
 590       GURL("http://www.evil.com/"),
 591       &matching_list, &prefix_hits,
 592       &full_hashes, now));
 593   EXPECT_TRUE(prefix_hits.empty());
 594
 595   EXPECT_FALSE(database_->ContainsBrowseUrl(
 596       GURL("http://www.evil.com/robots.txt"),
 597       &matching_list, &prefix_hits,
 598       &full_hashes, now));
 599
 600   // Attempt to re-add the first chunk (should be a no-op).
 601   // see bug: http://code.google.com/p/chromium/issues/detail?id=4522
 602   chunk.hosts.clear();
 603   InsertAddChunkHost2PrefixUrls(&chunk, 1, "www.evil.com/",
 604                                 "www.evil.com/phishing.html",
 605                                 "www.evil.com/malware.html");
 606   chunks.clear();
 607   chunks.push_back(chunk);
 608   EXPECT_TRUE(database_->UpdateStarted(&lists));
 609   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
 610   database_->UpdateFinished(true);
 611
 612   GetListsInfo(&lists);
 613   EXPECT_TRUE(lists[0].name == safe_browsing_util::kMalwareList);
 614   EXPECT_EQ(lists[0].adds, "1-3");
 615   EXPECT_TRUE(lists[0].subs.empty());
 616
 617   // Test removing a single prefix from the add chunk.
 618   chunk.hosts.clear();
 619   InsertSubChunkHostPrefixUrl(&chunk, 4, 2, "www.evil.com/",
 620                               "www.evil.com/notevil1.html");
 621   chunks.clear();
 622   chunks.push_back(chunk);
 623   EXPECT_TRUE(database_->UpdateStarted(&lists));
 624   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
 625
 626   database_->UpdateFinished(true);
 627
 628   EXPECT_TRUE(database_->ContainsBrowseUrl(
 629       GURL("http://www.evil.com/phishing.html"),
 630       &matching_list, &prefix_hits,
 631       &full_hashes, now));
 632   EXPECT_EQ(prefix_hits[0], Sha256Prefix("www.evil.com/phishing.html"));
 633   EXPECT_EQ(prefix_hits.size(), 1U);
 634
 635   EXPECT_FALSE(database_->ContainsBrowseUrl(
 636       GURL("http://www.evil.com/notevil1.html"),
 637       &matching_list, &prefix_hits,
 638       &full_hashes, now));
 639   EXPECT_TRUE(prefix_hits.empty());
 640
 641   EXPECT_TRUE(database_->ContainsBrowseUrl(
 642       GURL("http://www.evil.com/notevil2.html"),
 643       &matching_list, &prefix_hits,
 644       &full_hashes, now));
 645
 646   EXPECT_TRUE(database_->ContainsBrowseUrl(
 647       GURL("http://www.good.com/good1.html"),
 648       &matching_list, &prefix_hits,
 649       &full_hashes, now));
 650
 651   EXPECT_TRUE(database_->ContainsBrowseUrl(
 652       GURL("http://www.good.com/good2.html"),
 653       &matching_list, &prefix_hits,
 654       &full_hashes, now));
 655
 656   GetListsInfo(&lists);
 657   EXPECT_TRUE(lists[0].name == safe_browsing_util::kMalwareList);
 658   EXPECT_EQ(lists[0].subs, "4");
 659
 660   // Test the same sub chunk again.  This should be a no-op.
 661   // see bug: http://code.google.com/p/chromium/issues/detail?id=4522
 662   chunk.hosts.clear();
 663   InsertSubChunkHostPrefixUrl(&chunk, 4, 2, "www.evil.com/",
 664                               "www.evil.com/notevil1.html");
 665   chunks.clear();
 666   chunks.push_back(chunk);
 667
 668   EXPECT_TRUE(database_->UpdateStarted(&lists));
 669   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
 670   database_->UpdateFinished(true);
 671
 672   GetListsInfo(&lists);
 673   EXPECT_TRUE(lists[0].name == safe_browsing_util::kMalwareList);
 674   EXPECT_EQ(lists[0].subs, "4");
 675
 676   // Test removing all the prefixes from an add chunk.
 677   EXPECT_TRUE(database_->UpdateStarted(&lists));
 678   AddDelChunk(safe_browsing_util::kMalwareList, 2);
 679   database_->UpdateFinished(true);
 680
 681   EXPECT_FALSE(database_->ContainsBrowseUrl(
 682       GURL("http://www.evil.com/notevil2.html"),
 683       &matching_list, &prefix_hits,
 684       &full_hashes, now));
 685
 686   EXPECT_FALSE(database_->ContainsBrowseUrl(
 687       GURL("http://www.good.com/good1.html"),
 688       &matching_list, &prefix_hits,
 689       &full_hashes, now));
 690
 691   EXPECT_FALSE(database_->ContainsBrowseUrl(
 692       GURL("http://www.good.com/good2.html"),
 693       &matching_list, &prefix_hits,
 694       &full_hashes, now));
 695
 696   GetListsInfo(&lists);
 697   EXPECT_TRUE(lists[0].name == safe_browsing_util::kMalwareList);
 698   EXPECT_EQ(lists[0].adds, "1,3");
 699   EXPECT_EQ(lists[0].subs, "4");
 700
 701   // The adddel command exposed a bug in the transaction code where any
 702   // transaction after it would fail.  Add a dummy entry and remove it to
 703   // make sure the transcation works fine.
 704   chunk.hosts.clear();
 705   InsertAddChunkHostPrefixUrl(&chunk, 44, "www.redherring.com/",
 706                               "www.redherring.com/index.html");
 707   chunks.clear();
 708   chunks.push_back(chunk);
 709   EXPECT_TRUE(database_->UpdateStarted(&lists));
 710   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
 711
 712   // Now remove the dummy entry.  If there are any problems with the
 713   // transactions, asserts will fire.
 714   AddDelChunk(safe_browsing_util::kMalwareList, 44);
 715
 716   // Test the subdel command.
 717   SubDelChunk(safe_browsing_util::kMalwareList, 4);
 718   database_->UpdateFinished(true);
 719
 720   GetListsInfo(&lists);
 721   EXPECT_TRUE(lists[0].name == safe_browsing_util::kMalwareList);
 722   EXPECT_EQ(lists[0].adds, "1,3");
 723   EXPECT_EQ(lists[0].subs, "");
 724
 725   // Test a sub command coming in before the add.
 726   chunk.hosts.clear();
 727   InsertSubChunkHost2PrefixUrls(&chunk, 5, 10,
 728                                 "www.notevilanymore.com/",
 729                                 "www.notevilanymore.com/index.html",
 730                                 "www.notevilanymore.com/good.html");
 731   chunks.clear();
 732   chunks.push_back(chunk);
 733   EXPECT_TRUE(database_->UpdateStarted(&lists));
 734   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
 735   database_->UpdateFinished(true);
 736
 737   EXPECT_FALSE(database_->ContainsBrowseUrl(
 738       GURL("http://www.notevilanymore.com/index.html"),
 739       &matching_list, &prefix_hits, &full_hashes, now));
 740
 741   // Now insert the tardy add chunk and we don't expect them to appear
 742   // in database because of the previous sub chunk.
 743   chunk.hosts.clear();
 744   InsertAddChunkHost2PrefixUrls(&chunk, 10, "www.notevilanymore.com/",
 745                                 "www.notevilanymore.com/index.html",
 746                                 "www.notevilanymore.com/good.html");
 747   chunks.clear();
 748   chunks.push_back(chunk);
 749   EXPECT_TRUE(database_->UpdateStarted(&lists));
 750   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
 751   database_->UpdateFinished(true);
 752
 753   EXPECT_FALSE(database_->ContainsBrowseUrl(
 754       GURL("http://www.notevilanymore.com/index.html"),
 755       &matching_list, &prefix_hits, &full_hashes, now));
 756
 757   EXPECT_FALSE(database_->ContainsBrowseUrl(
 758       GURL("http://www.notevilanymore.com/good.html"),
 759       &matching_list, &prefix_hits, &full_hashes, now));
 760 }
 761
 762
 763 // Test adding zero length chunks to the database.
 764 TEST_F(SafeBrowsingDatabaseTest, ZeroSizeChunk) {
 765   SBChunkList chunks;
 766   SBChunk chunk;
 767
 768   // Populate with a couple of normal chunks.
 769   InsertAddChunkHost2PrefixUrls(&chunk, 1, "www.test.com/",
 770                                 "www.test.com/test1.html",
 771                                 "www.test.com/test2.html");
 772   chunks.clear();
 773   chunks.push_back(chunk);
 774
 775   chunk.hosts.clear();
 776   InsertAddChunkHost2PrefixUrls(&chunk, 10, "www.random.com/",
 777                                 "www.random.com/random1.html",
 778                                 "www.random.com/random2.html");
 779   chunks.push_back(chunk);
 780
 781   std::vector<SBListChunkRanges> lists;
 782   EXPECT_TRUE(database_->UpdateStarted(&lists));
 783   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
 784   database_->UpdateFinished(true);
 785
 786   // Add an empty ADD and SUB chunk.
 787   GetListsInfo(&lists);
 788   EXPECT_EQ(lists[0].adds, "1,10");
 789
 790   SBChunk empty_chunk;
 791   empty_chunk.chunk_number = 19;
 792   empty_chunk.is_add = true;
 793   chunks.clear();
 794   chunks.push_back(empty_chunk);
 795   EXPECT_TRUE(database_->UpdateStarted(&lists));
 796   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
 797   chunks.clear();
 798   empty_chunk.chunk_number = 7;
 799   empty_chunk.is_add = false;
 800   chunks.push_back(empty_chunk);
 801   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
 802   database_->UpdateFinished(true);
 803
 804   GetListsInfo(&lists);
 805   EXPECT_EQ(lists[0].adds, "1,10,19");
 806   EXPECT_EQ(lists[0].subs, "7");
 807
 808   // Add an empty chunk along with a couple that contain data. This should
 809   // result in the chunk range being reduced in size.
 810   empty_chunk.hosts.clear();
 811   InsertAddChunkHostPrefixUrl(&empty_chunk, 20, "www.notempy.com/",
 812                               "www.notempty.com/full1.html");
 813   chunks.clear();
 814   chunks.push_back(empty_chunk);
 815
 816   empty_chunk.chunk_number = 21;
 817   empty_chunk.is_add = true;
 818   empty_chunk.hosts.clear();
 819   chunks.push_back(empty_chunk);
 820
 821   empty_chunk.hosts.clear();
 822   InsertAddChunkHostPrefixUrl(&empty_chunk, 22, "www.notempy.com/",
 823                               "www.notempty.com/full2.html");
 824   chunks.push_back(empty_chunk);
 825
 826   EXPECT_TRUE(database_->UpdateStarted(&lists));
 827   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
 828   database_->UpdateFinished(true);
 829
 830   const Time now = Time::Now();
 831   std::vector<SBFullHashResult> full_hashes;
 832   std::vector<SBPrefix> prefix_hits;
 833   std::string matching_list;
 834   EXPECT_TRUE(database_->ContainsBrowseUrl(
 835       GURL("http://www.notempty.com/full1.html"),
 836       &matching_list, &prefix_hits,
 837       &full_hashes, now));
 838   EXPECT_TRUE(database_->ContainsBrowseUrl(
 839       GURL("http://www.notempty.com/full2.html"),
 840       &matching_list, &prefix_hits,
 841       &full_hashes, now));
 842
 843   GetListsInfo(&lists);
 844   EXPECT_EQ(lists[0].adds, "1,10,19-22");
 845   EXPECT_EQ(lists[0].subs, "7");
 846
 847   // Handle AddDel and SubDel commands for empty chunks.
 848   EXPECT_TRUE(database_->UpdateStarted(&lists));
 849   AddDelChunk(safe_browsing_util::kMalwareList, 21);
 850   database_->UpdateFinished(true);
 851
 852   GetListsInfo(&lists);
 853   EXPECT_EQ(lists[0].adds, "1,10,19-20,22");
 854   EXPECT_EQ(lists[0].subs, "7");
 855
 856   EXPECT_TRUE(database_->UpdateStarted(&lists));
 857   SubDelChunk(safe_browsing_util::kMalwareList, 7);
 858   database_->UpdateFinished(true);
 859
 860   GetListsInfo(&lists);
 861   EXPECT_EQ(lists[0].adds, "1,10,19-20,22");
 862   EXPECT_EQ(lists[0].subs, "");
 863 }
 864
 865 // Utility function for setting up the database for the caching test.
 866 void SafeBrowsingDatabaseTest::PopulateDatabaseForCacheTest() {
 867   SBChunkList chunks;
 868   SBChunk chunk;
 869   // Add a simple chunk with one hostkey and cache it.
 870   InsertAddChunkHost2PrefixUrls(&chunk, 1, "www.evil.com/",
 871                                 "www.evil.com/phishing.html",
 872                                 "www.evil.com/malware.html");
 873   chunks.push_back(chunk);
 874
 875   std::vector<SBListChunkRanges> lists;
 876   EXPECT_TRUE(database_->UpdateStarted(&lists));
 877   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
 878   database_->UpdateFinished(true);
 879
 880   // Add the GetHash results to the cache.
 881   SBFullHashResult full_hash;
 882   full_hash.hash = Sha256Hash("www.evil.com/phishing.html");
 883   full_hash.list_name = safe_browsing_util::kMalwareList;
 884   full_hash.add_chunk_id = 1;
 885
 886   std::vector<SBFullHashResult> results;
 887   results.push_back(full_hash);
 888
 889   full_hash.hash = Sha256Hash("www.evil.com/malware.html");
 890   results.push_back(full_hash);
 891
 892   std::vector<SBPrefix> prefixes;
 893   database_->CacheHashResults(prefixes, results);
 894 }
 895
 896 TEST_F(SafeBrowsingDatabaseTest, HashCaching) {
 897   PopulateDatabaseForCacheTest();
 898
 899   // We should have both full hashes in the cache.
 900   EXPECT_EQ(database_->pending_browse_hashes_.size(), 2U);
 901
 902   // Test the cache lookup for the first prefix.
 903   std::string listname;
 904   std::vector<SBPrefix> prefixes;
 905   std::vector<SBFullHashResult> full_hashes;
 906   database_->ContainsBrowseUrl(
 907       GURL("http://www.evil.com/phishing.html"),
 908       &listname, &prefixes, &full_hashes, Time::Now());
 909   EXPECT_EQ(full_hashes.size(), 1U);
 910   EXPECT_TRUE(SBFullHashEq(full_hashes[0].hash,
 911                            Sha256Hash("www.evil.com/phishing.html")));
 912
 913   prefixes.clear();
 914   full_hashes.clear();
 915
 916   // Test the cache lookup for the second prefix.
 917   database_->ContainsBrowseUrl(
 918       GURL("http://www.evil.com/malware.html"),
 919       &listname, &prefixes, &full_hashes, Time::Now());
 920   EXPECT_EQ(full_hashes.size(), 1U);
 921   EXPECT_TRUE(SBFullHashEq(full_hashes[0].hash,
 922                            Sha256Hash("www.evil.com/malware.html")));
 923
 924   prefixes.clear();
 925   full_hashes.clear();
 926
 927   // Test removing a prefix via a sub chunk.
 928   SBChunk chunk;
 929   SBChunkList chunks;
 930   InsertSubChunkHostPrefixUrl(&chunk, 2, 1, "www.evil.com/",
 931                               "www.evil.com/phishing.html");
 932   chunks.push_back(chunk);
 933
 934   std::vector<SBListChunkRanges> lists;
 935   EXPECT_TRUE(database_->UpdateStarted(&lists));
 936   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
 937   database_->UpdateFinished(true);
 938
 939   // This prefix should still be there.
 940   database_->ContainsBrowseUrl(
 941       GURL("http://www.evil.com/malware.html"),
 942       &listname, &prefixes, &full_hashes, Time::Now());
 943   EXPECT_EQ(full_hashes.size(), 1U);
 944   EXPECT_TRUE(SBFullHashEq(full_hashes[0].hash,
 945                            Sha256Hash("www.evil.com/malware.html")));
 946   prefixes.clear();
 947   full_hashes.clear();
 948
 949   // This prefix should be gone.
 950   database_->ContainsBrowseUrl(
 951       GURL("http://www.evil.com/phishing.html"),
 952       &listname, &prefixes, &full_hashes, Time::Now());
 953   EXPECT_TRUE(full_hashes.empty());
 954
 955   prefixes.clear();
 956   full_hashes.clear();
 957
 958   // Test that an AddDel for the original chunk removes the last cached entry.
 959   EXPECT_TRUE(database_->UpdateStarted(&lists));
 960   AddDelChunk(safe_browsing_util::kMalwareList, 1);
 961   database_->UpdateFinished(true);
 962   database_->ContainsBrowseUrl(
 963       GURL("http://www.evil.com/malware.html"),
 964       &listname, &prefixes, &full_hashes, Time::Now());
 965   EXPECT_TRUE(full_hashes.empty());
 966   EXPECT_TRUE(database_->full_browse_hashes_.empty());
 967   EXPECT_TRUE(database_->pending_browse_hashes_.empty());
 968
 969   prefixes.clear();
 970   full_hashes.clear();
 971
 972   // Test that the cache won't return expired values. First we have to adjust
 973   // the cached entries' received time to make them older, since the database
 974   // cache insert uses Time::Now(). First, store some entries.
 975   PopulateDatabaseForCacheTest();
 976
 977   std::vector<SBAddFullHash>* hash_cache = &database_->pending_browse_hashes_;
 978   EXPECT_EQ(hash_cache->size(), 2U);
 979
 980   // Now adjust one of the entries times to be in the past.
 981   base::Time expired = base::Time::Now() - base::TimeDelta::FromMinutes(60);
 982   const SBPrefix key = Sha256Prefix("www.evil.com/malware.html");
 983   std::vector<SBAddFullHash>::iterator iter;
 984   for (iter = hash_cache->begin(); iter != hash_cache->end(); ++iter) {
 985     if (iter->full_hash.prefix == key) {
 986       iter->received = static_cast<int32>(expired.ToTimeT());
 987       break;
 988     }
 989   }
 990   EXPECT_TRUE(iter != hash_cache->end());
 991
 992   database_->ContainsBrowseUrl(
 993       GURL("http://www.evil.com/malware.html"),
 994       &listname, &prefixes, &full_hashes, expired);
 995   EXPECT_TRUE(full_hashes.empty());
 996
 997   // This entry should still exist.
 998   database_->ContainsBrowseUrl(
 999       GURL("http://www.evil.com/phishing.html"),
1000       &listname, &prefixes, &full_hashes, expired);
1001   EXPECT_EQ(full_hashes.size(), 1U);
1002
1003
1004   // Testing prefix miss caching. First, we clear out the existing database,
1005   // Since PopulateDatabaseForCacheTest() doesn't handle adding duplicate
1006   // chunks.
1007   EXPECT_TRUE(database_->UpdateStarted(&lists));
1008   AddDelChunk(safe_browsing_util::kMalwareList, 1);
1009   database_->UpdateFinished(true);
1010
1011   std::vector<SBPrefix> prefix_misses;
1012   std::vector<SBFullHashResult> empty_full_hash;
1013   prefix_misses.push_back(Sha256Prefix("http://www.bad.com/malware.html"));
1014   prefix_misses.push_back(Sha256Prefix("http://www.bad.com/phishing.html"));
1015   database_->CacheHashResults(prefix_misses, empty_full_hash);
1016
1017   // Prefixes with no full results are misses.
1018   EXPECT_EQ(database_->prefix_miss_cache_.size(), 2U);
1019
1020   // Update the database.
1021   PopulateDatabaseForCacheTest();
1022
1023   // Prefix miss cache should be cleared.
1024   EXPECT_TRUE(database_->prefix_miss_cache_.empty());
1025
1026   // Cache a GetHash miss for a particular prefix, and even though the prefix is
1027   // in the database, it is flagged as a miss so looking up the associated URL
1028   // will not succeed.
1029   prefixes.clear();
1030   full_hashes.clear();
1031   prefix_misses.clear();
1032   empty_full_hash.clear();
1033   prefix_misses.push_back(Sha256Prefix("www.evil.com/phishing.html"));
1034   database_->CacheHashResults(prefix_misses, empty_full_hash);
1035   EXPECT_FALSE(database_->ContainsBrowseUrl(
1036       GURL("http://www.evil.com/phishing.html"),
1037       &listname, &prefixes,
1038       &full_hashes, Time::Now()));
1039
1040   prefixes.clear();
1041   full_hashes.clear();
1042
1043   // Test receiving a full add chunk.
1044   chunk.hosts.clear();
1045   InsertAddChunkHost2FullHashes(&chunk, 20, "www.fullevil.com/",
1046                                 "www.fullevil.com/bad1.html",
1047                                 "www.fullevil.com/bad2.html");
1048   chunks.clear();
1049   chunks.push_back(chunk);
1050   EXPECT_TRUE(database_->UpdateStarted(&lists));
1051   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
1052   database_->UpdateFinished(true);
1053
1054   EXPECT_TRUE(database_->ContainsBrowseUrl(
1055       GURL("http://www.fullevil.com/bad1.html"),
1056       &listname, &prefixes, &full_hashes,
1057       Time::Now()));
1058   EXPECT_EQ(full_hashes.size(), 1U);
1059   EXPECT_TRUE(SBFullHashEq(full_hashes[0].hash,
1060                            Sha256Hash("www.fullevil.com/bad1.html")));
1061   prefixes.clear();
1062   full_hashes.clear();
1063
1064   EXPECT_TRUE(database_->ContainsBrowseUrl(
1065       GURL("http://www.fullevil.com/bad2.html"),
1066       &listname, &prefixes, &full_hashes,
1067       Time::Now()));
1068   EXPECT_EQ(full_hashes.size(), 1U);
1069   EXPECT_TRUE(SBFullHashEq(full_hashes[0].hash,
1070                            Sha256Hash("www.fullevil.com/bad2.html")));
1071   prefixes.clear();
1072   full_hashes.clear();
1073
1074   // Test receiving a full sub chunk, which will remove one of the full adds.
1075   chunk.hosts.clear();
1076   InsertSubChunkHostFullHash(&chunk, 200, 20,
1077                              "www.fullevil.com/",
1078                              "www.fullevil.com/bad1.html");
1079   chunks.clear();
1080   chunks.push_back(chunk);
1081   EXPECT_TRUE(database_->UpdateStarted(&lists));
1082   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
1083   database_->UpdateFinished(true);
1084
1085   EXPECT_FALSE(database_->ContainsBrowseUrl(
1086       GURL("http://www.fullevil.com/bad1.html"),
1087       &listname, &prefixes, &full_hashes,
1088       Time::Now()));
1089   EXPECT_TRUE(full_hashes.empty());
1090
1091   // There should be one remaining full add.
1092   EXPECT_TRUE(database_->ContainsBrowseUrl(
1093       GURL("http://www.fullevil.com/bad2.html"),
1094       &listname, &prefixes, &full_hashes,
1095       Time::Now()));
1096   EXPECT_EQ(full_hashes.size(), 1U);
1097   EXPECT_TRUE(SBFullHashEq(full_hashes[0].hash,
1098                            Sha256Hash("www.fullevil.com/bad2.html")));
1099   prefixes.clear();
1100   full_hashes.clear();
1101
1102   // Now test an AddDel for the remaining full add.
1103   EXPECT_TRUE(database_->UpdateStarted(&lists));
1104   AddDelChunk(safe_browsing_util::kMalwareList, 20);
1105   database_->UpdateFinished(true);
1106
1107   EXPECT_FALSE(database_->ContainsBrowseUrl(
1108       GURL("http://www.fullevil.com/bad1.html"),
1109       &listname, &prefixes, &full_hashes,
1110       Time::Now()));
1111   EXPECT_FALSE(database_->ContainsBrowseUrl(
1112       GURL("http://www.fullevil.com/bad2.html"),
1113       &listname, &prefixes, &full_hashes,
1114       Time::Now()));
1115 }
1116
1117 // Test that corrupt databases are appropriately handled, even if the
1118 // corruption is detected in the midst of the update.
1119 // TODO(shess): Disabled until ScopedLogMessageIgnorer resolved.
1120 // http://crbug.com/56448
1121 TEST_F(SafeBrowsingDatabaseTest, DISABLED_FileCorruptionHandling) {
1122   // Re-create the database in a captive message loop so that we can
1123   // influence task-posting.  Database specifically needs to the
1124   // file-backed.
1125   database_.reset();
1126   base::MessageLoop loop;
1127   SafeBrowsingStoreFile* store = new SafeBrowsingStoreFile();
1128   database_.reset(new SafeBrowsingDatabaseNew(store, NULL, NULL, NULL, NULL,
1129                                               NULL, NULL));
1130   database_->Init(database_filename_);
1131
1132   // This will cause an empty database to be created.
1133   std::vector<SBListChunkRanges> lists;
1134   EXPECT_TRUE(database_->UpdateStarted(&lists));
1135   database_->UpdateFinished(true);
1136
1137   // Create a sub chunk to insert.
1138   SBChunkList chunks;
1139   SBChunk chunk;
1140   SBChunkHost host;
1141   host.host = Sha256Prefix("www.subbed.com/");
1142   host.entry = SBEntry::Create(SBEntry::SUB_PREFIX, 1);
1143   host.entry->set_chunk_id(7);
1144   host.entry->SetChunkIdAtPrefix(0, 19);
1145   host.entry->SetPrefixAt(0, Sha256Prefix("www.subbed.com/notevil1.html"));
1146   chunk.chunk_number = 7;
1147   chunk.is_add = false;
1148   chunk.hosts.clear();
1149   chunk.hosts.push_back(host);
1150   chunks.clear();
1151   chunks.push_back(chunk);
1152
1153   // Corrupt the file by corrupting the checksum, which is not checked
1154   // until the entire table is read in |UpdateFinished()|.
1155   FILE* fp = base::OpenFile(database_filename_, "r+");
1156   ASSERT_TRUE(fp);
1157   ASSERT_NE(-1, fseek(fp, -8, SEEK_END));
1158   for (size_t i = 0; i < 8; ++i) {
1159     fputc('!', fp);
1160   }
1161   fclose(fp);
1162
1163   {
1164     // The following code will cause DCHECKs, so suppress the crashes.
1165     ScopedLogMessageIgnorer ignorer;
1166
1167     // Start an update.  The insert will fail due to corruption.
1168     EXPECT_TRUE(database_->UpdateStarted(&lists));
1169     database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
1170     database_->UpdateFinished(true);
1171
1172     // Database file still exists until the corruption handler has run.
1173     EXPECT_TRUE(base::PathExists(database_filename_));
1174
1175     // Flush through the corruption-handler task.
1176     VLOG(1) << "Expect failed check on: SafeBrowsing database reset";
1177     base::MessageLoop::current()->RunUntilIdle();
1178   }
1179
1180   // Database file should not exist.
1181   EXPECT_FALSE(base::PathExists(database_filename_));
1182
1183   // Run the update again successfully.
1184   EXPECT_TRUE(database_->UpdateStarted(&lists));
1185   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
1186   database_->UpdateFinished(true);
1187   EXPECT_TRUE(base::PathExists(database_filename_));
1188
1189   database_.reset();
1190 }
1191
1192 // Checks database reading and writing.
1193 TEST_F(SafeBrowsingDatabaseTest, ContainsDownloadUrl) {
1194   database_.reset();
1195   base::MessageLoop loop;
1196   SafeBrowsingStoreFile* browse_store = new SafeBrowsingStoreFile();
1197   SafeBrowsingStoreFile* download_store = new SafeBrowsingStoreFile();
1198   SafeBrowsingStoreFile* csd_whitelist_store = new SafeBrowsingStoreFile();
1199   database_.reset(new SafeBrowsingDatabaseNew(browse_store,
1200                                               download_store,
1201                                               csd_whitelist_store,
1202                                               NULL,
1203                                               NULL,
1204                                               NULL,
1205                                               NULL));
1206   database_->Init(database_filename_);
1207
1208   const char kEvil1Host[] = "www.evil1.com/";
1209   const char kEvil1Url1[] = "www.evil1.com/download1/";
1210   const char kEvil1Url2[] = "www.evil1.com/download2.html";
1211
1212   SBChunkList chunks;
1213   SBChunk chunk;
1214   // Add a simple chunk with one hostkey for download url list.
1215   InsertAddChunkHost2PrefixUrls(&chunk, 1, kEvil1Host,
1216                                 kEvil1Url1, kEvil1Url2);
1217   chunks.push_back(chunk);
1218   std::vector<SBListChunkRanges> lists;
1219   EXPECT_TRUE(database_->UpdateStarted(&lists));
1220   database_->InsertChunks(safe_browsing_util::kBinUrlList, chunks);
1221   database_->UpdateFinished(true);
1222
1223   std::vector<SBPrefix> prefix_hits;
1224   std::vector<GURL> urls(1);
1225
1226   urls[0] = GURL(std::string("http://") + kEvil1Url1);
1227   EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1228   ASSERT_EQ(prefix_hits.size(), 1U);
1229   EXPECT_EQ(prefix_hits[0], Sha256Prefix(kEvil1Url1));
1230
1231   urls[0] = GURL(std::string("http://") + kEvil1Url2);
1232   EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1233   ASSERT_EQ(prefix_hits.size(), 1U);
1234   EXPECT_EQ(prefix_hits[0], Sha256Prefix(kEvil1Url2));
1235
1236   urls[0] = GURL(std::string("https://") + kEvil1Url2);
1237   EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1238   ASSERT_EQ(prefix_hits.size(), 1U);
1239   EXPECT_EQ(prefix_hits[0], Sha256Prefix(kEvil1Url2));
1240
1241   urls[0] = GURL(std::string("ftp://") + kEvil1Url2);
1242   EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1243   ASSERT_EQ(prefix_hits.size(), 1U);
1244   EXPECT_EQ(prefix_hits[0], Sha256Prefix(kEvil1Url2));
1245
1246   urls[0] = GURL("http://www.randomevil.com");
1247   EXPECT_FALSE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1248
1249   // Should match with query args stripped.
1250   urls[0] = GURL(std::string("http://") + kEvil1Url2 + "?blah");
1251   EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1252   ASSERT_EQ(prefix_hits.size(), 1U);
1253   EXPECT_EQ(prefix_hits[0], Sha256Prefix(kEvil1Url2));
1254
1255   // Should match with extra path stuff and query args stripped.
1256   urls[0] = GURL(std::string("http://") + kEvil1Url1 + "foo/bar?blah");
1257   EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1258   ASSERT_EQ(prefix_hits.size(), 1U);
1259   EXPECT_EQ(prefix_hits[0], Sha256Prefix(kEvil1Url1));
1260
1261   // First hit in redirect chain is malware.
1262   urls.clear();
1263   urls.push_back(GURL(std::string("http://") + kEvil1Url1));
1264   urls.push_back(GURL("http://www.randomevil.com"));
1265   EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1266   ASSERT_EQ(prefix_hits.size(), 1U);
1267   EXPECT_EQ(prefix_hits[0], Sha256Prefix(kEvil1Url1));
1268
1269   // Middle hit in redirect chain is malware.
1270   urls.clear();
1271   urls.push_back(GURL("http://www.randomevil.com"));
1272   urls.push_back(GURL(std::string("http://") + kEvil1Url1));
1273   urls.push_back(GURL("http://www.randomevil2.com"));
1274   EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1275   ASSERT_EQ(prefix_hits.size(), 1U);
1276   EXPECT_EQ(prefix_hits[0], Sha256Prefix(kEvil1Url1));
1277
1278   // Final hit in redirect chain is malware.
1279   urls.clear();
1280   urls.push_back(GURL("http://www.randomevil.com"));
1281   urls.push_back(GURL(std::string("http://") + kEvil1Url1));
1282   EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1283   ASSERT_EQ(prefix_hits.size(), 1U);
1284   EXPECT_EQ(prefix_hits[0], Sha256Prefix(kEvil1Url1));
1285
1286   // Multiple hits in redirect chain are in malware list.
1287   urls.clear();
1288   urls.push_back(GURL(std::string("http://") + kEvil1Url1));
1289   urls.push_back(GURL(std::string("https://") + kEvil1Url2));
1290   EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1291   ASSERT_EQ(prefix_hits.size(), 2U);
1292   EXPECT_EQ(prefix_hits[0], Sha256Prefix(kEvil1Url1));
1293   EXPECT_EQ(prefix_hits[1], Sha256Prefix(kEvil1Url2));
1294   database_.reset();
1295 }
1296
1297 // Checks that the whitelists are handled properly.
1298 TEST_F(SafeBrowsingDatabaseTest, Whitelists) {
1299   database_.reset();
1300   // We expect all calls to ContainsCsdWhitelistedUrl in particular to be made
1301   // from the IO thread.  In general the whitelist lookups are thread-safe.
1302   content::TestBrowserThreadBundle thread_bundle_;
1303
1304   // If the whitelist is disabled everything should match the whitelist.
1305   database_.reset(new SafeBrowsingDatabaseNew(new SafeBrowsingStoreFile(),
1306                                               NULL, NULL, NULL, NULL, NULL,
1307                                               NULL));
1308   database_->Init(database_filename_);
1309   EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1310       GURL(std::string("http://www.phishing.com/"))));
1311   EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1312       GURL(std::string("http://www.phishing.com/"))));
1313   EXPECT_TRUE(database_->ContainsDownloadWhitelistedString("asdf"));
1314
1315   SafeBrowsingStoreFile* browse_store = new SafeBrowsingStoreFile();
1316   SafeBrowsingStoreFile* csd_whitelist_store = new SafeBrowsingStoreFile();
1317   SafeBrowsingStoreFile* download_whitelist_store = new SafeBrowsingStoreFile();
1318   SafeBrowsingStoreFile* extension_blacklist_store =
1319       new SafeBrowsingStoreFile();
1320   database_.reset(new SafeBrowsingDatabaseNew(browse_store, NULL,
1321                                               csd_whitelist_store,
1322                                               download_whitelist_store,
1323                                               extension_blacklist_store,
1324                                               NULL, NULL));
1325   database_->Init(database_filename_);
1326
1327   const char kGood1Host[] = "www.good1.com/";
1328   const char kGood1Url1[] = "www.good1.com/a/b.html";
1329   const char kGood1Url2[] = "www.good1.com/b/";
1330
1331   const char kGood2Host[] = "www.good2.com/";
1332   const char kGood2Url1[] = "www.good2.com/c";  // Should match '/c/bla'.
1333
1334   // good3.com/a/b/c/d/e/f/g/ should match because it's a whitelist.
1335   const char kGood3Host[] = "good3.com/";
1336   const char kGood3Url1[] = "good3.com/";
1337
1338   const char kGoodString[] = "good_string";
1339
1340   SBChunkList download_chunks, csd_chunks;
1341   SBChunk chunk;
1342   // Add two simple chunks to the csd whitelist.
1343   InsertAddChunkHost2FullHashes(&chunk, 1, kGood1Host,
1344                                 kGood1Url1, kGood1Url2);
1345   csd_chunks.push_back(chunk);
1346
1347   chunk.hosts.clear();
1348   InsertAddChunkHostFullHashes(&chunk, 2, kGood2Host, kGood2Url1);
1349   csd_chunks.push_back(chunk);
1350
1351   chunk.hosts.clear();
1352   InsertAddChunkHostFullHashes(&chunk, 2, kGood2Host, kGood2Url1);
1353   download_chunks.push_back(chunk);
1354
1355   chunk.hosts.clear();
1356   InsertAddChunkHostFullHashes(&chunk, 3, kGoodString, kGoodString);
1357   download_chunks.push_back(chunk);
1358
1359   chunk.hosts.clear();
1360   InsertAddChunkHostFullHashes(&chunk, 4, kGood3Host, kGood3Url1);
1361   download_chunks.push_back(chunk);
1362
1363   std::vector<SBListChunkRanges> lists;
1364   EXPECT_TRUE(database_->UpdateStarted(&lists));
1365   database_->InsertChunks(safe_browsing_util::kCsdWhiteList,
1366                           csd_chunks);
1367   database_->InsertChunks(safe_browsing_util::kDownloadWhiteList,
1368                           download_chunks);
1369   database_->UpdateFinished(true);
1370
1371   EXPECT_FALSE(database_->ContainsCsdWhitelistedUrl(
1372       GURL(std::string("http://") + kGood1Host)));
1373
1374   EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1375       GURL(std::string("http://") + kGood1Url1)));
1376   EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1377       GURL(std::string("http://") + kGood1Url1 + "?a=b")));
1378
1379   EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1380       GURL(std::string("http://") + kGood1Url2)));
1381   EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1382       GURL(std::string("http://") + kGood1Url2 + "/c.html")));
1383
1384   EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1385       GURL(std::string("https://") + kGood1Url2 + "/c.html")));
1386
1387   EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1388       GURL(std::string("http://") + kGood2Url1 + "/c")));
1389   EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1390       GURL(std::string("http://") + kGood2Url1 + "/c?bla")));
1391   EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1392       GURL(std::string("http://") + kGood2Url1 + "/c/bla")));
1393
1394   EXPECT_FALSE(database_->ContainsCsdWhitelistedUrl(
1395       GURL(std::string("http://www.google.com/"))));
1396
1397   EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1398       GURL(std::string("http://") + kGood2Url1 + "/c")));
1399   EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1400       GURL(std::string("http://") + kGood2Url1 + "/c?bla")));
1401   EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1402       GURL(std::string("http://") + kGood2Url1 + "/c/bla")));
1403
1404   EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1405       GURL(std::string("http://good3.com/a/b/c/d/e/f/g/"))));
1406   EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1407       GURL(std::string("http://a.b.good3.com/"))));
1408
1409   EXPECT_FALSE(database_->ContainsDownloadWhitelistedString("asdf"));
1410   EXPECT_TRUE(database_->ContainsDownloadWhitelistedString(kGoodString));
1411
1412   EXPECT_FALSE(database_->ContainsDownloadWhitelistedUrl(
1413       GURL(std::string("http://www.google.com/"))));
1414
1415   // Test only add the malware IP killswitch
1416   csd_chunks.clear();
1417   chunk.hosts.clear();
1418   InsertAddChunkHostFullHashes(
1419       &chunk, 15, "sb-ssl.google.com/",
1420       "sb-ssl.google.com/safebrowsing/csd/killswitch_malware");
1421   csd_chunks.push_back(chunk);
1422   EXPECT_TRUE(database_->UpdateStarted(&lists));
1423   database_->InsertChunks(safe_browsing_util::kCsdWhiteList, csd_chunks);
1424   database_->UpdateFinished(true);
1425
1426   EXPECT_TRUE(database_->IsMalwareIPMatchKillSwitchOn());
1427
1428   // Test that the kill-switch works as intended.
1429   csd_chunks.clear();
1430   download_chunks.clear();
1431   lists.clear();
1432   chunk.hosts.clear();
1433   InsertAddChunkHostFullHashes(&chunk, 5, "sb-ssl.google.com/",
1434                                "sb-ssl.google.com/safebrowsing/csd/killswitch");
1435   csd_chunks.push_back(chunk);
1436   chunk.hosts.clear();
1437   InsertAddChunkHostFullHashes(&chunk, 5, "sb-ssl.google.com/",
1438                                "sb-ssl.google.com/safebrowsing/csd/killswitch");
1439   download_chunks.push_back(chunk);
1440
1441   EXPECT_TRUE(database_->UpdateStarted(&lists));
1442   database_->InsertChunks(safe_browsing_util::kCsdWhiteList, csd_chunks);
1443   database_->InsertChunks(safe_browsing_util::kDownloadWhiteList,
1444                           download_chunks);
1445   database_->UpdateFinished(true);
1446
1447   EXPECT_TRUE(database_->IsMalwareIPMatchKillSwitchOn());
1448   EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1449       GURL(std::string("https://") + kGood1Url2 + "/c.html")));
1450   EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1451       GURL(std::string("http://www.google.com/"))));
1452   EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1453       GURL(std::string("http://www.phishing_url.com/"))));
1454
1455   EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1456       GURL(std::string("https://") + kGood1Url2 + "/c.html")));
1457   EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1458       GURL(std::string("http://www.google.com/"))));
1459   EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1460       GURL(std::string("http://www.phishing_url.com/"))));
1461
1462   EXPECT_TRUE(database_->ContainsDownloadWhitelistedString("asdf"));
1463   EXPECT_TRUE(database_->ContainsDownloadWhitelistedString(kGoodString));
1464
1465   // Remove the kill-switch and verify that we can recover.
1466   csd_chunks.clear();
1467   download_chunks.clear();
1468   lists.clear();
1469   SBChunk sub_chunk;
1470   InsertSubChunkHostFullHash(&sub_chunk, 1, 5,
1471                              "sb-ssl.google.com/",
1472                              "sb-ssl.google.com/safebrowsing/csd/killswitch");
1473   csd_chunks.push_back(sub_chunk);
1474
1475   sub_chunk.hosts.clear();
1476   InsertSubChunkHostFullHash(
1477       &sub_chunk, 10, 15, "sb-ssl.google.com/",
1478       "sb-ssl.google.com/safebrowsing/csd/killswitch_malware");
1479   csd_chunks.push_back(sub_chunk);
1480
1481   sub_chunk.hosts.clear();
1482   InsertSubChunkHostFullHash(&sub_chunk, 1, 5,
1483                              "sb-ssl.google.com/",
1484                              "sb-ssl.google.com/safebrowsing/csd/killswitch");
1485   download_chunks.push_back(sub_chunk);
1486
1487   EXPECT_TRUE(database_->UpdateStarted(&lists));
1488   database_->InsertChunks(safe_browsing_util::kCsdWhiteList, csd_chunks);
1489   database_->InsertChunks(safe_browsing_util::kDownloadWhiteList,
1490                           download_chunks);
1491   database_->UpdateFinished(true);
1492
1493   EXPECT_FALSE(database_->IsMalwareIPMatchKillSwitchOn());
1494   EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1495       GURL(std::string("https://") + kGood1Url2 + "/c.html")));
1496   EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1497       GURL(std::string("https://") + kGood2Url1 + "/c/bla")));
1498   EXPECT_FALSE(database_->ContainsCsdWhitelistedUrl(
1499       GURL(std::string("http://www.google.com/"))));
1500   EXPECT_FALSE(database_->ContainsCsdWhitelistedUrl(
1501       GURL(std::string("http://www.phishing_url.com/"))));
1502
1503   EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1504       GURL(std::string("https://") + kGood2Url1 + "/c/bla")));
1505   EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1506       GURL(std::string("https://good3.com/"))));
1507   EXPECT_TRUE(database_->ContainsDownloadWhitelistedString(kGoodString));
1508   EXPECT_FALSE(database_->ContainsDownloadWhitelistedUrl(
1509       GURL(std::string("http://www.google.com/"))));
1510   EXPECT_FALSE(database_->ContainsDownloadWhitelistedUrl(
1511       GURL(std::string("http://www.phishing_url.com/"))));
1512   EXPECT_FALSE(database_->ContainsDownloadWhitelistedString("asdf"));
1513
1514   database_.reset();
1515 }
1516
1517 // Test to make sure we could insert chunk list that
1518 // contains entries for the same host.
1519 TEST_F(SafeBrowsingDatabaseTest, SameHostEntriesOkay) {
1520   SBChunk chunk;
1521
1522   // Add a malware add chunk with two entries of the same host.
1523   InsertAddChunkHostPrefixUrl(&chunk, 1, "www.evil.com/",
1524                               "www.evil.com/malware1.html");
1525   InsertAddChunkHostPrefixUrl(&chunk, 1, "www.evil.com/",
1526                               "www.evil.com/malware2.html");
1527   SBChunkList chunks;
1528   chunks.push_back(chunk);
1529
1530   // Insert the testing chunks into database.
1531   std::vector<SBListChunkRanges> lists;
1532   EXPECT_TRUE(database_->UpdateStarted(&lists));
1533   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
1534   database_->UpdateFinished(true);
1535
1536   GetListsInfo(&lists);
1537   EXPECT_EQ(std::string(safe_browsing_util::kMalwareList), lists[0].name);
1538   EXPECT_EQ("1", lists[0].adds);
1539   EXPECT_TRUE(lists[0].subs.empty());
1540
1541   // Add a phishing add chunk with two entries of the same host.
1542   chunk.hosts.clear();
1543   InsertAddChunkHostPrefixUrl(&chunk, 47, "www.evil.com/",
1544                               "www.evil.com/phishing1.html");
1545   InsertAddChunkHostPrefixUrl(&chunk, 47, "www.evil.com/",
1546                               "www.evil.com/phishing2.html");
1547   chunks.clear();
1548   chunks.push_back(chunk);
1549
1550   EXPECT_TRUE(database_->UpdateStarted(&lists));
1551   database_->InsertChunks(safe_browsing_util::kPhishingList, chunks);
1552   database_->UpdateFinished(true);
1553
1554   GetListsInfo(&lists);
1555   EXPECT_EQ(std::string(safe_browsing_util::kMalwareList), lists[0].name);
1556   EXPECT_EQ("1", lists[0].adds);
1557   EXPECT_EQ(std::string(safe_browsing_util::kPhishingList), lists[1].name);
1558   EXPECT_EQ("47", lists[1].adds);
1559
1560   const Time now = Time::Now();
1561   std::vector<SBPrefix> prefixes;
1562   std::vector<SBFullHashResult> full_hashes;
1563   std::vector<SBPrefix> prefix_hits;
1564   std::string matching_list;
1565   std::string listname;
1566
1567   EXPECT_TRUE(database_->ContainsBrowseUrl(
1568       GURL("http://www.evil.com/malware1.html"),
1569       &listname, &prefixes, &full_hashes, now));
1570   EXPECT_TRUE(database_->ContainsBrowseUrl(
1571       GURL("http://www.evil.com/malware2.html"),
1572       &listname, &prefixes, &full_hashes, now));
1573   EXPECT_TRUE(database_->ContainsBrowseUrl(
1574       GURL("http://www.evil.com/phishing1.html"),
1575       &listname, &prefixes, &full_hashes, now));
1576   EXPECT_TRUE(database_->ContainsBrowseUrl(
1577       GURL("http://www.evil.com/phishing2.html"),
1578       &listname, &prefixes, &full_hashes, now));
1579
1580   // Test removing a single prefix from the add chunk.
1581   // Remove the prefix that added first.
1582   chunk.hosts.clear();
1583   InsertSubChunkHostPrefixUrl(&chunk, 4, 1, "www.evil.com/",
1584                               "www.evil.com/malware1.html");
1585   chunks.clear();
1586   chunks.push_back(chunk);
1587   EXPECT_TRUE(database_->UpdateStarted(&lists));
1588   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
1589   database_->UpdateFinished(true);
1590
1591   // Remove the prefix that added last.
1592   chunk.hosts.clear();
1593   InsertSubChunkHostPrefixUrl(&chunk, 5, 47, "www.evil.com/",
1594                               "www.evil.com/phishing2.html");
1595   chunks.clear();
1596   chunks.push_back(chunk);
1597   EXPECT_TRUE(database_->UpdateStarted(&lists));
1598   database_->InsertChunks(safe_browsing_util::kPhishingList, chunks);
1599   database_->UpdateFinished(true);
1600
1601   // Verify that the database contains urls expected.
1602   EXPECT_FALSE(database_->ContainsBrowseUrl(
1603       GURL("http://www.evil.com/malware1.html"),
1604       &listname, &prefixes, &full_hashes, now));
1605   EXPECT_TRUE(database_->ContainsBrowseUrl(
1606       GURL("http://www.evil.com/malware2.html"),
1607       &listname, &prefixes, &full_hashes, now));
1608   EXPECT_TRUE(database_->ContainsBrowseUrl(
1609       GURL("http://www.evil.com/phishing1.html"),
1610       &listname, &prefixes, &full_hashes, now));
1611   EXPECT_FALSE(database_->ContainsBrowseUrl(
1612       GURL("http://www.evil.com/phishing2.html"),
1613       &listname, &prefixes, &full_hashes, now));
1614 }
1615
1616 // Test that an empty update doesn't actually update the database.
1617 // This isn't a functionality requirement, but it is a useful
1618 // optimization.
1619 TEST_F(SafeBrowsingDatabaseTest, EmptyUpdate) {
1620   SBChunkList chunks;
1621   SBChunk chunk;
1622
1623   base::FilePath filename = database_->BrowseDBFilename(database_filename_);
1624
1625   // Prime the database.
1626   std::vector<SBListChunkRanges> lists;
1627   EXPECT_TRUE(database_->UpdateStarted(&lists));
1628
1629   InsertAddChunkHostPrefixUrl(&chunk, 1, "www.evil.com/",
1630                               "www.evil.com/malware.html");
1631   chunks.clear();
1632   chunks.push_back(chunk);
1633   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
1634   database_->UpdateFinished(true);
1635
1636   // Get an older time to reset the lastmod time for detecting whether
1637   // the file has been updated.
1638   base::File::Info before_info, after_info;
1639   ASSERT_TRUE(base::GetFileInfo(filename, &before_info));
1640   const base::Time old_last_modified =
1641       before_info.last_modified - base::TimeDelta::FromSeconds(10);
1642
1643   // Inserting another chunk updates the database file.  The sleep is
1644   // needed because otherwise the entire test can finish w/in the
1645   // resolution of the lastmod time.
1646   ASSERT_TRUE(base::TouchFile(filename, old_last_modified, old_last_modified));
1647   ASSERT_TRUE(base::GetFileInfo(filename, &before_info));
1648   EXPECT_TRUE(database_->UpdateStarted(&lists));
1649   chunk.hosts.clear();
1650   InsertAddChunkHostPrefixUrl(&chunk, 2, "www.foo.com/",
1651                               "www.foo.com/malware.html");
1652   chunks.clear();
1653   chunks.push_back(chunk);
1654   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
1655   database_->UpdateFinished(true);
1656   ASSERT_TRUE(base::GetFileInfo(filename, &after_info));
1657   EXPECT_LT(before_info.last_modified, after_info.last_modified);
1658
1659   // Deleting a chunk updates the database file.
1660   ASSERT_TRUE(base::TouchFile(filename, old_last_modified, old_last_modified));
1661   ASSERT_TRUE(base::GetFileInfo(filename, &before_info));
1662   EXPECT_TRUE(database_->UpdateStarted(&lists));
1663   AddDelChunk(safe_browsing_util::kMalwareList, chunk.chunk_number);
1664   database_->UpdateFinished(true);
1665   ASSERT_TRUE(base::GetFileInfo(filename, &after_info));
1666   EXPECT_LT(before_info.last_modified, after_info.last_modified);
1667
1668   // Simply calling |UpdateStarted()| then |UpdateFinished()| does not
1669   // update the database file.
1670   ASSERT_TRUE(base::TouchFile(filename, old_last_modified, old_last_modified));
1671   ASSERT_TRUE(base::GetFileInfo(filename, &before_info));
1672   EXPECT_TRUE(database_->UpdateStarted(&lists));
1673   database_->UpdateFinished(true);
1674   ASSERT_TRUE(base::GetFileInfo(filename, &after_info));
1675   EXPECT_EQ(before_info.last_modified, after_info.last_modified);
1676 }
1677
1678 // Test that a filter file is written out during update and read back
1679 // in during setup.
1680 TEST_F(SafeBrowsingDatabaseTest, FilterFile) {
1681   // Create a database with trivial example data and write it out.
1682   {
1683     SBChunkList chunks;
1684     SBChunk chunk;
1685
1686     // Prime the database.
1687     std::vector<SBListChunkRanges> lists;
1688     EXPECT_TRUE(database_->UpdateStarted(&lists));
1689
1690     InsertAddChunkHostPrefixUrl(&chunk, 1, "www.evil.com/",
1691                                 "www.evil.com/malware.html");
1692     chunks.clear();
1693     chunks.push_back(chunk);
1694     database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
1695     database_->UpdateFinished(true);
1696   }
1697
1698   // Find the malware url in the database, don't find a good url.
1699   const Time now = Time::Now();
1700   std::vector<SBFullHashResult> full_hashes;
1701   std::vector<SBPrefix> prefix_hits;
1702   std::string matching_list;
1703   EXPECT_TRUE(database_->ContainsBrowseUrl(
1704       GURL("http://www.evil.com/malware.html"),
1705       &matching_list, &prefix_hits, &full_hashes, now));
1706   EXPECT_FALSE(database_->ContainsBrowseUrl(
1707       GURL("http://www.good.com/goodware.html"),
1708       &matching_list, &prefix_hits, &full_hashes, now));
1709
1710   base::FilePath filter_file = database_->PrefixSetForFilename(
1711       database_->BrowseDBFilename(database_filename_));
1712
1713   // After re-creating the database, it should have a filter read from
1714   // a file, so it should find the same results.
1715   ASSERT_TRUE(base::PathExists(filter_file));
1716   database_.reset(new SafeBrowsingDatabaseNew);
1717   database_->Init(database_filename_);
1718   EXPECT_TRUE(database_->ContainsBrowseUrl(
1719       GURL("http://www.evil.com/malware.html"),
1720       &matching_list, &prefix_hits, &full_hashes, now));
1721   EXPECT_FALSE(database_->ContainsBrowseUrl(
1722       GURL("http://www.good.com/goodware.html"),
1723       &matching_list, &prefix_hits, &full_hashes, now));
1724
1725   // If there is no filter file, the database cannot find malware urls.
1726   base::DeleteFile(filter_file, false);
1727   ASSERT_FALSE(base::PathExists(filter_file));
1728   database_.reset(new SafeBrowsingDatabaseNew);
1729   database_->Init(database_filename_);
1730   EXPECT_FALSE(database_->ContainsBrowseUrl(
1731       GURL("http://www.evil.com/malware.html"),
1732       &matching_list, &prefix_hits, &full_hashes, now));
1733   EXPECT_FALSE(database_->ContainsBrowseUrl(
1734       GURL("http://www.good.com/goodware.html"),
1735       &matching_list, &prefix_hits, &full_hashes, now));
1736 }
1737
1738 TEST_F(SafeBrowsingDatabaseTest, MalwareIpBlacklist) {
1739   database_.reset();
1740   SafeBrowsingStoreFile* browse_store = new SafeBrowsingStoreFile();
1741   SafeBrowsingStoreFile* ip_blacklist_store = new SafeBrowsingStoreFile();
1742   database_.reset(new SafeBrowsingDatabaseNew(browse_store,
1743                                               NULL,
1744                                               NULL,
1745                                               NULL,
1746                                               NULL,
1747                                               NULL,
1748                                               ip_blacklist_store));
1749   database_->Init(database_filename_);
1750   std::vector<SBListChunkRanges> lists;
1751   EXPECT_TRUE(database_->UpdateStarted(&lists));
1752
1753   // IPv4 prefix match for ::ffff:192.168.1.0/120.
1754   SBChunkList chunks;
1755   SBChunk chunk;
1756   InsertAddChunkFullHash(&chunk, 1, "::ffff:192.168.1.0", 120);
1757   chunks.push_back(chunk);
1758   database_->InsertChunks(safe_browsing_util::kIPBlacklist, chunks);
1759
1760   // IPv4 exact match for ::ffff:192.1.1.1.
1761   chunks.clear();
1762   chunk.hosts.clear();
1763   InsertAddChunkFullHash(&chunk, 2, "::ffff:192.1.1.1", 128);
1764   chunks.push_back(chunk);
1765   database_->InsertChunks(safe_browsing_util::kIPBlacklist, chunks);
1766
1767   // IPv6 exact match for: fe80::31a:a0ff:fe10:786e/128.
1768   chunks.clear();
1769   chunk.hosts.clear();
1770   InsertAddChunkFullHash(&chunk, 3, "fe80::31a:a0ff:fe10:786e", 128);
1771   chunks.push_back(chunk);
1772   database_->InsertChunks(safe_browsing_util::kIPBlacklist, chunks);
1773
1774   // IPv6 prefix match for: 2620:0:1000:3103::/64.
1775   chunks.clear();
1776   chunk.hosts.clear();
1777   InsertAddChunkFullHash(&chunk, 4, "2620:0:1000:3103::", 64);
1778   chunks.push_back(chunk);
1779   database_->InsertChunks(safe_browsing_util::kIPBlacklist, chunks);
1780
1781   // IPv4 prefix match for ::ffff:192.1.122.0/119.
1782   chunks.clear();
1783   chunk.hosts.clear();
1784   InsertAddChunkFullHash(&chunk, 5, "::ffff:192.1.122.0", 119);
1785   chunks.push_back(chunk);
1786   database_->InsertChunks(safe_browsing_util::kIPBlacklist, chunks);
1787
1788   // IPv4 prefix match for ::ffff:192.1.128.0/113.
1789   chunks.clear();
1790   chunk.hosts.clear();
1791   InsertAddChunkFullHash(&chunk, 6, "::ffff:192.1.128.0", 113);
1792   chunks.push_back(chunk);
1793   database_->InsertChunks(safe_browsing_util::kIPBlacklist, chunks);
1794
1795   database_->UpdateFinished(true);
1796
1797   EXPECT_FALSE(database_->ContainsMalwareIP("192.168.0.255"));
1798   EXPECT_TRUE(database_->ContainsMalwareIP("192.168.1.0"));
1799   EXPECT_TRUE(database_->ContainsMalwareIP("192.168.1.255"));
1800   EXPECT_TRUE(database_->ContainsMalwareIP("192.168.1.10"));
1801   EXPECT_TRUE(database_->ContainsMalwareIP("::ffff:192.168.1.2"));
1802   EXPECT_FALSE(database_->ContainsMalwareIP("192.168.2.0"));
1803
1804   EXPECT_FALSE(database_->ContainsMalwareIP("192.1.1.0"));
1805   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.1.1"));
1806   EXPECT_FALSE(database_->ContainsMalwareIP("192.1.1.2"));
1807
1808   EXPECT_FALSE(database_->ContainsMalwareIP(
1809       "2620:0:1000:3102:ffff:ffff:ffff:ffff"));
1810   EXPECT_TRUE(database_->ContainsMalwareIP("2620:0:1000:3103::"));
1811   EXPECT_TRUE(database_->ContainsMalwareIP(
1812       "2620:0:1000:3103:ffff:ffff:ffff:ffff"));
1813   EXPECT_FALSE(database_->ContainsMalwareIP("2620:0:1000:3104::"));
1814
1815   EXPECT_FALSE(database_->ContainsMalwareIP("fe80::21a:a0ff:fe10:786d"));
1816   EXPECT_TRUE(database_->ContainsMalwareIP("fe80::31a:a0ff:fe10:786e"));
1817   EXPECT_FALSE(database_->ContainsMalwareIP("fe80::21a:a0ff:fe10:786f"));
1818
1819   EXPECT_FALSE(database_->ContainsMalwareIP("192.1.121.255"));
1820   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.122.0"));
1821   EXPECT_TRUE(database_->ContainsMalwareIP("::ffff:192.1.122.1"));
1822   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.122.255"));
1823   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.123.0"));
1824   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.123.255"));
1825   EXPECT_FALSE(database_->ContainsMalwareIP("192.1.124.0"));
1826
1827   EXPECT_FALSE(database_->ContainsMalwareIP("192.1.127.255"));
1828   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.128.0"));
1829   EXPECT_TRUE(database_->ContainsMalwareIP("::ffff:192.1.128.1"));
1830   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.128.255"));
1831   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.255.0"));
1832   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.255.255"));
1833   EXPECT_FALSE(database_->ContainsMalwareIP("192.2.0.0"));
1834 }