chrome/browser/safe_browsing/safe_browsing_database_unittest.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4 //
   5 // Unit tests for the SafeBrowsing storage system.
   6
   7 #include "base/file_util.h"
   8 #include "base/files/scoped_temp_dir.h"
   9 #include "base/logging.h"
  10 #include "base/message_loop/message_loop.h"
  11 #include "base/sha1.h"
  12 #include "base/time/time.h"
  13 #include "chrome/browser/safe_browsing/safe_browsing_database.h"
  14 #include "chrome/browser/safe_browsing/safe_browsing_store_file.h"
  15 #include "content/public/test/test_browser_thread_bundle.h"
  16 #include "crypto/sha2.h"
  17 #include "net/base/net_util.h"
  18 #include "sql/connection.h"
  19 #include "sql/statement.h"
  20 #include "testing/gtest/include/gtest/gtest.h"
  21 #include "testing/platform_test.h"
  22 #include "url/gurl.h"
  23
  24 using base::Time;
  25
  26 namespace {
  27
  28 SBPrefix SBPrefixForString(const std::string& str) {
  29   return SBFullHashForString(str).prefix;
  30 }
  31
  32 std::string HashedIpPrefix(const std::string& ip_prefix, size_t prefix_size) {
  33   net::IPAddressNumber ip_number;
  34   EXPECT_TRUE(net::ParseIPLiteralToNumber(ip_prefix, &ip_number));
  35   EXPECT_EQ(net::kIPv6AddressSize, ip_number.size());
  36   const std::string hashed_ip_prefix = base::SHA1HashString(
  37       net::IPAddressToPackedString(ip_number));
  38   std::string hash(crypto::kSHA256Length, '\0');
  39   hash.replace(0, hashed_ip_prefix.size(), hashed_ip_prefix);
  40   hash[base::kSHA1Length] = static_cast<char>(prefix_size);
  41   return hash;
  42 }
  43
  44 // Same as InsertAddChunkHostPrefixUrl, but with pre-computed
  45 // prefix values.
  46 void InsertAddChunkHostPrefixValue(SBChunk* chunk,
  47                                    int chunk_number,
  48                                    const SBPrefix& host_prefix,
  49                                    const SBPrefix& url_prefix) {
  50   chunk->chunk_number = chunk_number;
  51   chunk->is_add = true;
  52   SBChunkHost host;
  53   host.host = host_prefix;
  54   host.entry = SBEntry::Create(SBEntry::ADD_PREFIX, 1);
  55   host.entry->set_chunk_id(chunk->chunk_number);
  56   host.entry->SetPrefixAt(0, url_prefix);
  57   chunk->hosts.push_back(host);
  58 }
  59
  60 // A helper function that appends one AddChunkHost to chunk with
  61 // one url for prefix.
  62 void InsertAddChunkHostPrefixUrl(SBChunk* chunk,
  63                                  int chunk_number,
  64                                  const std::string& host_name,
  65                                  const std::string& url) {
  66   InsertAddChunkHostPrefixValue(chunk, chunk_number,
  67                                 SBPrefixForString(host_name),
  68                                 SBPrefixForString(url));
  69 }
  70
  71 // Same as InsertAddChunkHostPrefixUrl, but with full hashes.
  72 void InsertAddChunkHostFullHashes(SBChunk* chunk,
  73                                   int chunk_number,
  74                                   const std::string& host_name,
  75                                   const std::string& url) {
  76   chunk->chunk_number = chunk_number;
  77   chunk->is_add = true;
  78   SBChunkHost host;
  79   host.host = SBPrefixForString(host_name);
  80   host.entry = SBEntry::Create(SBEntry::ADD_FULL_HASH, 1);
  81   host.entry->set_chunk_id(chunk->chunk_number);
  82   host.entry->SetFullHashAt(0, SBFullHashForString(url));
  83   chunk->hosts.push_back(host);
  84 }
  85
  86 // TODO(shess): This sounds like something to insert a full-hash chunk, but it's
  87 // actually specific to IP blacklist.
  88 void InsertAddChunkFullHash(SBChunk* chunk,
  89                             int chunk_number,
  90                             const std::string& ip_str,
  91                             size_t prefix_size) {
  92   const std::string full_hash_str = HashedIpPrefix(ip_str, prefix_size);
  93   EXPECT_EQ(sizeof(SBFullHash), full_hash_str.size());
  94   SBFullHash full_hash;
  95   std::memcpy(&(full_hash.full_hash), full_hash_str.data(), sizeof(SBFullHash));
  96
  97   chunk->chunk_number = chunk_number;
  98   chunk->is_add = true;
  99   SBChunkHost host;
 100   host.host = full_hash.prefix;
 101   host.entry = SBEntry::Create(SBEntry::ADD_FULL_HASH, 1);
 102   host.entry->set_chunk_id(chunk->chunk_number);
 103   host.entry->SetFullHashAt(0, full_hash);
 104   chunk->hosts.push_back(host);
 105 }
 106
 107 // Same as InsertAddChunkHostPrefixUrl, but with two urls for prefixes.
 108 void InsertAddChunkHost2PrefixUrls(SBChunk* chunk,
 109                                    int chunk_number,
 110                                    const std::string& host_name,
 111                                    const std::string& url1,
 112                                    const std::string& url2) {
 113   chunk->chunk_number = chunk_number;
 114   chunk->is_add = true;
 115   SBChunkHost host;
 116   host.host = SBPrefixForString(host_name);
 117   host.entry = SBEntry::Create(SBEntry::ADD_PREFIX, 2);
 118   host.entry->set_chunk_id(chunk->chunk_number);
 119   host.entry->SetPrefixAt(0, SBPrefixForString(url1));
 120   host.entry->SetPrefixAt(1, SBPrefixForString(url2));
 121   chunk->hosts.push_back(host);
 122 }
 123
 124 // Same as InsertAddChunkHost2PrefixUrls, but with full hashes.
 125 void InsertAddChunkHost2FullHashes(SBChunk* chunk,
 126                                    int chunk_number,
 127                                    const std::string& host_name,
 128                                    const std::string& url1,
 129                                    const std::string& url2) {
 130   chunk->chunk_number = chunk_number;
 131   chunk->is_add = true;
 132   SBChunkHost host;
 133   host.host = SBPrefixForString(host_name);
 134   host.entry = SBEntry::Create(SBEntry::ADD_FULL_HASH, 2);
 135   host.entry->set_chunk_id(chunk->chunk_number);
 136   host.entry->SetFullHashAt(0, SBFullHashForString(url1));
 137   host.entry->SetFullHashAt(1, SBFullHashForString(url2));
 138   chunk->hosts.push_back(host);
 139 }
 140
 141 // Same as InsertSubChunkHostPrefixUrl, but with pre-computed
 142 // prefix values.
 143 void InsertSubChunkHostPrefixValue(SBChunk* chunk,
 144                                    int chunk_number,
 145                                    int chunk_id_to_sub,
 146                                    const SBPrefix& host_prefix,
 147                                    const SBPrefix& url_prefix) {
 148   chunk->chunk_number = chunk_number;
 149   chunk->is_add = false;
 150   SBChunkHost host;
 151   host.host = host_prefix;
 152   host.entry = SBEntry::Create(SBEntry::SUB_PREFIX, 1);
 153   host.entry->set_chunk_id(chunk->chunk_number);
 154   host.entry->SetChunkIdAtPrefix(0, chunk_id_to_sub);
 155   host.entry->SetPrefixAt(0, url_prefix);
 156   chunk->hosts.push_back(host);
 157 }
 158
 159 // A helper function that adds one SubChunkHost to chunk with
 160 // one url for prefix.
 161 void InsertSubChunkHostPrefixUrl(SBChunk* chunk,
 162                                  int chunk_number,
 163                                  int chunk_id_to_sub,
 164                                  const std::string& host_name,
 165                                  const std::string& url) {
 166   InsertSubChunkHostPrefixValue(chunk, chunk_number,
 167                                 chunk_id_to_sub,
 168                                 SBPrefixForString(host_name),
 169                                 SBPrefixForString(url));
 170 }
 171
 172 // Same as InsertSubChunkHostPrefixUrl, but with two urls for prefixes.
 173 void InsertSubChunkHost2PrefixUrls(SBChunk* chunk,
 174                                    int chunk_number,
 175                                    int chunk_id_to_sub,
 176                                    const std::string& host_name,
 177                                    const std::string& url1,
 178                                    const std::string& url2) {
 179   chunk->chunk_number = chunk_number;
 180   chunk->is_add = false;
 181   SBChunkHost host;
 182   host.host = SBPrefixForString(host_name);
 183   host.entry = SBEntry::Create(SBEntry::SUB_PREFIX, 2);
 184   host.entry->set_chunk_id(chunk->chunk_number);
 185   host.entry->SetPrefixAt(0, SBPrefixForString(url1));
 186   host.entry->SetChunkIdAtPrefix(0, chunk_id_to_sub);
 187   host.entry->SetPrefixAt(1, SBPrefixForString(url2));
 188   host.entry->SetChunkIdAtPrefix(1, chunk_id_to_sub);
 189   chunk->hosts.push_back(host);
 190 }
 191
 192 // Same as InsertSubChunkHost2PrefixUrls, but with full hashes.
 193 void InsertSubChunkHostFullHash(SBChunk* chunk,
 194                                 int chunk_number,
 195                                 int chunk_id_to_sub,
 196                                 const std::string& host_name,
 197                                 const std::string& url) {
 198   chunk->chunk_number = chunk_number;
 199   chunk->is_add = false;
 200   SBChunkHost host;
 201   host.host = SBPrefixForString(host_name);
 202   host.entry = SBEntry::Create(SBEntry::SUB_FULL_HASH, 2);
 203   host.entry->set_chunk_id(chunk->chunk_number);
 204   host.entry->SetFullHashAt(0, SBFullHashForString(url));
 205   host.entry->SetChunkIdAtPrefix(0, chunk_id_to_sub);
 206   chunk->hosts.push_back(host);
 207 }
 208
 209 // Prevent DCHECK from killing tests.
 210 // TODO(shess): Pawel disputes the use of this, so the test which uses
 211 // it is DISABLED.  http://crbug.com/56448
 212 class ScopedLogMessageIgnorer {
 213  public:
 214   ScopedLogMessageIgnorer() {
 215     logging::SetLogMessageHandler(&LogMessageIgnorer);
 216   }
 217   ~ScopedLogMessageIgnorer() {
 218     // TODO(shess): Would be better to verify whether anyone else
 219     // changed it, and then restore it to the previous value.
 220     logging::SetLogMessageHandler(NULL);
 221   }
 222
 223  private:
 224   static bool LogMessageIgnorer(int severity, const char* file, int line,
 225       size_t message_start, const std::string& str) {
 226     // Intercept FATAL, strip the stack backtrace, and log it without
 227     // the crash part.
 228     if (severity == logging::LOG_FATAL) {
 229       size_t newline = str.find('\n');
 230       if (newline != std::string::npos) {
 231         const std::string msg = str.substr(0, newline + 1);
 232         fprintf(stderr, "%s", msg.c_str());
 233         fflush(stderr);
 234       }
 235       return true;
 236     }
 237
 238     return false;
 239   }
 240 };
 241
 242 }  // namespace
 243
 244 class SafeBrowsingDatabaseTest : public PlatformTest {
 245  public:
 246   virtual void SetUp() {
 247     PlatformTest::SetUp();
 248
 249     // Setup a database in a temporary directory.
 250     ASSERT_TRUE(temp_dir_.CreateUniqueTempDir());
 251     database_.reset(new SafeBrowsingDatabaseNew);
 252     database_filename_ =
 253         temp_dir_.path().AppendASCII("SafeBrowsingTestDatabase");
 254     database_->Init(database_filename_);
 255   }
 256
 257   virtual void TearDown() {
 258     database_.reset();
 259
 260     PlatformTest::TearDown();
 261   }
 262
 263   void GetListsInfo(std::vector<SBListChunkRanges>* lists) {
 264     lists->clear();
 265     EXPECT_TRUE(database_->UpdateStarted(lists));
 266     database_->UpdateFinished(true);
 267   }
 268
 269   // Helper function to do an AddDel or SubDel command.
 270   void DelChunk(const std::string& list,
 271                 int chunk_id,
 272                 bool is_sub_del) {
 273     std::vector<SBChunkDelete> deletes;
 274     SBChunkDelete chunk_delete;
 275     chunk_delete.list_name = list;
 276     chunk_delete.is_sub_del = is_sub_del;
 277     chunk_delete.chunk_del.push_back(ChunkRange(chunk_id));
 278     deletes.push_back(chunk_delete);
 279     database_->DeleteChunks(deletes);
 280   }
 281
 282   void AddDelChunk(const std::string& list, int chunk_id) {
 283     DelChunk(list, chunk_id, false);
 284   }
 285
 286   void SubDelChunk(const std::string& list, int chunk_id) {
 287     DelChunk(list, chunk_id, true);
 288   }
 289
 290   // Utility function for setting up the database for the caching test.
 291   void PopulateDatabaseForCacheTest();
 292
 293   scoped_ptr<SafeBrowsingDatabaseNew> database_;
 294   base::FilePath database_filename_;
 295   base::ScopedTempDir temp_dir_;
 296 };
 297
 298 // Tests retrieving list name information.
 299 TEST_F(SafeBrowsingDatabaseTest, ListNameForBrowse) {
 300   SBChunkList chunks;
 301   SBChunk chunk;
 302
 303   InsertAddChunkHostPrefixUrl(&chunk, 1, "www.evil.com/",
 304                               "www.evil.com/malware.html");
 305   chunks.clear();
 306   chunks.push_back(chunk);
 307   std::vector<SBListChunkRanges> lists;
 308   EXPECT_TRUE(database_->UpdateStarted(&lists));
 309   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
 310
 311   chunk.hosts.clear();
 312   InsertAddChunkHostPrefixUrl(&chunk, 2, "www.foo.com/",
 313                               "www.foo.com/malware.html");
 314   chunks.clear();
 315   chunks.push_back(chunk);
 316   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
 317
 318   chunk.hosts.clear();
 319   InsertAddChunkHostPrefixUrl(&chunk, 3, "www.whatever.com/",
 320                               "www.whatever.com/malware.html");
 321   chunks.clear();
 322   chunks.push_back(chunk);
 323   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
 324   database_->UpdateFinished(true);
 325
 326   GetListsInfo(&lists);
 327   EXPECT_TRUE(lists[0].name == safe_browsing_util::kMalwareList);
 328   EXPECT_EQ(lists[0].adds, "1-3");
 329   EXPECT_TRUE(lists[0].subs.empty());
 330
 331   // Insert a malware sub chunk.
 332   chunk.hosts.clear();
 333   InsertSubChunkHostPrefixUrl(&chunk, 7, 19, "www.subbed.com/",
 334                               "www.subbed.com/noteveil1.html");
 335   chunks.clear();
 336   chunks.push_back(chunk);
 337
 338   EXPECT_TRUE(database_->UpdateStarted(&lists));
 339   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
 340   database_->UpdateFinished(true);
 341
 342   GetListsInfo(&lists);
 343   EXPECT_TRUE(lists[0].name == safe_browsing_util::kMalwareList);
 344   EXPECT_EQ(lists[0].adds, "1-3");
 345   EXPECT_EQ(lists[0].subs, "7");
 346   if (lists.size() == 2) {
 347     // Old style database won't have the second entry since it creates the lists
 348     // when it receives an update containing that list. The filter-based
 349     // database has these values hard coded.
 350     EXPECT_TRUE(lists[1].name == safe_browsing_util::kPhishingList);
 351     EXPECT_TRUE(lists[1].adds.empty());
 352     EXPECT_TRUE(lists[1].subs.empty());
 353   }
 354
 355   // Add a phishing add chunk.
 356   chunk.hosts.clear();
 357   InsertAddChunkHostPrefixUrl(&chunk, 47, "www.evil.com/",
 358                               "www.evil.com/phishing.html");
 359   chunks.clear();
 360   chunks.push_back(chunk);
 361   EXPECT_TRUE(database_->UpdateStarted(&lists));
 362   database_->InsertChunks(safe_browsing_util::kPhishingList, chunks);
 363
 364   // Insert some phishing sub chunks.
 365   chunk.hosts.clear();
 366   InsertSubChunkHostPrefixUrl(&chunk, 200, 1999, "www.phishy.com/",
 367                               "www.phishy.com/notevil1.html");
 368   chunks.clear();
 369   chunks.push_back(chunk);
 370   database_->InsertChunks(safe_browsing_util::kPhishingList, chunks);
 371
 372   chunk.hosts.clear();
 373   InsertSubChunkHostPrefixUrl(&chunk, 201, 1999, "www.phishy2.com/",
 374                               "www.phishy2.com/notevil1.html");
 375   chunks.clear();
 376   chunks.push_back(chunk);
 377   database_->InsertChunks(safe_browsing_util::kPhishingList, chunks);
 378   database_->UpdateFinished(true);
 379
 380   GetListsInfo(&lists);
 381   EXPECT_TRUE(lists[0].name == safe_browsing_util::kMalwareList);
 382   EXPECT_EQ(lists[0].adds, "1-3");
 383   EXPECT_EQ(lists[0].subs, "7");
 384   EXPECT_TRUE(lists[1].name == safe_browsing_util::kPhishingList);
 385   EXPECT_EQ(lists[1].adds, "47");
 386   EXPECT_EQ(lists[1].subs, "200-201");
 387 }
 388
 389 TEST_F(SafeBrowsingDatabaseTest, ListNameForBrowseAndDownload) {
 390   database_.reset();
 391   base::MessageLoop loop;
 392   SafeBrowsingStoreFile* browse_store = new SafeBrowsingStoreFile();
 393   SafeBrowsingStoreFile* download_store = new SafeBrowsingStoreFile();
 394   SafeBrowsingStoreFile* csd_whitelist_store = new SafeBrowsingStoreFile();
 395   SafeBrowsingStoreFile* download_whitelist_store = new SafeBrowsingStoreFile();
 396   SafeBrowsingStoreFile* extension_blacklist_store =
 397       new SafeBrowsingStoreFile();
 398   SafeBrowsingStoreFile* ip_blacklist_store = new SafeBrowsingStoreFile();
 399   database_.reset(new SafeBrowsingDatabaseNew(browse_store,
 400                                               download_store,
 401                                               csd_whitelist_store,
 402                                               download_whitelist_store,
 403                                               extension_blacklist_store,
 404                                               NULL,
 405                                               ip_blacklist_store));
 406   database_->Init(database_filename_);
 407
 408   SBChunkList chunks;
 409   SBChunk chunk;
 410
 411   // Insert malware, phish, binurl and bindownload add chunks.
 412   InsertAddChunkHostPrefixUrl(&chunk, 1, "www.evil.com/",
 413                               "www.evil.com/malware.html");
 414   chunks.push_back(chunk);
 415   std::vector<SBListChunkRanges> lists;
 416   EXPECT_TRUE(database_->UpdateStarted(&lists));
 417   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
 418
 419   chunk.hosts.clear();
 420   InsertAddChunkHostPrefixUrl(&chunk, 2, "www.foo.com/",
 421                               "www.foo.com/malware.html");
 422   chunks.clear();
 423   chunks.push_back(chunk);
 424   database_->InsertChunks(safe_browsing_util::kPhishingList, chunks);
 425
 426   chunk.hosts.clear();
 427   InsertAddChunkHostPrefixUrl(&chunk, 3, "www.whatever.com/",
 428                               "www.whatever.com/download.html");
 429   chunks.clear();
 430   chunks.push_back(chunk);
 431   database_->InsertChunks(safe_browsing_util::kBinUrlList, chunks);
 432
 433   chunk.hosts.clear();
 434   InsertAddChunkHostFullHashes(&chunk, 5, "www.forwhitelist.com/",
 435                                "www.forwhitelist.com/a.html");
 436   chunks.clear();
 437   chunks.push_back(chunk);
 438   database_->InsertChunks(safe_browsing_util::kCsdWhiteList, chunks);
 439
 440   chunk.hosts.clear();
 441   InsertAddChunkHostFullHashes(&chunk, 6, "www.download.com/",
 442                                "www.download.com/");
 443
 444   chunks.clear();
 445   chunks.push_back(chunk);
 446   database_->InsertChunks(safe_browsing_util::kDownloadWhiteList, chunks);
 447
 448   chunk.hosts.clear();
 449   InsertAddChunkHostFullHashes(&chunk, 8,
 450                                "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
 451                                "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
 452
 453   chunks.clear();
 454   chunks.push_back(chunk);
 455   database_->InsertChunks(safe_browsing_util::kExtensionBlacklist, chunks);
 456
 457   chunk.hosts.clear();
 458   InsertAddChunkFullHash(&chunk, 9, "::ffff:192.168.1.0", 120);
 459
 460   chunks.clear();
 461   chunks.push_back(chunk);
 462   database_->InsertChunks(safe_browsing_util::kIPBlacklist, chunks);
 463
 464   database_->UpdateFinished(true);
 465
 466   GetListsInfo(&lists);
 467   ASSERT_EQ(7U, lists.size());
 468   EXPECT_TRUE(lists[0].name == safe_browsing_util::kMalwareList);
 469   EXPECT_EQ(lists[0].adds, "1");
 470   EXPECT_TRUE(lists[0].subs.empty());
 471   EXPECT_TRUE(lists[1].name == safe_browsing_util::kPhishingList);
 472   EXPECT_EQ(lists[1].adds, "2");
 473   EXPECT_TRUE(lists[1].subs.empty());
 474   EXPECT_TRUE(lists[2].name == safe_browsing_util::kBinUrlList);
 475   EXPECT_EQ(lists[2].adds, "3");
 476   EXPECT_TRUE(lists[2].subs.empty());
 477   EXPECT_TRUE(lists[3].name == safe_browsing_util::kCsdWhiteList);
 478   EXPECT_EQ(lists[3].adds, "5");
 479   EXPECT_TRUE(lists[3].subs.empty());
 480   EXPECT_TRUE(lists[4].name == safe_browsing_util::kDownloadWhiteList);
 481   EXPECT_EQ(lists[4].adds, "6");
 482   EXPECT_TRUE(lists[4].subs.empty());
 483   EXPECT_TRUE(lists[5].name == safe_browsing_util::kExtensionBlacklist);
 484   EXPECT_EQ(lists[5].adds, "8");
 485   EXPECT_TRUE(lists[5].subs.empty());
 486   EXPECT_TRUE(lists[6].name == safe_browsing_util::kIPBlacklist);
 487   EXPECT_EQ(lists[6].adds, "9");
 488   EXPECT_TRUE(lists[6].subs.empty());
 489
 490   database_.reset();
 491 }
 492
 493 // Checks database reading and writing for browse.
 494 TEST_F(SafeBrowsingDatabaseTest, BrowseDatabase) {
 495   SBChunkList chunks;
 496   SBChunk chunk;
 497
 498   // Add a simple chunk with one hostkey.
 499   InsertAddChunkHost2PrefixUrls(&chunk, 1, "www.evil.com/",
 500                                 "www.evil.com/phishing.html",
 501                                 "www.evil.com/malware.html");
 502   chunks.push_back(chunk);
 503   std::vector<SBListChunkRanges> lists;
 504   EXPECT_TRUE(database_->UpdateStarted(&lists));
 505   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
 506
 507   chunk.hosts.clear();
 508   InsertAddChunkHost2PrefixUrls(&chunk, 2, "www.evil.com/",
 509                                 "www.evil.com/notevil1.html",
 510                                 "www.evil.com/notevil2.html");
 511   InsertAddChunkHost2PrefixUrls(&chunk, 2, "www.good.com/",
 512                                 "www.good.com/good1.html",
 513                                 "www.good.com/good2.html");
 514   chunks.clear();
 515   chunks.push_back(chunk);
 516   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
 517
 518   // and a chunk with an IP-based host
 519   chunk.hosts.clear();
 520   InsertAddChunkHostPrefixUrl(&chunk, 3, "192.168.0.1/",
 521                               "192.168.0.1/malware.html");
 522   chunks.clear();
 523   chunks.push_back(chunk);
 524   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
 525
 526   // A chunk with a full hash.
 527   chunk.hosts.clear();
 528   InsertAddChunkHostFullHashes(&chunk, 7, "www.evil.com/",
 529                                "www.evil.com/evil.html");
 530   chunks.clear();
 531   chunks.push_back(chunk);
 532   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
 533
 534   database_->UpdateFinished(true);
 535
 536   // Make sure they were added correctly.
 537   GetListsInfo(&lists);
 538   EXPECT_TRUE(lists[0].name == safe_browsing_util::kMalwareList);
 539   EXPECT_EQ(lists[0].adds, "1-3,7");
 540   EXPECT_TRUE(lists[0].subs.empty());
 541
 542   const Time now = Time::Now();
 543   std::vector<SBFullHashResult> cached_hashes;
 544   std::vector<SBPrefix> prefix_hits;
 545   EXPECT_TRUE(database_->ContainsBrowseUrl(
 546       GURL("http://www.evil.com/phishing.html"),
 547       &prefix_hits, &cached_hashes, now));
 548   EXPECT_EQ(prefix_hits[0], SBPrefixForString("www.evil.com/phishing.html"));
 549   EXPECT_EQ(prefix_hits.size(), 1U);
 550
 551   EXPECT_TRUE(database_->ContainsBrowseUrl(
 552       GURL("http://www.evil.com/malware.html"),
 553       &prefix_hits, &cached_hashes, now));
 554
 555   EXPECT_TRUE(database_->ContainsBrowseUrl(
 556       GURL("http://www.evil.com/notevil1.html"),
 557       &prefix_hits, &cached_hashes, now));
 558
 559   EXPECT_TRUE(database_->ContainsBrowseUrl(
 560       GURL("http://www.evil.com/notevil2.html"),
 561       &prefix_hits, &cached_hashes, now));
 562
 563   EXPECT_TRUE(database_->ContainsBrowseUrl(
 564       GURL("http://www.good.com/good1.html"),
 565       &prefix_hits, &cached_hashes, now));
 566
 567   EXPECT_TRUE(database_->ContainsBrowseUrl(
 568       GURL("http://www.good.com/good2.html"),
 569       &prefix_hits, &cached_hashes, now));
 570
 571   EXPECT_TRUE(database_->ContainsBrowseUrl(
 572       GURL("http://192.168.0.1/malware.html"),
 573       &prefix_hits, &cached_hashes, now));
 574
 575   EXPECT_FALSE(database_->ContainsBrowseUrl(
 576       GURL("http://www.evil.com/"),
 577       &prefix_hits, &cached_hashes, now));
 578   EXPECT_TRUE(prefix_hits.empty());
 579
 580   EXPECT_FALSE(database_->ContainsBrowseUrl(
 581       GURL("http://www.evil.com/robots.txt"),
 582       &prefix_hits, &cached_hashes, now));
 583
 584   EXPECT_TRUE(database_->ContainsBrowseUrl(
 585       GURL("http://www.evil.com/evil.html"),
 586       &prefix_hits, &cached_hashes, now));
 587   ASSERT_EQ(1U, prefix_hits.size());
 588   EXPECT_EQ(prefix_hits[0], SBPrefixForString("www.evil.com/evil.html"));
 589
 590   // Attempt to re-add the first chunk (should be a no-op).
 591   // see bug: http://code.google.com/p/chromium/issues/detail?id=4522
 592   chunk.hosts.clear();
 593   InsertAddChunkHost2PrefixUrls(&chunk, 1, "www.evil.com/",
 594                                 "www.evil.com/phishing.html",
 595                                 "www.evil.com/malware.html");
 596   chunks.clear();
 597   chunks.push_back(chunk);
 598   EXPECT_TRUE(database_->UpdateStarted(&lists));
 599   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
 600   database_->UpdateFinished(true);
 601
 602   GetListsInfo(&lists);
 603   EXPECT_TRUE(lists[0].name == safe_browsing_util::kMalwareList);
 604   EXPECT_EQ(lists[0].adds, "1-3,7");
 605   EXPECT_TRUE(lists[0].subs.empty());
 606
 607   // Test removing a single prefix from the add chunk.
 608   chunk.hosts.clear();
 609   InsertSubChunkHostPrefixUrl(&chunk, 4, 2, "www.evil.com/",
 610                               "www.evil.com/notevil1.html");
 611   chunks.clear();
 612   chunks.push_back(chunk);
 613   EXPECT_TRUE(database_->UpdateStarted(&lists));
 614   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
 615
 616   database_->UpdateFinished(true);
 617
 618   EXPECT_TRUE(database_->ContainsBrowseUrl(
 619       GURL("http://www.evil.com/phishing.html"),
 620       &prefix_hits, &cached_hashes, now));
 621   EXPECT_EQ(prefix_hits[0], SBPrefixForString("www.evil.com/phishing.html"));
 622   EXPECT_EQ(prefix_hits.size(), 1U);
 623
 624   EXPECT_FALSE(database_->ContainsBrowseUrl(
 625       GURL("http://www.evil.com/notevil1.html"),
 626       &prefix_hits, &cached_hashes, now));
 627   EXPECT_TRUE(prefix_hits.empty());
 628
 629   EXPECT_TRUE(database_->ContainsBrowseUrl(
 630       GURL("http://www.evil.com/notevil2.html"),
 631       &prefix_hits, &cached_hashes, now));
 632
 633   EXPECT_TRUE(database_->ContainsBrowseUrl(
 634       GURL("http://www.good.com/good1.html"),
 635       &prefix_hits, &cached_hashes, now));
 636
 637   EXPECT_TRUE(database_->ContainsBrowseUrl(
 638       GURL("http://www.good.com/good2.html"),
 639       &prefix_hits, &cached_hashes, now));
 640
 641   GetListsInfo(&lists);
 642   EXPECT_TRUE(lists[0].name == safe_browsing_util::kMalwareList);
 643   EXPECT_EQ(lists[0].subs, "4");
 644
 645   // Test the same sub chunk again.  This should be a no-op.
 646   // see bug: http://code.google.com/p/chromium/issues/detail?id=4522
 647   chunk.hosts.clear();
 648   InsertSubChunkHostPrefixUrl(&chunk, 4, 2, "www.evil.com/",
 649                               "www.evil.com/notevil1.html");
 650   chunks.clear();
 651   chunks.push_back(chunk);
 652
 653   EXPECT_TRUE(database_->UpdateStarted(&lists));
 654   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
 655   database_->UpdateFinished(true);
 656
 657   GetListsInfo(&lists);
 658   EXPECT_TRUE(lists[0].name == safe_browsing_util::kMalwareList);
 659   EXPECT_EQ(lists[0].subs, "4");
 660
 661   // Test removing all the prefixes from an add chunk.
 662   EXPECT_TRUE(database_->UpdateStarted(&lists));
 663   AddDelChunk(safe_browsing_util::kMalwareList, 2);
 664   database_->UpdateFinished(true);
 665
 666   EXPECT_FALSE(database_->ContainsBrowseUrl(
 667       GURL("http://www.evil.com/notevil2.html"),
 668       &prefix_hits, &cached_hashes, now));
 669
 670   EXPECT_FALSE(database_->ContainsBrowseUrl(
 671       GURL("http://www.good.com/good1.html"),
 672       &prefix_hits, &cached_hashes, now));
 673
 674   EXPECT_FALSE(database_->ContainsBrowseUrl(
 675       GURL("http://www.good.com/good2.html"),
 676       &prefix_hits, &cached_hashes, now));
 677
 678   GetListsInfo(&lists);
 679   EXPECT_TRUE(lists[0].name == safe_browsing_util::kMalwareList);
 680   EXPECT_EQ(lists[0].adds, "1,3,7");
 681   EXPECT_EQ(lists[0].subs, "4");
 682
 683   // The adddel command exposed a bug in the transaction code where any
 684   // transaction after it would fail.  Add a dummy entry and remove it to
 685   // make sure the transcation works fine.
 686   chunk.hosts.clear();
 687   InsertAddChunkHostPrefixUrl(&chunk, 44, "www.redherring.com/",
 688                               "www.redherring.com/index.html");
 689   chunks.clear();
 690   chunks.push_back(chunk);
 691   EXPECT_TRUE(database_->UpdateStarted(&lists));
 692   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
 693
 694   // Now remove the dummy entry.  If there are any problems with the
 695   // transactions, asserts will fire.
 696   AddDelChunk(safe_browsing_util::kMalwareList, 44);
 697
 698   // Test the subdel command.
 699   SubDelChunk(safe_browsing_util::kMalwareList, 4);
 700   database_->UpdateFinished(true);
 701
 702   GetListsInfo(&lists);
 703   EXPECT_TRUE(lists[0].name == safe_browsing_util::kMalwareList);
 704   EXPECT_EQ(lists[0].adds, "1,3,7");
 705   EXPECT_EQ(lists[0].subs, "");
 706
 707   // Test a sub command coming in before the add.
 708   chunk.hosts.clear();
 709   InsertSubChunkHost2PrefixUrls(&chunk, 5, 10,
 710                                 "www.notevilanymore.com/",
 711                                 "www.notevilanymore.com/index.html",
 712                                 "www.notevilanymore.com/good.html");
 713   chunks.clear();
 714   chunks.push_back(chunk);
 715   EXPECT_TRUE(database_->UpdateStarted(&lists));
 716   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
 717   database_->UpdateFinished(true);
 718
 719   EXPECT_FALSE(database_->ContainsBrowseUrl(
 720       GURL("http://www.notevilanymore.com/index.html"),
 721       &prefix_hits, &cached_hashes, now));
 722
 723   // Now insert the tardy add chunk and we don't expect them to appear
 724   // in database because of the previous sub chunk.
 725   chunk.hosts.clear();
 726   InsertAddChunkHost2PrefixUrls(&chunk, 10, "www.notevilanymore.com/",
 727                                 "www.notevilanymore.com/index.html",
 728                                 "www.notevilanymore.com/good.html");
 729   chunks.clear();
 730   chunks.push_back(chunk);
 731   EXPECT_TRUE(database_->UpdateStarted(&lists));
 732   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
 733   database_->UpdateFinished(true);
 734
 735   EXPECT_FALSE(database_->ContainsBrowseUrl(
 736       GURL("http://www.notevilanymore.com/index.html"),
 737       &prefix_hits, &cached_hashes, now));
 738
 739   EXPECT_FALSE(database_->ContainsBrowseUrl(
 740       GURL("http://www.notevilanymore.com/good.html"),
 741       &prefix_hits, &cached_hashes, now));
 742
 743   // Reset and reload the database.  The database will rely on the prefix set.
 744   database_.reset(new SafeBrowsingDatabaseNew);
 745   database_->Init(database_filename_);
 746
 747   // Check that a prefix still hits.
 748   EXPECT_TRUE(database_->ContainsBrowseUrl(
 749       GURL("http://www.evil.com/phishing.html"),
 750       &prefix_hits, &cached_hashes, now));
 751   EXPECT_EQ(prefix_hits[0], SBPrefixForString("www.evil.com/phishing.html"));
 752   EXPECT_EQ(prefix_hits.size(), 1U);
 753
 754   // Also check that it's not just always returning true in this case.
 755   EXPECT_FALSE(database_->ContainsBrowseUrl(
 756       GURL("http://www.evil.com/"),
 757       &prefix_hits, &cached_hashes, now));
 758
 759   // Check that the full hash is still present.
 760   EXPECT_TRUE(database_->ContainsBrowseUrl(
 761       GURL("http://www.evil.com/evil.html"),
 762       &prefix_hits, &cached_hashes, now));
 763   ASSERT_EQ(1U, prefix_hits.size());
 764   EXPECT_EQ(prefix_hits[0], SBPrefixForString("www.evil.com/evil.html"));
 765 }
 766
 767
 768 // Test adding zero length chunks to the database.
 769 TEST_F(SafeBrowsingDatabaseTest, ZeroSizeChunk) {
 770   SBChunkList chunks;
 771   SBChunk chunk;
 772
 773   // Populate with a couple of normal chunks.
 774   InsertAddChunkHost2PrefixUrls(&chunk, 1, "www.test.com/",
 775                                 "www.test.com/test1.html",
 776                                 "www.test.com/test2.html");
 777   chunks.clear();
 778   chunks.push_back(chunk);
 779
 780   chunk.hosts.clear();
 781   InsertAddChunkHost2PrefixUrls(&chunk, 10, "www.random.com/",
 782                                 "www.random.com/random1.html",
 783                                 "www.random.com/random2.html");
 784   chunks.push_back(chunk);
 785
 786   std::vector<SBListChunkRanges> lists;
 787   EXPECT_TRUE(database_->UpdateStarted(&lists));
 788   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
 789   database_->UpdateFinished(true);
 790
 791   // Add an empty ADD and SUB chunk.
 792   GetListsInfo(&lists);
 793   EXPECT_EQ(lists[0].adds, "1,10");
 794
 795   SBChunk empty_chunk;
 796   empty_chunk.chunk_number = 19;
 797   empty_chunk.is_add = true;
 798   chunks.clear();
 799   chunks.push_back(empty_chunk);
 800   EXPECT_TRUE(database_->UpdateStarted(&lists));
 801   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
 802   chunks.clear();
 803   empty_chunk.chunk_number = 7;
 804   empty_chunk.is_add = false;
 805   chunks.push_back(empty_chunk);
 806   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
 807   database_->UpdateFinished(true);
 808
 809   GetListsInfo(&lists);
 810   EXPECT_EQ(lists[0].adds, "1,10,19");
 811   EXPECT_EQ(lists[0].subs, "7");
 812
 813   // Add an empty chunk along with a couple that contain data. This should
 814   // result in the chunk range being reduced in size.
 815   empty_chunk.hosts.clear();
 816   InsertAddChunkHostPrefixUrl(&empty_chunk, 20, "www.notempy.com/",
 817                               "www.notempty.com/full1.html");
 818   chunks.clear();
 819   chunks.push_back(empty_chunk);
 820
 821   empty_chunk.chunk_number = 21;
 822   empty_chunk.is_add = true;
 823   empty_chunk.hosts.clear();
 824   chunks.push_back(empty_chunk);
 825
 826   empty_chunk.hosts.clear();
 827   InsertAddChunkHostPrefixUrl(&empty_chunk, 22, "www.notempy.com/",
 828                               "www.notempty.com/full2.html");
 829   chunks.push_back(empty_chunk);
 830
 831   EXPECT_TRUE(database_->UpdateStarted(&lists));
 832   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
 833   database_->UpdateFinished(true);
 834
 835   const Time now = Time::Now();
 836   std::vector<SBFullHashResult> cached_hashes;
 837   std::vector<SBPrefix> prefix_hits;
 838   EXPECT_TRUE(database_->ContainsBrowseUrl(
 839       GURL("http://www.notempty.com/full1.html"),
 840       &prefix_hits, &cached_hashes, now));
 841   EXPECT_TRUE(database_->ContainsBrowseUrl(
 842       GURL("http://www.notempty.com/full2.html"),
 843       &prefix_hits, &cached_hashes, now));
 844
 845   GetListsInfo(&lists);
 846   EXPECT_EQ(lists[0].adds, "1,10,19-22");
 847   EXPECT_EQ(lists[0].subs, "7");
 848
 849   // Handle AddDel and SubDel commands for empty chunks.
 850   EXPECT_TRUE(database_->UpdateStarted(&lists));
 851   AddDelChunk(safe_browsing_util::kMalwareList, 21);
 852   database_->UpdateFinished(true);
 853
 854   GetListsInfo(&lists);
 855   EXPECT_EQ(lists[0].adds, "1,10,19-20,22");
 856   EXPECT_EQ(lists[0].subs, "7");
 857
 858   EXPECT_TRUE(database_->UpdateStarted(&lists));
 859   SubDelChunk(safe_browsing_util::kMalwareList, 7);
 860   database_->UpdateFinished(true);
 861
 862   GetListsInfo(&lists);
 863   EXPECT_EQ(lists[0].adds, "1,10,19-20,22");
 864   EXPECT_EQ(lists[0].subs, "");
 865 }
 866
 867 // Utility function for setting up the database for the caching test.
 868 void SafeBrowsingDatabaseTest::PopulateDatabaseForCacheTest() {
 869   SBChunkList chunks;
 870   SBChunk chunk;
 871   // Add a simple chunk with one hostkey and cache it.
 872   InsertAddChunkHost2PrefixUrls(&chunk, 1, "www.evil.com/",
 873                                 "www.evil.com/phishing.html",
 874                                 "www.evil.com/malware.html");
 875   chunks.push_back(chunk);
 876
 877   std::vector<SBListChunkRanges> lists;
 878   EXPECT_TRUE(database_->UpdateStarted(&lists));
 879   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
 880   database_->UpdateFinished(true);
 881
 882   // Add the GetHash results to the cache.
 883   SBFullHashResult full_hash;
 884   full_hash.hash = SBFullHashForString("www.evil.com/phishing.html");
 885   full_hash.list_id = safe_browsing_util::MALWARE;
 886
 887   std::vector<SBFullHashResult> results;
 888   results.push_back(full_hash);
 889
 890   full_hash.hash = SBFullHashForString("www.evil.com/malware.html");
 891   results.push_back(full_hash);
 892
 893   std::vector<SBPrefix> prefixes;
 894   database_->CacheHashResults(prefixes, results);
 895 }
 896
 897 TEST_F(SafeBrowsingDatabaseTest, HashCaching) {
 898   PopulateDatabaseForCacheTest();
 899
 900   // We should have both full hashes in the cache.
 901   EXPECT_EQ(2U, database_->cached_browse_hashes_.size());
 902
 903   // Test the cache lookup for the first prefix.
 904   std::vector<SBPrefix> prefixes;
 905   std::vector<SBFullHashResult> cached_hashes;
 906   database_->ContainsBrowseUrl(
 907       GURL("http://www.evil.com/phishing.html"),
 908       &prefixes, &cached_hashes, Time::Now());
 909   ASSERT_EQ(1U, cached_hashes.size());
 910   EXPECT_TRUE(
 911       SBFullHashEqual(cached_hashes[0].hash,
 912                       SBFullHashForString("www.evil.com/phishing.html")));
 913
 914   prefixes.clear();
 915   cached_hashes.clear();
 916
 917   // Test the cache lookup for the second prefix.
 918   database_->ContainsBrowseUrl(
 919       GURL("http://www.evil.com/malware.html"),
 920       &prefixes, &cached_hashes, Time::Now());
 921   ASSERT_EQ(1U, cached_hashes.size());
 922   EXPECT_TRUE(
 923       SBFullHashEqual(cached_hashes[0].hash,
 924                       SBFullHashForString("www.evil.com/malware.html")));
 925
 926   prefixes.clear();
 927   cached_hashes.clear();
 928
 929   // Test removing a prefix via a sub chunk.
 930   SBChunk chunk;
 931   SBChunkList chunks;
 932   InsertSubChunkHostPrefixUrl(&chunk, 2, 1, "www.evil.com/",
 933                               "www.evil.com/phishing.html");
 934   chunks.push_back(chunk);
 935
 936   std::vector<SBListChunkRanges> lists;
 937   EXPECT_TRUE(database_->UpdateStarted(&lists));
 938   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
 939   database_->UpdateFinished(true);
 940
 941   // This prefix should still be there, but the fullhash is gone.
 942   EXPECT_TRUE(database_->ContainsBrowseUrl(
 943       GURL("http://www.evil.com/malware.html"),
 944       &prefixes, &cached_hashes, Time::Now()));
 945   ASSERT_EQ(1U, prefixes.size());
 946   EXPECT_EQ(SBPrefixForString("www.evil.com/malware.html"), prefixes[0]);
 947   EXPECT_TRUE(cached_hashes.empty());
 948   prefixes.clear();
 949   cached_hashes.clear();
 950
 951   // This prefix should be gone.
 952   database_->ContainsBrowseUrl(
 953       GURL("http://www.evil.com/phishing.html"),
 954       &prefixes, &cached_hashes, Time::Now());
 955   EXPECT_TRUE(cached_hashes.empty());
 956
 957   prefixes.clear();
 958   cached_hashes.clear();
 959
 960   // Test that an AddDel for the original chunk removes the last cached entry.
 961   EXPECT_TRUE(database_->UpdateStarted(&lists));
 962   AddDelChunk(safe_browsing_util::kMalwareList, 1);
 963   database_->UpdateFinished(true);
 964   database_->ContainsBrowseUrl(
 965       GURL("http://www.evil.com/malware.html"),
 966       &prefixes, &cached_hashes, Time::Now());
 967   EXPECT_TRUE(cached_hashes.empty());
 968   EXPECT_TRUE(database_->cached_browse_hashes_.empty());
 969
 970   prefixes.clear();
 971   cached_hashes.clear();
 972
 973   // Test that the cache won't return expired values. First we have to adjust
 974   // the cached entries' received time to make them older, since the database
 975   // cache insert uses Time::Now(). First, store some entries.
 976   PopulateDatabaseForCacheTest();
 977
 978   std::vector<SBFullHashCached>* hash_cache = &database_->cached_browse_hashes_;
 979   EXPECT_EQ(2U, hash_cache->size());
 980
 981   // Now adjust one of the entries times to be in the past.
 982   base::Time expired = base::Time::Now() - base::TimeDelta::FromMinutes(60);
 983   const SBPrefix key = SBPrefixForString("www.evil.com/malware.html");
 984   std::vector<SBFullHashCached>::iterator iter;
 985   for (iter = hash_cache->begin(); iter != hash_cache->end(); ++iter) {
 986     if (iter->hash.prefix == key) {
 987       iter->received = static_cast<int32>(expired.ToTimeT());
 988       break;
 989     }
 990   }
 991   EXPECT_TRUE(iter != hash_cache->end());
 992
 993   database_->ContainsBrowseUrl(
 994       GURL("http://www.evil.com/malware.html"),
 995       &prefixes, &cached_hashes, expired);
 996   EXPECT_TRUE(cached_hashes.empty());
 997
 998   // This entry should still exist.
 999   database_->ContainsBrowseUrl(
1000       GURL("http://www.evil.com/phishing.html"),
1001       &prefixes, &cached_hashes, expired);
1002   EXPECT_EQ(1U, cached_hashes.size());
1003
1004   // Testing prefix miss caching. First, we clear out the existing database,
1005   // Since PopulateDatabaseForCacheTest() doesn't handle adding duplicate
1006   // chunks.
1007   EXPECT_TRUE(database_->UpdateStarted(&lists));
1008   AddDelChunk(safe_browsing_util::kMalwareList, 1);
1009   database_->UpdateFinished(true);
1010
1011   std::vector<SBPrefix> prefix_misses;
1012   std::vector<SBFullHashResult> empty_full_hash;
1013   prefix_misses.push_back(SBPrefixForString("http://www.bad.com/malware.html"));
1014   prefix_misses.push_back(
1015       SBPrefixForString("http://www.bad.com/phishing.html"));
1016   database_->CacheHashResults(prefix_misses, empty_full_hash);
1017
1018   // Prefixes with no full results are misses.
1019   EXPECT_EQ(2U, database_->prefix_miss_cache_.size());
1020
1021   // Update the database.
1022   PopulateDatabaseForCacheTest();
1023
1024   // Prefix miss cache should be cleared.
1025   EXPECT_TRUE(database_->prefix_miss_cache_.empty());
1026
1027   // Cache a GetHash miss for a particular prefix, and even though the prefix is
1028   // in the database, it is flagged as a miss so looking up the associated URL
1029   // will not succeed.
1030   prefixes.clear();
1031   cached_hashes.clear();
1032   prefix_misses.clear();
1033   empty_full_hash.clear();
1034   prefix_misses.push_back(SBPrefixForString("www.evil.com/phishing.html"));
1035   database_->CacheHashResults(prefix_misses, empty_full_hash);
1036   EXPECT_FALSE(database_->ContainsBrowseUrl(
1037       GURL("http://www.evil.com/phishing.html"),
1038       &prefixes, &cached_hashes, Time::Now()));
1039
1040   prefixes.clear();
1041   cached_hashes.clear();
1042
1043   // Test receiving a full add chunk.
1044   chunk.hosts.clear();
1045   InsertAddChunkHost2FullHashes(&chunk, 20, "www.fullevil.com/",
1046                                 "www.fullevil.com/bad1.html",
1047                                 "www.fullevil.com/bad2.html");
1048   chunks.clear();
1049   chunks.push_back(chunk);
1050   EXPECT_TRUE(database_->UpdateStarted(&lists));
1051   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
1052   database_->UpdateFinished(true);
1053
1054   EXPECT_TRUE(database_->ContainsBrowseUrl(
1055       GURL("http://www.fullevil.com/bad1.html"),
1056       &prefixes, &cached_hashes, Time::Now()));
1057   EXPECT_TRUE(cached_hashes.empty());
1058   ASSERT_EQ(1U, prefixes.size());
1059   EXPECT_EQ(SBPrefixForString("www.fullevil.com/bad1.html"), prefixes[0]);
1060   prefixes.clear();
1061   cached_hashes.clear();
1062
1063   EXPECT_TRUE(database_->ContainsBrowseUrl(
1064       GURL("http://www.fullevil.com/bad2.html"),
1065       &prefixes, &cached_hashes, Time::Now()));
1066   EXPECT_TRUE(cached_hashes.empty());
1067   ASSERT_EQ(1U, prefixes.size());
1068   EXPECT_EQ(SBPrefixForString("www.fullevil.com/bad2.html"), prefixes[0]);
1069   prefixes.clear();
1070   cached_hashes.clear();
1071
1072   // Test receiving a full sub chunk, which will remove one of the full adds.
1073   chunk.hosts.clear();
1074   InsertSubChunkHostFullHash(&chunk, 200, 20,
1075                              "www.fullevil.com/",
1076                              "www.fullevil.com/bad1.html");
1077   chunks.clear();
1078   chunks.push_back(chunk);
1079   EXPECT_TRUE(database_->UpdateStarted(&lists));
1080   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
1081   database_->UpdateFinished(true);
1082
1083   EXPECT_FALSE(database_->ContainsBrowseUrl(
1084       GURL("http://www.fullevil.com/bad1.html"),
1085       &prefixes, &cached_hashes, Time::Now()));
1086   EXPECT_TRUE(cached_hashes.empty());
1087
1088   // There should be one remaining full add.
1089   EXPECT_TRUE(database_->ContainsBrowseUrl(
1090       GURL("http://www.fullevil.com/bad2.html"),
1091       &prefixes, &cached_hashes, Time::Now()));
1092   EXPECT_TRUE(cached_hashes.empty());
1093   ASSERT_EQ(1U, prefixes.size());
1094   EXPECT_EQ(SBPrefixForString("www.fullevil.com/bad2.html"), prefixes[0]);
1095   prefixes.clear();
1096   cached_hashes.clear();
1097
1098   // Now test an AddDel for the remaining full add.
1099   EXPECT_TRUE(database_->UpdateStarted(&lists));
1100   AddDelChunk(safe_browsing_util::kMalwareList, 20);
1101   database_->UpdateFinished(true);
1102
1103   EXPECT_FALSE(database_->ContainsBrowseUrl(
1104       GURL("http://www.fullevil.com/bad1.html"),
1105       &prefixes, &cached_hashes, Time::Now()));
1106   EXPECT_FALSE(database_->ContainsBrowseUrl(
1107       GURL("http://www.fullevil.com/bad2.html"),
1108       &prefixes, &cached_hashes, Time::Now()));
1109
1110   // Add a fullhash which has a prefix collision for a known url.
1111   static const char kExampleFine[] = "www.example.com/fine.html";
1112   static const char kExampleCollision[] =
1113       "www.example.com/3123364814/malware.htm";
1114   ASSERT_EQ(SBPrefixForString(kExampleFine),
1115             SBPrefixForString(kExampleCollision));
1116   EXPECT_TRUE(database_->UpdateStarted(&lists));
1117   {
1118     SBChunkList chunks;
1119     SBChunk chunk;
1120     InsertAddChunkHostPrefixUrl(&chunk, 21, "www.example.com/",
1121                                 kExampleCollision);
1122     chunks.push_back(chunk);
1123     database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
1124   }
1125   database_->UpdateFinished(true);
1126
1127   // Cache gethash response for |kExampleCollision|.
1128   {
1129     SBFullHashResult result;
1130     result.hash = SBFullHashForString(kExampleCollision);
1131     result.list_id = safe_browsing_util::MALWARE;
1132     database_->CacheHashResults(std::vector<SBPrefix>(1, result.hash.prefix),
1133                                 std::vector<SBFullHashResult>(1, result));
1134   }
1135
1136   // Expect a prefix hit due to the collision between |kExampleFine| and
1137   // |kExampleCollision|, with the gethash showing only |kExampleCollision|.
1138   EXPECT_TRUE(database_->ContainsBrowseUrl(
1139       GURL(std::string("http://") + kExampleFine),
1140       &prefixes, &cached_hashes, Time::Now()));
1141   ASSERT_EQ(1U, prefixes.size());
1142   EXPECT_EQ(SBPrefixForString(kExampleFine), prefixes[0]);
1143   ASSERT_EQ(1U, cached_hashes.size());
1144   EXPECT_TRUE(SBFullHashEqual(cached_hashes[0].hash,
1145                               SBFullHashForString(kExampleCollision)));
1146 }
1147
1148 // Test that corrupt databases are appropriately handled, even if the
1149 // corruption is detected in the midst of the update.
1150 // TODO(shess): Disabled until ScopedLogMessageIgnorer resolved.
1151 // http://crbug.com/56448
1152 TEST_F(SafeBrowsingDatabaseTest, DISABLED_FileCorruptionHandling) {
1153   // Re-create the database in a captive message loop so that we can
1154   // influence task-posting.  Database specifically needs to the
1155   // file-backed.
1156   database_.reset();
1157   base::MessageLoop loop;
1158   SafeBrowsingStoreFile* store = new SafeBrowsingStoreFile();
1159   database_.reset(new SafeBrowsingDatabaseNew(store, NULL, NULL, NULL, NULL,
1160                                               NULL, NULL));
1161   database_->Init(database_filename_);
1162
1163   // This will cause an empty database to be created.
1164   std::vector<SBListChunkRanges> lists;
1165   EXPECT_TRUE(database_->UpdateStarted(&lists));
1166   database_->UpdateFinished(true);
1167
1168   // Create a sub chunk to insert.
1169   SBChunkList chunks;
1170   SBChunk chunk;
1171   SBChunkHost host;
1172   host.host = SBPrefixForString("www.subbed.com/");
1173   host.entry = SBEntry::Create(SBEntry::SUB_PREFIX, 1);
1174   host.entry->set_chunk_id(7);
1175   host.entry->SetChunkIdAtPrefix(0, 19);
1176   host.entry->SetPrefixAt(0, SBPrefixForString("www.subbed.com/notevil1.html"));
1177   chunk.chunk_number = 7;
1178   chunk.is_add = false;
1179   chunk.hosts.clear();
1180   chunk.hosts.push_back(host);
1181   chunks.clear();
1182   chunks.push_back(chunk);
1183
1184   // Corrupt the file by corrupting the checksum, which is not checked
1185   // until the entire table is read in |UpdateFinished()|.
1186   FILE* fp = base::OpenFile(database_filename_, "r+");
1187   ASSERT_TRUE(fp);
1188   ASSERT_NE(-1, fseek(fp, -8, SEEK_END));
1189   for (size_t i = 0; i < 8; ++i) {
1190     fputc('!', fp);
1191   }
1192   fclose(fp);
1193
1194   {
1195     // The following code will cause DCHECKs, so suppress the crashes.
1196     ScopedLogMessageIgnorer ignorer;
1197
1198     // Start an update.  The insert will fail due to corruption.
1199     EXPECT_TRUE(database_->UpdateStarted(&lists));
1200     database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
1201     database_->UpdateFinished(true);
1202
1203     // Database file still exists until the corruption handler has run.
1204     EXPECT_TRUE(base::PathExists(database_filename_));
1205
1206     // Flush through the corruption-handler task.
1207     VLOG(1) << "Expect failed check on: SafeBrowsing database reset";
1208     base::MessageLoop::current()->RunUntilIdle();
1209   }
1210
1211   // Database file should not exist.
1212   EXPECT_FALSE(base::PathExists(database_filename_));
1213
1214   // Run the update again successfully.
1215   EXPECT_TRUE(database_->UpdateStarted(&lists));
1216   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
1217   database_->UpdateFinished(true);
1218   EXPECT_TRUE(base::PathExists(database_filename_));
1219
1220   database_.reset();
1221 }
1222
1223 // Checks database reading and writing.
1224 TEST_F(SafeBrowsingDatabaseTest, ContainsDownloadUrl) {
1225   database_.reset();
1226   base::MessageLoop loop;
1227   SafeBrowsingStoreFile* browse_store = new SafeBrowsingStoreFile();
1228   SafeBrowsingStoreFile* download_store = new SafeBrowsingStoreFile();
1229   SafeBrowsingStoreFile* csd_whitelist_store = new SafeBrowsingStoreFile();
1230   database_.reset(new SafeBrowsingDatabaseNew(browse_store,
1231                                               download_store,
1232                                               csd_whitelist_store,
1233                                               NULL,
1234                                               NULL,
1235                                               NULL,
1236                                               NULL));
1237   database_->Init(database_filename_);
1238
1239   const char kEvil1Host[] = "www.evil1.com/";
1240   const char kEvil1Url1[] = "www.evil1.com/download1/";
1241   const char kEvil1Url2[] = "www.evil1.com/download2.html";
1242
1243   SBChunkList chunks;
1244   SBChunk chunk;
1245   // Add a simple chunk with one hostkey for download url list.
1246   InsertAddChunkHost2PrefixUrls(&chunk, 1, kEvil1Host,
1247                                 kEvil1Url1, kEvil1Url2);
1248   chunks.push_back(chunk);
1249   std::vector<SBListChunkRanges> lists;
1250   EXPECT_TRUE(database_->UpdateStarted(&lists));
1251   database_->InsertChunks(safe_browsing_util::kBinUrlList, chunks);
1252   database_->UpdateFinished(true);
1253
1254   std::vector<SBPrefix> prefix_hits;
1255   std::vector<GURL> urls(1);
1256
1257   urls[0] = GURL(std::string("http://") + kEvil1Url1);
1258   EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1259   ASSERT_EQ(prefix_hits.size(), 1U);
1260   EXPECT_EQ(prefix_hits[0], SBPrefixForString(kEvil1Url1));
1261
1262   urls[0] = GURL(std::string("http://") + kEvil1Url2);
1263   EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1264   ASSERT_EQ(prefix_hits.size(), 1U);
1265   EXPECT_EQ(prefix_hits[0], SBPrefixForString(kEvil1Url2));
1266
1267   urls[0] = GURL(std::string("https://") + kEvil1Url2);
1268   EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1269   ASSERT_EQ(prefix_hits.size(), 1U);
1270   EXPECT_EQ(prefix_hits[0], SBPrefixForString(kEvil1Url2));
1271
1272   urls[0] = GURL(std::string("ftp://") + kEvil1Url2);
1273   EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1274   ASSERT_EQ(prefix_hits.size(), 1U);
1275   EXPECT_EQ(prefix_hits[0], SBPrefixForString(kEvil1Url2));
1276
1277   urls[0] = GURL("http://www.randomevil.com");
1278   EXPECT_FALSE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1279
1280   // Should match with query args stripped.
1281   urls[0] = GURL(std::string("http://") + kEvil1Url2 + "?blah");
1282   EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1283   ASSERT_EQ(prefix_hits.size(), 1U);
1284   EXPECT_EQ(prefix_hits[0], SBPrefixForString(kEvil1Url2));
1285
1286   // Should match with extra path stuff and query args stripped.
1287   urls[0] = GURL(std::string("http://") + kEvil1Url1 + "foo/bar?blah");
1288   EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1289   ASSERT_EQ(prefix_hits.size(), 1U);
1290   EXPECT_EQ(prefix_hits[0], SBPrefixForString(kEvil1Url1));
1291
1292   // First hit in redirect chain is malware.
1293   urls.clear();
1294   urls.push_back(GURL(std::string("http://") + kEvil1Url1));
1295   urls.push_back(GURL("http://www.randomevil.com"));
1296   EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1297   ASSERT_EQ(prefix_hits.size(), 1U);
1298   EXPECT_EQ(prefix_hits[0], SBPrefixForString(kEvil1Url1));
1299
1300   // Middle hit in redirect chain is malware.
1301   urls.clear();
1302   urls.push_back(GURL("http://www.randomevil.com"));
1303   urls.push_back(GURL(std::string("http://") + kEvil1Url1));
1304   urls.push_back(GURL("http://www.randomevil2.com"));
1305   EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1306   ASSERT_EQ(prefix_hits.size(), 1U);
1307   EXPECT_EQ(prefix_hits[0], SBPrefixForString(kEvil1Url1));
1308
1309   // Final hit in redirect chain is malware.
1310   urls.clear();
1311   urls.push_back(GURL("http://www.randomevil.com"));
1312   urls.push_back(GURL(std::string("http://") + kEvil1Url1));
1313   EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1314   ASSERT_EQ(prefix_hits.size(), 1U);
1315   EXPECT_EQ(prefix_hits[0], SBPrefixForString(kEvil1Url1));
1316
1317   // Multiple hits in redirect chain are in malware list.
1318   urls.clear();
1319   urls.push_back(GURL(std::string("http://") + kEvil1Url1));
1320   urls.push_back(GURL(std::string("https://") + kEvil1Url2));
1321   EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1322   ASSERT_EQ(prefix_hits.size(), 2U);
1323   EXPECT_EQ(prefix_hits[0], SBPrefixForString(kEvil1Url1));
1324   EXPECT_EQ(prefix_hits[1], SBPrefixForString(kEvil1Url2));
1325   database_.reset();
1326 }
1327
1328 // Checks that the whitelists are handled properly.
1329 TEST_F(SafeBrowsingDatabaseTest, Whitelists) {
1330   database_.reset();
1331   // We expect all calls to ContainsCsdWhitelistedUrl in particular to be made
1332   // from the IO thread.  In general the whitelist lookups are thread-safe.
1333   content::TestBrowserThreadBundle thread_bundle_;
1334
1335   // If the whitelist is disabled everything should match the whitelist.
1336   database_.reset(new SafeBrowsingDatabaseNew(new SafeBrowsingStoreFile(),
1337                                               NULL, NULL, NULL, NULL, NULL,
1338                                               NULL));
1339   database_->Init(database_filename_);
1340   EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1341       GURL(std::string("http://www.phishing.com/"))));
1342   EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1343       GURL(std::string("http://www.phishing.com/"))));
1344   EXPECT_TRUE(database_->ContainsDownloadWhitelistedString("asdf"));
1345
1346   SafeBrowsingStoreFile* browse_store = new SafeBrowsingStoreFile();
1347   SafeBrowsingStoreFile* csd_whitelist_store = new SafeBrowsingStoreFile();
1348   SafeBrowsingStoreFile* download_whitelist_store = new SafeBrowsingStoreFile();
1349   SafeBrowsingStoreFile* extension_blacklist_store =
1350       new SafeBrowsingStoreFile();
1351   database_.reset(new SafeBrowsingDatabaseNew(browse_store, NULL,
1352                                               csd_whitelist_store,
1353                                               download_whitelist_store,
1354                                               extension_blacklist_store,
1355                                               NULL, NULL));
1356   database_->Init(database_filename_);
1357
1358   const char kGood1Host[] = "www.good1.com/";
1359   const char kGood1Url1[] = "www.good1.com/a/b.html";
1360   const char kGood1Url2[] = "www.good1.com/b/";
1361
1362   const char kGood2Host[] = "www.good2.com/";
1363   const char kGood2Url1[] = "www.good2.com/c";  // Should match '/c/bla'.
1364
1365   // good3.com/a/b/c/d/e/f/g/ should match because it's a whitelist.
1366   const char kGood3Host[] = "good3.com/";
1367   const char kGood3Url1[] = "good3.com/";
1368
1369   const char kGoodString[] = "good_string";
1370
1371   SBChunkList download_chunks, csd_chunks;
1372   SBChunk chunk;
1373   // Add two simple chunks to the csd whitelist.
1374   InsertAddChunkHost2FullHashes(&chunk, 1, kGood1Host,
1375                                 kGood1Url1, kGood1Url2);
1376   csd_chunks.push_back(chunk);
1377
1378   chunk.hosts.clear();
1379   InsertAddChunkHostFullHashes(&chunk, 2, kGood2Host, kGood2Url1);
1380   csd_chunks.push_back(chunk);
1381
1382   chunk.hosts.clear();
1383   InsertAddChunkHostFullHashes(&chunk, 2, kGood2Host, kGood2Url1);
1384   download_chunks.push_back(chunk);
1385
1386   chunk.hosts.clear();
1387   InsertAddChunkHostFullHashes(&chunk, 3, kGoodString, kGoodString);
1388   download_chunks.push_back(chunk);
1389
1390   chunk.hosts.clear();
1391   InsertAddChunkHostFullHashes(&chunk, 4, kGood3Host, kGood3Url1);
1392   download_chunks.push_back(chunk);
1393
1394   std::vector<SBListChunkRanges> lists;
1395   EXPECT_TRUE(database_->UpdateStarted(&lists));
1396   database_->InsertChunks(safe_browsing_util::kCsdWhiteList,
1397                           csd_chunks);
1398   database_->InsertChunks(safe_browsing_util::kDownloadWhiteList,
1399                           download_chunks);
1400   database_->UpdateFinished(true);
1401
1402   EXPECT_FALSE(database_->ContainsCsdWhitelistedUrl(
1403       GURL(std::string("http://") + kGood1Host)));
1404
1405   EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1406       GURL(std::string("http://") + kGood1Url1)));
1407   EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1408       GURL(std::string("http://") + kGood1Url1 + "?a=b")));
1409
1410   EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1411       GURL(std::string("http://") + kGood1Url2)));
1412   EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1413       GURL(std::string("http://") + kGood1Url2 + "/c.html")));
1414
1415   EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1416       GURL(std::string("https://") + kGood1Url2 + "/c.html")));
1417
1418   EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1419       GURL(std::string("http://") + kGood2Url1 + "/c")));
1420   EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1421       GURL(std::string("http://") + kGood2Url1 + "/c?bla")));
1422   EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1423       GURL(std::string("http://") + kGood2Url1 + "/c/bla")));
1424
1425   EXPECT_FALSE(database_->ContainsCsdWhitelistedUrl(
1426       GURL(std::string("http://www.google.com/"))));
1427
1428   EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1429       GURL(std::string("http://") + kGood2Url1 + "/c")));
1430   EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1431       GURL(std::string("http://") + kGood2Url1 + "/c?bla")));
1432   EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1433       GURL(std::string("http://") + kGood2Url1 + "/c/bla")));
1434
1435   EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1436       GURL(std::string("http://good3.com/a/b/c/d/e/f/g/"))));
1437   EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1438       GURL(std::string("http://a.b.good3.com/"))));
1439
1440   EXPECT_FALSE(database_->ContainsDownloadWhitelistedString("asdf"));
1441   EXPECT_TRUE(database_->ContainsDownloadWhitelistedString(kGoodString));
1442
1443   EXPECT_FALSE(database_->ContainsDownloadWhitelistedUrl(
1444       GURL(std::string("http://www.google.com/"))));
1445
1446   // Test only add the malware IP killswitch
1447   csd_chunks.clear();
1448   chunk.hosts.clear();
1449   InsertAddChunkHostFullHashes(
1450       &chunk, 15, "sb-ssl.google.com/",
1451       "sb-ssl.google.com/safebrowsing/csd/killswitch_malware");
1452   csd_chunks.push_back(chunk);
1453   EXPECT_TRUE(database_->UpdateStarted(&lists));
1454   database_->InsertChunks(safe_browsing_util::kCsdWhiteList, csd_chunks);
1455   database_->UpdateFinished(true);
1456
1457   EXPECT_TRUE(database_->IsMalwareIPMatchKillSwitchOn());
1458
1459   // Test that the kill-switch works as intended.
1460   csd_chunks.clear();
1461   download_chunks.clear();
1462   lists.clear();
1463   chunk.hosts.clear();
1464   InsertAddChunkHostFullHashes(&chunk, 5, "sb-ssl.google.com/",
1465                                "sb-ssl.google.com/safebrowsing/csd/killswitch");
1466   csd_chunks.push_back(chunk);
1467   chunk.hosts.clear();
1468   InsertAddChunkHostFullHashes(&chunk, 5, "sb-ssl.google.com/",
1469                                "sb-ssl.google.com/safebrowsing/csd/killswitch");
1470   download_chunks.push_back(chunk);
1471
1472   EXPECT_TRUE(database_->UpdateStarted(&lists));
1473   database_->InsertChunks(safe_browsing_util::kCsdWhiteList, csd_chunks);
1474   database_->InsertChunks(safe_browsing_util::kDownloadWhiteList,
1475                           download_chunks);
1476   database_->UpdateFinished(true);
1477
1478   EXPECT_TRUE(database_->IsMalwareIPMatchKillSwitchOn());
1479   EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1480       GURL(std::string("https://") + kGood1Url2 + "/c.html")));
1481   EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1482       GURL(std::string("http://www.google.com/"))));
1483   EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1484       GURL(std::string("http://www.phishing_url.com/"))));
1485
1486   EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1487       GURL(std::string("https://") + kGood1Url2 + "/c.html")));
1488   EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1489       GURL(std::string("http://www.google.com/"))));
1490   EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1491       GURL(std::string("http://www.phishing_url.com/"))));
1492
1493   EXPECT_TRUE(database_->ContainsDownloadWhitelistedString("asdf"));
1494   EXPECT_TRUE(database_->ContainsDownloadWhitelistedString(kGoodString));
1495
1496   // Remove the kill-switch and verify that we can recover.
1497   csd_chunks.clear();
1498   download_chunks.clear();
1499   lists.clear();
1500   SBChunk sub_chunk;
1501   InsertSubChunkHostFullHash(&sub_chunk, 1, 5,
1502                              "sb-ssl.google.com/",
1503                              "sb-ssl.google.com/safebrowsing/csd/killswitch");
1504   csd_chunks.push_back(sub_chunk);
1505
1506   sub_chunk.hosts.clear();
1507   InsertSubChunkHostFullHash(
1508       &sub_chunk, 10, 15, "sb-ssl.google.com/",
1509       "sb-ssl.google.com/safebrowsing/csd/killswitch_malware");
1510   csd_chunks.push_back(sub_chunk);
1511
1512   sub_chunk.hosts.clear();
1513   InsertSubChunkHostFullHash(&sub_chunk, 1, 5,
1514                              "sb-ssl.google.com/",
1515                              "sb-ssl.google.com/safebrowsing/csd/killswitch");
1516   download_chunks.push_back(sub_chunk);
1517
1518   EXPECT_TRUE(database_->UpdateStarted(&lists));
1519   database_->InsertChunks(safe_browsing_util::kCsdWhiteList, csd_chunks);
1520   database_->InsertChunks(safe_browsing_util::kDownloadWhiteList,
1521                           download_chunks);
1522   database_->UpdateFinished(true);
1523
1524   EXPECT_FALSE(database_->IsMalwareIPMatchKillSwitchOn());
1525   EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1526       GURL(std::string("https://") + kGood1Url2 + "/c.html")));
1527   EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1528       GURL(std::string("https://") + kGood2Url1 + "/c/bla")));
1529   EXPECT_FALSE(database_->ContainsCsdWhitelistedUrl(
1530       GURL(std::string("http://www.google.com/"))));
1531   EXPECT_FALSE(database_->ContainsCsdWhitelistedUrl(
1532       GURL(std::string("http://www.phishing_url.com/"))));
1533
1534   EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1535       GURL(std::string("https://") + kGood2Url1 + "/c/bla")));
1536   EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1537       GURL(std::string("https://good3.com/"))));
1538   EXPECT_TRUE(database_->ContainsDownloadWhitelistedString(kGoodString));
1539   EXPECT_FALSE(database_->ContainsDownloadWhitelistedUrl(
1540       GURL(std::string("http://www.google.com/"))));
1541   EXPECT_FALSE(database_->ContainsDownloadWhitelistedUrl(
1542       GURL(std::string("http://www.phishing_url.com/"))));
1543   EXPECT_FALSE(database_->ContainsDownloadWhitelistedString("asdf"));
1544
1545   database_.reset();
1546 }
1547
1548 // Test to make sure we could insert chunk list that
1549 // contains entries for the same host.
1550 TEST_F(SafeBrowsingDatabaseTest, SameHostEntriesOkay) {
1551   SBChunk chunk;
1552
1553   // Add a malware add chunk with two entries of the same host.
1554   InsertAddChunkHostPrefixUrl(&chunk, 1, "www.evil.com/",
1555                               "www.evil.com/malware1.html");
1556   InsertAddChunkHostPrefixUrl(&chunk, 1, "www.evil.com/",
1557                               "www.evil.com/malware2.html");
1558   SBChunkList chunks;
1559   chunks.push_back(chunk);
1560
1561   // Insert the testing chunks into database.
1562   std::vector<SBListChunkRanges> lists;
1563   EXPECT_TRUE(database_->UpdateStarted(&lists));
1564   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
1565   database_->UpdateFinished(true);
1566
1567   GetListsInfo(&lists);
1568   EXPECT_EQ(std::string(safe_browsing_util::kMalwareList), lists[0].name);
1569   EXPECT_EQ("1", lists[0].adds);
1570   EXPECT_TRUE(lists[0].subs.empty());
1571
1572   // Add a phishing add chunk with two entries of the same host.
1573   chunk.hosts.clear();
1574   InsertAddChunkHostPrefixUrl(&chunk, 47, "www.evil.com/",
1575                               "www.evil.com/phishing1.html");
1576   InsertAddChunkHostPrefixUrl(&chunk, 47, "www.evil.com/",
1577                               "www.evil.com/phishing2.html");
1578   chunks.clear();
1579   chunks.push_back(chunk);
1580
1581   EXPECT_TRUE(database_->UpdateStarted(&lists));
1582   database_->InsertChunks(safe_browsing_util::kPhishingList, chunks);
1583   database_->UpdateFinished(true);
1584
1585   GetListsInfo(&lists);
1586   EXPECT_EQ(std::string(safe_browsing_util::kMalwareList), lists[0].name);
1587   EXPECT_EQ("1", lists[0].adds);
1588   EXPECT_EQ(std::string(safe_browsing_util::kPhishingList), lists[1].name);
1589   EXPECT_EQ("47", lists[1].adds);
1590
1591   const Time now = Time::Now();
1592   std::vector<SBPrefix> prefixes;
1593   std::vector<SBFullHashResult> cached_hashes;
1594   std::vector<SBPrefix> prefix_hits;
1595
1596   EXPECT_TRUE(database_->ContainsBrowseUrl(
1597       GURL("http://www.evil.com/malware1.html"),
1598       &prefixes, &cached_hashes, now));
1599   EXPECT_TRUE(database_->ContainsBrowseUrl(
1600       GURL("http://www.evil.com/malware2.html"),
1601       &prefixes, &cached_hashes, now));
1602   EXPECT_TRUE(database_->ContainsBrowseUrl(
1603       GURL("http://www.evil.com/phishing1.html"),
1604       &prefixes, &cached_hashes, now));
1605   EXPECT_TRUE(database_->ContainsBrowseUrl(
1606       GURL("http://www.evil.com/phishing2.html"),
1607       &prefixes, &cached_hashes, now));
1608
1609   // Test removing a single prefix from the add chunk.
1610   // Remove the prefix that added first.
1611   chunk.hosts.clear();
1612   InsertSubChunkHostPrefixUrl(&chunk, 4, 1, "www.evil.com/",
1613                               "www.evil.com/malware1.html");
1614   chunks.clear();
1615   chunks.push_back(chunk);
1616   EXPECT_TRUE(database_->UpdateStarted(&lists));
1617   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
1618   database_->UpdateFinished(true);
1619
1620   // Remove the prefix that added last.
1621   chunk.hosts.clear();
1622   InsertSubChunkHostPrefixUrl(&chunk, 5, 47, "www.evil.com/",
1623                               "www.evil.com/phishing2.html");
1624   chunks.clear();
1625   chunks.push_back(chunk);
1626   EXPECT_TRUE(database_->UpdateStarted(&lists));
1627   database_->InsertChunks(safe_browsing_util::kPhishingList, chunks);
1628   database_->UpdateFinished(true);
1629
1630   // Verify that the database contains urls expected.
1631   EXPECT_FALSE(database_->ContainsBrowseUrl(
1632       GURL("http://www.evil.com/malware1.html"),
1633       &prefixes, &cached_hashes, now));
1634   EXPECT_TRUE(database_->ContainsBrowseUrl(
1635       GURL("http://www.evil.com/malware2.html"),
1636       &prefixes, &cached_hashes, now));
1637   EXPECT_TRUE(database_->ContainsBrowseUrl(
1638       GURL("http://www.evil.com/phishing1.html"),
1639       &prefixes, &cached_hashes, now));
1640   EXPECT_FALSE(database_->ContainsBrowseUrl(
1641       GURL("http://www.evil.com/phishing2.html"),
1642       &prefixes, &cached_hashes, now));
1643 }
1644
1645 // Test that an empty update doesn't actually update the database.
1646 // This isn't a functionality requirement, but it is a useful
1647 // optimization.
1648 TEST_F(SafeBrowsingDatabaseTest, EmptyUpdate) {
1649   SBChunkList chunks;
1650   SBChunk chunk;
1651
1652   base::FilePath filename = database_->BrowseDBFilename(database_filename_);
1653
1654   // Prime the database.
1655   std::vector<SBListChunkRanges> lists;
1656   EXPECT_TRUE(database_->UpdateStarted(&lists));
1657
1658   InsertAddChunkHostPrefixUrl(&chunk, 1, "www.evil.com/",
1659                               "www.evil.com/malware.html");
1660   chunks.clear();
1661   chunks.push_back(chunk);
1662   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
1663   database_->UpdateFinished(true);
1664
1665   // Get an older time to reset the lastmod time for detecting whether
1666   // the file has been updated.
1667   base::File::Info before_info, after_info;
1668   ASSERT_TRUE(base::GetFileInfo(filename, &before_info));
1669   const base::Time old_last_modified =
1670       before_info.last_modified - base::TimeDelta::FromSeconds(10);
1671
1672   // Inserting another chunk updates the database file.  The sleep is
1673   // needed because otherwise the entire test can finish w/in the
1674   // resolution of the lastmod time.
1675   ASSERT_TRUE(base::TouchFile(filename, old_last_modified, old_last_modified));
1676   ASSERT_TRUE(base::GetFileInfo(filename, &before_info));
1677   EXPECT_TRUE(database_->UpdateStarted(&lists));
1678   chunk.hosts.clear();
1679   InsertAddChunkHostPrefixUrl(&chunk, 2, "www.foo.com/",
1680                               "www.foo.com/malware.html");
1681   chunks.clear();
1682   chunks.push_back(chunk);
1683   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
1684   database_->UpdateFinished(true);
1685   ASSERT_TRUE(base::GetFileInfo(filename, &after_info));
1686   EXPECT_LT(before_info.last_modified, after_info.last_modified);
1687
1688   // Deleting a chunk updates the database file.
1689   ASSERT_TRUE(base::TouchFile(filename, old_last_modified, old_last_modified));
1690   ASSERT_TRUE(base::GetFileInfo(filename, &before_info));
1691   EXPECT_TRUE(database_->UpdateStarted(&lists));
1692   AddDelChunk(safe_browsing_util::kMalwareList, chunk.chunk_number);
1693   database_->UpdateFinished(true);
1694   ASSERT_TRUE(base::GetFileInfo(filename, &after_info));
1695   EXPECT_LT(before_info.last_modified, after_info.last_modified);
1696
1697   // Simply calling |UpdateStarted()| then |UpdateFinished()| does not
1698   // update the database file.
1699   ASSERT_TRUE(base::TouchFile(filename, old_last_modified, old_last_modified));
1700   ASSERT_TRUE(base::GetFileInfo(filename, &before_info));
1701   EXPECT_TRUE(database_->UpdateStarted(&lists));
1702   database_->UpdateFinished(true);
1703   ASSERT_TRUE(base::GetFileInfo(filename, &after_info));
1704   EXPECT_EQ(before_info.last_modified, after_info.last_modified);
1705 }
1706
1707 // Test that a filter file is written out during update and read back
1708 // in during setup.
1709 TEST_F(SafeBrowsingDatabaseTest, FilterFile) {
1710   // Create a database with trivial example data and write it out.
1711   {
1712     SBChunkList chunks;
1713     SBChunk chunk;
1714
1715     // Prime the database.
1716     std::vector<SBListChunkRanges> lists;
1717     EXPECT_TRUE(database_->UpdateStarted(&lists));
1718
1719     InsertAddChunkHostPrefixUrl(&chunk, 1, "www.evil.com/",
1720                                 "www.evil.com/malware.html");
1721     chunks.clear();
1722     chunks.push_back(chunk);
1723     database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
1724     database_->UpdateFinished(true);
1725   }
1726
1727   // Find the malware url in the database, don't find a good url.
1728   const Time now = Time::Now();
1729   std::vector<SBFullHashResult> cached_hashes;
1730   std::vector<SBPrefix> prefix_hits;
1731   EXPECT_TRUE(database_->ContainsBrowseUrl(
1732       GURL("http://www.evil.com/malware.html"),
1733       &prefix_hits, &cached_hashes, now));
1734   EXPECT_FALSE(database_->ContainsBrowseUrl(
1735       GURL("http://www.good.com/goodware.html"),
1736       &prefix_hits, &cached_hashes, now));
1737
1738   base::FilePath filter_file = database_->PrefixSetForFilename(
1739       database_->BrowseDBFilename(database_filename_));
1740
1741   // After re-creating the database, it should have a filter read from
1742   // a file, so it should find the same results.
1743   ASSERT_TRUE(base::PathExists(filter_file));
1744   database_.reset(new SafeBrowsingDatabaseNew);
1745   database_->Init(database_filename_);
1746   EXPECT_TRUE(database_->ContainsBrowseUrl(
1747       GURL("http://www.evil.com/malware.html"),
1748       &prefix_hits, &cached_hashes, now));
1749   EXPECT_FALSE(database_->ContainsBrowseUrl(
1750       GURL("http://www.good.com/goodware.html"),
1751       &prefix_hits, &cached_hashes, now));
1752
1753   // If there is no filter file, the database cannot find malware urls.
1754   base::DeleteFile(filter_file, false);
1755   ASSERT_FALSE(base::PathExists(filter_file));
1756   database_.reset(new SafeBrowsingDatabaseNew);
1757   database_->Init(database_filename_);
1758   EXPECT_FALSE(database_->ContainsBrowseUrl(
1759       GURL("http://www.evil.com/malware.html"),
1760       &prefix_hits, &cached_hashes, now));
1761   EXPECT_FALSE(database_->ContainsBrowseUrl(
1762       GURL("http://www.good.com/goodware.html"),
1763       &prefix_hits, &cached_hashes, now));
1764 }
1765
1766 TEST_F(SafeBrowsingDatabaseTest, MalwareIpBlacklist) {
1767   database_.reset();
1768   SafeBrowsingStoreFile* browse_store = new SafeBrowsingStoreFile();
1769   SafeBrowsingStoreFile* ip_blacklist_store = new SafeBrowsingStoreFile();
1770   database_.reset(new SafeBrowsingDatabaseNew(browse_store,
1771                                               NULL,
1772                                               NULL,
1773                                               NULL,
1774                                               NULL,
1775                                               NULL,
1776                                               ip_blacklist_store));
1777   database_->Init(database_filename_);
1778   std::vector<SBListChunkRanges> lists;
1779   EXPECT_TRUE(database_->UpdateStarted(&lists));
1780
1781   // IPv4 prefix match for ::ffff:192.168.1.0/120.
1782   SBChunkList chunks;
1783   SBChunk chunk;
1784   InsertAddChunkFullHash(&chunk, 1, "::ffff:192.168.1.0", 120);
1785   chunks.push_back(chunk);
1786   database_->InsertChunks(safe_browsing_util::kIPBlacklist, chunks);
1787
1788   // IPv4 exact match for ::ffff:192.1.1.1.
1789   chunks.clear();
1790   chunk.hosts.clear();
1791   InsertAddChunkFullHash(&chunk, 2, "::ffff:192.1.1.1", 128);
1792   chunks.push_back(chunk);
1793   database_->InsertChunks(safe_browsing_util::kIPBlacklist, chunks);
1794
1795   // IPv6 exact match for: fe80::31a:a0ff:fe10:786e/128.
1796   chunks.clear();
1797   chunk.hosts.clear();
1798   InsertAddChunkFullHash(&chunk, 3, "fe80::31a:a0ff:fe10:786e", 128);
1799   chunks.push_back(chunk);
1800   database_->InsertChunks(safe_browsing_util::kIPBlacklist, chunks);
1801
1802   // IPv6 prefix match for: 2620:0:1000:3103::/64.
1803   chunks.clear();
1804   chunk.hosts.clear();
1805   InsertAddChunkFullHash(&chunk, 4, "2620:0:1000:3103::", 64);
1806   chunks.push_back(chunk);
1807   database_->InsertChunks(safe_browsing_util::kIPBlacklist, chunks);
1808
1809   // IPv4 prefix match for ::ffff:192.1.122.0/119.
1810   chunks.clear();
1811   chunk.hosts.clear();
1812   InsertAddChunkFullHash(&chunk, 5, "::ffff:192.1.122.0", 119);
1813   chunks.push_back(chunk);
1814   database_->InsertChunks(safe_browsing_util::kIPBlacklist, chunks);
1815
1816   // IPv4 prefix match for ::ffff:192.1.128.0/113.
1817   chunks.clear();
1818   chunk.hosts.clear();
1819   InsertAddChunkFullHash(&chunk, 6, "::ffff:192.1.128.0", 113);
1820   chunks.push_back(chunk);
1821   database_->InsertChunks(safe_browsing_util::kIPBlacklist, chunks);
1822
1823   database_->UpdateFinished(true);
1824
1825   EXPECT_FALSE(database_->ContainsMalwareIP("192.168.0.255"));
1826   EXPECT_TRUE(database_->ContainsMalwareIP("192.168.1.0"));
1827   EXPECT_TRUE(database_->ContainsMalwareIP("192.168.1.255"));
1828   EXPECT_TRUE(database_->ContainsMalwareIP("192.168.1.10"));
1829   EXPECT_TRUE(database_->ContainsMalwareIP("::ffff:192.168.1.2"));
1830   EXPECT_FALSE(database_->ContainsMalwareIP("192.168.2.0"));
1831
1832   EXPECT_FALSE(database_->ContainsMalwareIP("192.1.1.0"));
1833   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.1.1"));
1834   EXPECT_FALSE(database_->ContainsMalwareIP("192.1.1.2"));
1835
1836   EXPECT_FALSE(database_->ContainsMalwareIP(
1837       "2620:0:1000:3102:ffff:ffff:ffff:ffff"));
1838   EXPECT_TRUE(database_->ContainsMalwareIP("2620:0:1000:3103::"));
1839   EXPECT_TRUE(database_->ContainsMalwareIP(
1840       "2620:0:1000:3103:ffff:ffff:ffff:ffff"));
1841   EXPECT_FALSE(database_->ContainsMalwareIP("2620:0:1000:3104::"));
1842
1843   EXPECT_FALSE(database_->ContainsMalwareIP("fe80::21a:a0ff:fe10:786d"));
1844   EXPECT_TRUE(database_->ContainsMalwareIP("fe80::31a:a0ff:fe10:786e"));
1845   EXPECT_FALSE(database_->ContainsMalwareIP("fe80::21a:a0ff:fe10:786f"));
1846
1847   EXPECT_FALSE(database_->ContainsMalwareIP("192.1.121.255"));
1848   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.122.0"));
1849   EXPECT_TRUE(database_->ContainsMalwareIP("::ffff:192.1.122.1"));
1850   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.122.255"));
1851   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.123.0"));
1852   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.123.255"));
1853   EXPECT_FALSE(database_->ContainsMalwareIP("192.1.124.0"));
1854
1855   EXPECT_FALSE(database_->ContainsMalwareIP("192.1.127.255"));
1856   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.128.0"));
1857   EXPECT_TRUE(database_->ContainsMalwareIP("::ffff:192.1.128.1"));
1858   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.128.255"));
1859   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.255.0"));
1860   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.255.255"));
1861   EXPECT_FALSE(database_->ContainsMalwareIP("192.2.0.0"));
1862 }