chrome/browser/safe_browsing/safe_browsing_database_unittest.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4 //
   5 // Unit tests for the SafeBrowsing storage system.
   6
   7 #include "chrome/browser/safe_browsing/safe_browsing_database.h"
   8
   9 #include "base/files/file_util.h"
  10 #include "base/files/scoped_temp_dir.h"
  11 #include "base/logging.h"
  12 #include "base/memory/scoped_vector.h"
  13 #include "base/message_loop/message_loop.h"
  14 #include "base/sha1.h"
  15 #include "base/strings/string_number_conversions.h"
  16 #include "base/strings/string_split.h"
  17 #include "base/test/test_simple_task_runner.h"
  18 #include "base/time/time.h"
  19 #include "chrome/browser/safe_browsing/chunk.pb.h"
  20 #include "chrome/browser/safe_browsing/safe_browsing_store_file.h"
  21 #include "crypto/sha2.h"
  22 #include "net/base/ip_address_number.h"
  23 #include "testing/gtest/include/gtest/gtest.h"
  24 #include "testing/platform_test.h"
  25 #include "url/gurl.h"
  26
  27 using base::Time;
  28 using base::TimeDelta;
  29
  30 namespace {
  31
  32 const TimeDelta kCacheLifetime = TimeDelta::FromMinutes(45);
  33
  34 SBPrefix SBPrefixForString(const std::string& str) {
  35   return SBFullHashForString(str).prefix;
  36 }
  37
  38 // Construct a full hash which has the given prefix, with the given
  39 // suffix data coming after the prefix.
  40 SBFullHash SBFullHashForPrefixAndSuffix(SBPrefix prefix,
  41                                         const base::StringPiece& suffix) {
  42   SBFullHash full_hash;
  43   memset(&full_hash, 0, sizeof(SBFullHash));
  44   full_hash.prefix = prefix;
  45   CHECK_LE(suffix.size() + sizeof(SBPrefix), sizeof(SBFullHash));
  46   memcpy(full_hash.full_hash + sizeof(SBPrefix), suffix.data(), suffix.size());
  47   return full_hash;
  48 }
  49
  50 std::string HashedIpPrefix(const std::string& ip_prefix, size_t prefix_size) {
  51   net::IPAddressNumber ip_number;
  52   EXPECT_TRUE(net::ParseIPLiteralToNumber(ip_prefix, &ip_number));
  53   EXPECT_EQ(net::kIPv6AddressSize, ip_number.size());
  54   const std::string hashed_ip_prefix = base::SHA1HashString(
  55       net::IPAddressToPackedString(ip_number));
  56   std::string hash(crypto::kSHA256Length, '\0');
  57   hash.replace(0, hashed_ip_prefix.size(), hashed_ip_prefix);
  58   hash[base::kSHA1Length] = static_cast<char>(prefix_size);
  59   return hash;
  60 }
  61
  62 // Helper to build a chunk.  Caller takes ownership.
  63 SBChunkData* BuildChunk(int chunk_number,
  64                         safe_browsing::ChunkData::ChunkType chunk_type,
  65                         safe_browsing::ChunkData::PrefixType prefix_type,
  66                         const void* data, size_t data_size,
  67                         const std::vector<int>& add_chunk_numbers) {
  68   scoped_ptr<safe_browsing::ChunkData> raw_data(new safe_browsing::ChunkData);
  69   raw_data->set_chunk_number(chunk_number);
  70   raw_data->set_chunk_type(chunk_type);
  71   raw_data->set_prefix_type(prefix_type);
  72   raw_data->set_hashes(data, data_size);
  73   raw_data->clear_add_numbers();
  74   for (size_t i = 0; i < add_chunk_numbers.size(); ++i) {
  75     raw_data->add_add_numbers(add_chunk_numbers[i]);
  76   }
  77
  78   return new SBChunkData(raw_data.release());
  79 }
  80
  81 // Create add chunk with a single prefix.
  82 SBChunkData* AddChunkPrefix(int chunk_number, SBPrefix prefix) {
  83   return BuildChunk(chunk_number, safe_browsing::ChunkData::ADD,
  84                     safe_browsing::ChunkData::PREFIX_4B,
  85                     &prefix, sizeof(prefix),
  86                     std::vector<int>());
  87 }
  88
  89 // Create add chunk with a single prefix generated from |value|.
  90 SBChunkData* AddChunkPrefixValue(int chunk_number,
  91                                  const std::string& value) {
  92   return AddChunkPrefix(chunk_number, SBPrefixForString(value));
  93 }
  94
  95 // Generate an add chunk with two prefixes.
  96 SBChunkData* AddChunkPrefix2Value(int chunk_number,
  97                                   const std::string& value1,
  98                                   const std::string& value2) {
  99   const SBPrefix prefixes[2] = {
 100     SBPrefixForString(value1),
 101     SBPrefixForString(value2),
 102   };
 103   return BuildChunk(chunk_number, safe_browsing::ChunkData::ADD,
 104                     safe_browsing::ChunkData::PREFIX_4B,
 105                     &prefixes[0], sizeof(prefixes),
 106                     std::vector<int>());
 107 }
 108
 109 // Generate an add chunk with four prefixes.
 110 SBChunkData* AddChunkPrefix4Value(int chunk_number,
 111                                   const std::string& value1,
 112                                   const std::string& value2,
 113                                   const std::string& value3,
 114                                   const std::string& value4) {
 115   const SBPrefix prefixes[4] = {
 116     SBPrefixForString(value1),
 117     SBPrefixForString(value2),
 118     SBPrefixForString(value3),
 119     SBPrefixForString(value4),
 120   };
 121   return BuildChunk(chunk_number, safe_browsing::ChunkData::ADD,
 122                     safe_browsing::ChunkData::PREFIX_4B,
 123                     &prefixes[0], sizeof(prefixes),
 124                     std::vector<int>());
 125 }
 126
 127 // Generate an add chunk with a full hash.
 128 SBChunkData* AddChunkFullHash(int chunk_number, SBFullHash full_hash) {
 129   return BuildChunk(chunk_number, safe_browsing::ChunkData::ADD,
 130                     safe_browsing::ChunkData::FULL_32B,
 131                     &full_hash, sizeof(full_hash),
 132                     std::vector<int>());
 133 }
 134
 135 // Generate an add chunk with a full hash generated from |value|.
 136 SBChunkData* AddChunkFullHashValue(int chunk_number,
 137                                    const std::string& value) {
 138   return AddChunkFullHash(chunk_number, SBFullHashForString(value));
 139 }
 140
 141 // Generate an add chunk with two full hashes.
 142 SBChunkData* AddChunkFullHash2Value(int chunk_number,
 143                                    const std::string& value1,
 144                                    const std::string& value2) {
 145   const SBFullHash full_hashes[2] = {
 146     SBFullHashForString(value1),
 147     SBFullHashForString(value2),
 148   };
 149   return BuildChunk(chunk_number, safe_browsing::ChunkData::ADD,
 150                     safe_browsing::ChunkData::FULL_32B,
 151                     &full_hashes[0], sizeof(full_hashes),
 152                     std::vector<int>());
 153 }
 154
 155 // Generate a sub chunk with a prefix generated from |value|.
 156 SBChunkData* SubChunkPrefixValue(int chunk_number,
 157                                  const std::string& value,
 158                                  int add_chunk_number) {
 159   const SBPrefix prefix = SBPrefixForString(value);
 160   return BuildChunk(chunk_number, safe_browsing::ChunkData::SUB,
 161                     safe_browsing::ChunkData::PREFIX_4B,
 162                     &prefix, sizeof(prefix),
 163                     std::vector<int>(1, add_chunk_number));
 164 }
 165
 166 // Generate a sub chunk with two prefixes.
 167 SBChunkData* SubChunkPrefix2Value(int chunk_number,
 168                                   const std::string& value1,
 169                                   int add_chunk_number1,
 170                                   const std::string& value2,
 171                                   int add_chunk_number2) {
 172   const SBPrefix prefixes[2] = {
 173     SBPrefixForString(value1),
 174     SBPrefixForString(value2),
 175   };
 176   std::vector<int> add_chunk_numbers;
 177   add_chunk_numbers.push_back(add_chunk_number1);
 178   add_chunk_numbers.push_back(add_chunk_number2);
 179   return BuildChunk(chunk_number, safe_browsing::ChunkData::SUB,
 180                     safe_browsing::ChunkData::PREFIX_4B,
 181                     &prefixes[0], sizeof(prefixes),
 182                     add_chunk_numbers);
 183 }
 184
 185 // Generate a sub chunk with a full hash.
 186 SBChunkData* SubChunkFullHash(int chunk_number,
 187                               SBFullHash full_hash,
 188                               int add_chunk_number) {
 189   return BuildChunk(chunk_number, safe_browsing::ChunkData::SUB,
 190                     safe_browsing::ChunkData::FULL_32B,
 191                     &full_hash, sizeof(full_hash),
 192                     std::vector<int>(1, add_chunk_number));
 193 }
 194
 195 // Generate a sub chunk with a full hash generated from |value|.
 196 SBChunkData* SubChunkFullHashValue(int chunk_number,
 197                                    const std::string& value,
 198                                    int add_chunk_number) {
 199   return SubChunkFullHash(chunk_number,
 200                           SBFullHashForString(value),
 201                           add_chunk_number);
 202 }
 203
 204 // Generate an add chunk with a single full hash for the ip blacklist.
 205 SBChunkData* AddChunkHashedIpValue(int chunk_number,
 206                                    const std::string& ip_str,
 207                                    size_t prefix_size) {
 208   const std::string full_hash_str = HashedIpPrefix(ip_str, prefix_size);
 209   EXPECT_EQ(sizeof(SBFullHash), full_hash_str.size());
 210   SBFullHash full_hash;
 211   std::memcpy(&(full_hash.full_hash), full_hash_str.data(), sizeof(SBFullHash));
 212   return BuildChunk(chunk_number, safe_browsing::ChunkData::ADD,
 213                     safe_browsing::ChunkData::FULL_32B,
 214                     &full_hash, sizeof(full_hash),
 215                     std::vector<int>());
 216 }
 217
 218 // Prevent DCHECK from killing tests.
 219 // TODO(shess): Pawel disputes the use of this, so the test which uses
 220 // it is DISABLED.  http://crbug.com/56448
 221 class ScopedLogMessageIgnorer {
 222  public:
 223   ScopedLogMessageIgnorer() {
 224     logging::SetLogMessageHandler(&LogMessageIgnorer);
 225   }
 226   ~ScopedLogMessageIgnorer() {
 227     // TODO(shess): Would be better to verify whether anyone else
 228     // changed it, and then restore it to the previous value.
 229     logging::SetLogMessageHandler(NULL);
 230   }
 231
 232  private:
 233   static bool LogMessageIgnorer(int severity, const char* file, int line,
 234       size_t message_start, const std::string& str) {
 235     // Intercept FATAL, strip the stack backtrace, and log it without
 236     // the crash part.
 237     if (severity == logging::LOG_FATAL) {
 238       size_t newline = str.find('\n');
 239       if (newline != std::string::npos) {
 240         const std::string msg = str.substr(0, newline + 1);
 241         fprintf(stderr, "%s", msg.c_str());
 242         fflush(stderr);
 243       }
 244       return true;
 245     }
 246
 247     return false;
 248   }
 249 };
 250
 251 }  // namespace
 252
 253 class SafeBrowsingDatabaseTest : public PlatformTest {
 254  public:
 255   SafeBrowsingDatabaseTest() : task_runner_(new base::TestSimpleTaskRunner) {}
 256
 257   void SetUp() override {
 258     PlatformTest::SetUp();
 259
 260     // Setup a database in a temporary directory.
 261     ASSERT_TRUE(temp_dir_.CreateUniqueTempDir());
 262     database_filename_ =
 263         temp_dir_.path().AppendASCII("SafeBrowsingTestDatabase");
 264
 265     ResetAndReloadFullDatabase();
 266   }
 267
 268   void TearDown() override {
 269     database_.reset();
 270
 271     PlatformTest::TearDown();
 272   }
 273
 274   // Reloads the |database_| in a new SafeBrowsingDatabaseNew object with all
 275   // stores enabled.
 276   void ResetAndReloadFullDatabase() {
 277     SafeBrowsingStoreFile* browse_store =
 278         new SafeBrowsingStoreFile(task_runner_);
 279     SafeBrowsingStoreFile* download_store =
 280         new SafeBrowsingStoreFile(task_runner_);
 281     SafeBrowsingStoreFile* csd_whitelist_store =
 282         new SafeBrowsingStoreFile(task_runner_);
 283     SafeBrowsingStoreFile* download_whitelist_store =
 284         new SafeBrowsingStoreFile(task_runner_);
 285     SafeBrowsingStoreFile* inclusion_whitelist_store =
 286         new SafeBrowsingStoreFile(task_runner_);
 287     SafeBrowsingStoreFile* extension_blacklist_store =
 288         new SafeBrowsingStoreFile(task_runner_);
 289     SafeBrowsingStoreFile* ip_blacklist_store =
 290         new SafeBrowsingStoreFile(task_runner_);
 291     SafeBrowsingStoreFile* unwanted_software_store =
 292         new SafeBrowsingStoreFile(task_runner_);
 293     database_.reset(new SafeBrowsingDatabaseNew(
 294         task_runner_, browse_store, download_store, csd_whitelist_store,
 295         download_whitelist_store, inclusion_whitelist_store,
 296         extension_blacklist_store, ip_blacklist_store,
 297         unwanted_software_store));
 298     database_->Init(database_filename_);
 299   }
 300
 301   bool ContainsDownloadUrl(const std::vector<GURL>& urls,
 302                            std::vector<SBPrefix>* prefix_hits) {
 303     std::vector<SBPrefix> prefixes;
 304     SafeBrowsingDatabase::GetDownloadUrlPrefixes(urls, &prefixes);
 305     return database_->ContainsDownloadUrlPrefixes(prefixes, prefix_hits);
 306   }
 307
 308   void GetListsInfo(std::vector<SBListChunkRanges>* lists) {
 309     lists->clear();
 310     ASSERT_TRUE(database_->UpdateStarted(lists));
 311     database_->UpdateFinished(true);
 312   }
 313
 314   // Helper function to do an AddDel or SubDel command.
 315   void DelChunk(const std::string& list,
 316                 int chunk_id,
 317                 bool is_sub_del) {
 318     std::vector<SBChunkDelete> deletes;
 319     SBChunkDelete chunk_delete;
 320     chunk_delete.list_name = list;
 321     chunk_delete.is_sub_del = is_sub_del;
 322     chunk_delete.chunk_del.push_back(ChunkRange(chunk_id));
 323     deletes.push_back(chunk_delete);
 324     database_->DeleteChunks(deletes);
 325   }
 326
 327   void AddDelChunk(const std::string& list, int chunk_id) {
 328     DelChunk(list, chunk_id, false);
 329   }
 330
 331   void SubDelChunk(const std::string& list, int chunk_id) {
 332     DelChunk(list, chunk_id, true);
 333   }
 334
 335   // Utility function for setting up the database for the caching test.
 336   void PopulateDatabaseForCacheTest();
 337
 338   scoped_refptr<base::TestSimpleTaskRunner> task_runner_;
 339   scoped_ptr<SafeBrowsingDatabaseNew> database_;
 340   base::FilePath database_filename_;
 341   base::ScopedTempDir temp_dir_;
 342 };
 343
 344 // Tests retrieving list name information.
 345 TEST_F(SafeBrowsingDatabaseTest, BrowseListsInfo) {
 346   std::vector<SBListChunkRanges> lists;
 347   ScopedVector<SBChunkData> chunks;
 348
 349   chunks.push_back(AddChunkPrefixValue(1, "www.evil.com/malware.html"));
 350   chunks.push_back(AddChunkPrefixValue(2, "www.foo.com/malware.html"));
 351   chunks.push_back(AddChunkPrefixValue(3, "www.whatever.com/malware.html"));
 352
 353   ASSERT_TRUE(database_->UpdateStarted(&lists));
 354   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
 355   database_->UpdateFinished(true);
 356
 357   GetListsInfo(&lists);
 358   ASSERT_LE(1U, lists.size());
 359   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
 360   EXPECT_EQ("1-3", lists[0].adds);
 361   EXPECT_TRUE(lists[0].subs.empty());
 362
 363   // Insert a malware sub chunk.
 364   chunks.clear();
 365   chunks.push_back(SubChunkPrefixValue(7, "www.subbed.com/noteveil1.html", 19));
 366
 367   ASSERT_TRUE(database_->UpdateStarted(&lists));
 368   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
 369   database_->UpdateFinished(true);
 370
 371   GetListsInfo(&lists);
 372   ASSERT_LE(1U, lists.size());
 373   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
 374   EXPECT_EQ("1-3", lists[0].adds);
 375   EXPECT_EQ("7", lists[0].subs);
 376   if (lists.size() == 2) {
 377     // Old style database won't have the second entry since it creates the lists
 378     // when it receives an update containing that list. The filter-based
 379     // database has these values hard coded.
 380     EXPECT_EQ(safe_browsing_util::kPhishingList, lists[1].name);
 381     EXPECT_TRUE(lists[1].adds.empty());
 382     EXPECT_TRUE(lists[1].subs.empty());
 383   }
 384
 385   // Add phishing chunks.
 386   chunks.clear();
 387   chunks.push_back(AddChunkPrefixValue(47, "www.evil.com/phishing.html"));
 388   chunks.push_back(
 389       SubChunkPrefixValue(200, "www.phishy.com/notevil1.html", 1999));
 390   chunks.push_back(
 391       SubChunkPrefixValue(201, "www.phishy2.com/notevil1.html", 1999));
 392
 393   ASSERT_TRUE(database_->UpdateStarted(&lists));
 394   database_->InsertChunks(safe_browsing_util::kPhishingList, chunks.get());
 395   database_->UpdateFinished(true);
 396
 397   GetListsInfo(&lists);
 398   ASSERT_LE(2U, lists.size());
 399   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
 400   EXPECT_EQ("1-3", lists[0].adds);
 401   EXPECT_EQ("7", lists[0].subs);
 402   EXPECT_EQ(safe_browsing_util::kPhishingList, lists[1].name);
 403   EXPECT_EQ("47", lists[1].adds);
 404   EXPECT_EQ("200-201", lists[1].subs);
 405 }
 406
 407 TEST_F(SafeBrowsingDatabaseTest, ListNames) {
 408   ScopedVector<SBChunkData> chunks;
 409
 410   std::vector<SBListChunkRanges> lists;
 411   ASSERT_TRUE(database_->UpdateStarted(&lists));
 412
 413   // Insert malware, phish, binurl and bindownload add chunks.
 414   chunks.push_back(AddChunkPrefixValue(1, "www.evil.com/malware.html"));
 415   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
 416
 417   chunks.clear();
 418   chunks.push_back(AddChunkPrefixValue(2, "www.foo.com/malware.html"));
 419   database_->InsertChunks(safe_browsing_util::kPhishingList, chunks.get());
 420
 421   chunks.clear();
 422   chunks.push_back(AddChunkPrefixValue(3, "www.whatever.com/download.html"));
 423   database_->InsertChunks(safe_browsing_util::kBinUrlList, chunks.get());
 424
 425   chunks.clear();
 426   chunks.push_back(AddChunkFullHashValue(5, "www.forwhitelist.com/a.html"));
 427   database_->InsertChunks(safe_browsing_util::kCsdWhiteList, chunks.get());
 428
 429   chunks.clear();
 430   chunks.push_back(AddChunkFullHashValue(6, "www.download.com/"));
 431   database_->InsertChunks(safe_browsing_util::kDownloadWhiteList, chunks.get());
 432
 433   chunks.clear();
 434   chunks.push_back(AddChunkFullHashValue(7, "www.inclusion.com/"));
 435   database_->InsertChunks(safe_browsing_util::kInclusionWhitelist,
 436                           chunks.get());
 437
 438   chunks.clear();
 439   chunks.push_back(AddChunkFullHashValue(8,
 440                                          "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
 441                                          "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"));
 442   database_->InsertChunks(safe_browsing_util::kExtensionBlacklist,
 443                           chunks.get());
 444
 445   chunks.clear();
 446   chunks.push_back(AddChunkHashedIpValue(10, "::ffff:192.168.1.0", 120));
 447   database_->InsertChunks(safe_browsing_util::kIPBlacklist, chunks.get());
 448
 449   chunks.clear();
 450   chunks.push_back(AddChunkPrefixValue(11, "www.unwanted.com/software.html"));
 451   database_->InsertChunks(safe_browsing_util::kUnwantedUrlList, chunks.get());
 452
 453   database_->UpdateFinished(true);
 454
 455   GetListsInfo(&lists);
 456   ASSERT_EQ(9U, lists.size());
 457   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
 458   EXPECT_EQ("1", lists[0].adds);
 459   EXPECT_TRUE(lists[0].subs.empty());
 460   EXPECT_EQ(safe_browsing_util::kPhishingList, lists[1].name);
 461   EXPECT_EQ("2", lists[1].adds);
 462   EXPECT_TRUE(lists[1].subs.empty());
 463   EXPECT_EQ(safe_browsing_util::kBinUrlList, lists[2].name);
 464   EXPECT_EQ("3", lists[2].adds);
 465   EXPECT_TRUE(lists[2].subs.empty());
 466   EXPECT_EQ(safe_browsing_util::kCsdWhiteList, lists[3].name);
 467   EXPECT_EQ("5", lists[3].adds);
 468   EXPECT_TRUE(lists[3].subs.empty());
 469   EXPECT_EQ(safe_browsing_util::kDownloadWhiteList, lists[4].name);
 470   EXPECT_EQ("6", lists[4].adds);
 471   EXPECT_TRUE(lists[4].subs.empty());
 472   EXPECT_EQ(safe_browsing_util::kInclusionWhitelist, lists[5].name);
 473   EXPECT_EQ("7", lists[5].adds);
 474   EXPECT_TRUE(lists[5].subs.empty());
 475   EXPECT_EQ(safe_browsing_util::kExtensionBlacklist, lists[6].name);
 476   EXPECT_EQ("8", lists[6].adds);
 477   EXPECT_TRUE(lists[6].subs.empty());
 478   EXPECT_EQ(safe_browsing_util::kIPBlacklist, lists[7].name);
 479   EXPECT_EQ("10", lists[7].adds);
 480   EXPECT_TRUE(lists[7].subs.empty());
 481   EXPECT_EQ(safe_browsing_util::kUnwantedUrlList, lists[8].name);
 482   EXPECT_EQ("11", lists[8].adds);
 483   EXPECT_TRUE(lists[8].subs.empty());
 484
 485   database_.reset();
 486 }
 487
 488 // Checks database reading and writing for browse and unwanted PrefixSets.
 489 TEST_F(SafeBrowsingDatabaseTest, BrowseAndUnwantedDatabasesAndPrefixSets) {
 490   struct TestCase {
 491     using TestListContainsBadUrl = bool (SafeBrowsingDatabase::*)(
 492         const GURL& url,
 493         std::vector<SBPrefix>* prefix_hits,
 494         std::vector<SBFullHashResult>* cache_hits);
 495
 496     const char* test_list_name;
 497     size_t expected_list_index;
 498     TestListContainsBadUrl test_list_contains_bad_url;
 499   } const kTestCases[] {
 500     { safe_browsing_util::kMalwareList, 0U,
 501       &SafeBrowsingDatabase::ContainsBrowseUrl },
 502     { safe_browsing_util::kPhishingList, 1U,
 503       &SafeBrowsingDatabase::ContainsBrowseUrl },
 504     { safe_browsing_util::kUnwantedUrlList, 8U,
 505       &SafeBrowsingDatabase::ContainsUnwantedSoftwareUrl },
 506   };
 507
 508   for (const auto& test_case : kTestCases) {
 509     SCOPED_TRACE(std::string("Tested list at fault => ") +
 510                  test_case.test_list_name);
 511
 512     std::vector<SBListChunkRanges> lists;
 513     ScopedVector<SBChunkData> chunks;
 514
 515     chunks.push_back(AddChunkPrefix2Value(1,
 516                                           "www.evil.com/phishing.html",
 517                                           "www.evil.com/malware.html"));
 518     chunks.push_back(AddChunkPrefix4Value(2,
 519                                           "www.evil.com/notevil1.html",
 520                                           "www.evil.com/notevil2.html",
 521                                           "www.good.com/good1.html",
 522                                           "www.good.com/good2.html"));
 523     chunks.push_back(AddChunkPrefixValue(3, "192.168.0.1/malware.html"));
 524     chunks.push_back(AddChunkFullHashValue(7, "www.evil.com/evil.html"));
 525
 526     ASSERT_TRUE(database_->UpdateStarted(&lists));
 527     database_->InsertChunks(test_case.test_list_name, chunks.get());
 528     database_->UpdateFinished(true);
 529
 530     // Make sure they were added correctly.
 531     GetListsInfo(&lists);
 532
 533     ASSERT_LE(1U, lists.size());
 534     EXPECT_EQ(test_case.test_list_name,
 535               lists[test_case.expected_list_index].name);
 536     EXPECT_EQ("1-3,7", lists[test_case.expected_list_index].adds);
 537     EXPECT_TRUE(lists[test_case.expected_list_index].subs.empty());
 538
 539     std::vector<SBPrefix> prefix_hits;
 540     std::vector<SBFullHashResult> cache_hits;
 541     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 542         GURL("http://www.evil.com/phishing.html"), &prefix_hits, &cache_hits));
 543     ASSERT_EQ(1U, prefix_hits.size());
 544     EXPECT_EQ(SBPrefixForString("www.evil.com/phishing.html"), prefix_hits[0]);
 545     EXPECT_TRUE(cache_hits.empty());
 546
 547     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 548         GURL("http://www.evil.com/malware.html"), &prefix_hits, &cache_hits));
 549
 550     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 551         GURL("http://www.evil.com/notevil1.html"), &prefix_hits, &cache_hits));
 552
 553     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 554         GURL("http://www.evil.com/notevil2.html"), &prefix_hits, &cache_hits));
 555
 556     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 557         GURL("http://www.good.com/good1.html"), &prefix_hits, &cache_hits));
 558
 559     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 560         GURL("http://www.good.com/good2.html"), &prefix_hits, &cache_hits));
 561
 562     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 563         GURL("http://192.168.0.1/malware.html"), &prefix_hits, &cache_hits));
 564
 565     EXPECT_FALSE((database_.get()->*test_case.test_list_contains_bad_url)(
 566         GURL("http://www.evil.com/"), &prefix_hits, &cache_hits));
 567     EXPECT_TRUE(prefix_hits.empty());
 568     EXPECT_TRUE(cache_hits.empty());
 569
 570     EXPECT_FALSE((database_.get()->*test_case.test_list_contains_bad_url)(
 571         GURL("http://www.evil.com/robots.txt"), &prefix_hits, &cache_hits));
 572
 573     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 574         GURL("http://www.evil.com/evil.html"), &prefix_hits, &cache_hits));
 575     ASSERT_EQ(1U, prefix_hits.size());
 576     EXPECT_EQ(SBPrefixForString("www.evil.com/evil.html"), prefix_hits[0]);
 577
 578     // Attempt to re-add the first chunk (should be a no-op).
 579     // see bug: http://code.google.com/p/chromium/issues/detail?id=4522
 580     chunks.clear();
 581     chunks.push_back(AddChunkPrefix2Value(1,
 582                                           "www.evil.com/phishing.html",
 583                                           "www.evil.com/malware.html"));
 584     ASSERT_TRUE(database_->UpdateStarted(&lists));
 585     database_->InsertChunks(test_case.test_list_name, chunks.get());
 586     database_->UpdateFinished(true);
 587
 588     GetListsInfo(&lists);
 589     ASSERT_LE(1U, lists.size());
 590     EXPECT_EQ(test_case.test_list_name,
 591               lists[test_case.expected_list_index].name);
 592     EXPECT_EQ("1-3,7", lists[test_case.expected_list_index].adds);
 593     EXPECT_TRUE(lists[test_case.expected_list_index].subs.empty());
 594
 595     // Test removing a single prefix from the add chunk.
 596     chunks.clear();
 597     chunks.push_back(SubChunkPrefixValue(4, "www.evil.com/notevil1.html", 2));
 598     ASSERT_TRUE(database_->UpdateStarted(&lists));
 599     database_->InsertChunks(test_case.test_list_name, chunks.get());
 600     database_->UpdateFinished(true);
 601
 602     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 603         GURL("http://www.evil.com/phishing.html"), &prefix_hits, &cache_hits));
 604     ASSERT_EQ(1U, prefix_hits.size());
 605     EXPECT_EQ(SBPrefixForString("www.evil.com/phishing.html"), prefix_hits[0]);
 606     EXPECT_TRUE(cache_hits.empty());
 607
 608     EXPECT_FALSE((database_.get()->*test_case.test_list_contains_bad_url)(
 609         GURL("http://www.evil.com/notevil1.html"), &prefix_hits, &cache_hits));
 610     EXPECT_TRUE(prefix_hits.empty());
 611     EXPECT_TRUE(cache_hits.empty());
 612
 613     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 614         GURL("http://www.evil.com/notevil2.html"), &prefix_hits, &cache_hits));
 615
 616     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 617         GURL("http://www.good.com/good1.html"), &prefix_hits, &cache_hits));
 618
 619     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 620         GURL("http://www.good.com/good2.html"), &prefix_hits, &cache_hits));
 621
 622     GetListsInfo(&lists);
 623     ASSERT_LE(1U, lists.size());
 624     EXPECT_EQ(test_case.test_list_name,
 625               lists[test_case.expected_list_index].name);
 626     EXPECT_EQ("1-3,7", lists[test_case.expected_list_index].adds);
 627     EXPECT_EQ("4", lists[test_case.expected_list_index].subs);
 628
 629     // Test the same sub chunk again.  This should be a no-op.
 630     // see bug: http://code.google.com/p/chromium/issues/detail?id=4522
 631     chunks.clear();
 632     chunks.push_back(SubChunkPrefixValue(4, "www.evil.com/notevil1.html", 2));
 633
 634     ASSERT_TRUE(database_->UpdateStarted(&lists));
 635     database_->InsertChunks(test_case.test_list_name, chunks.get());
 636     database_->UpdateFinished(true);
 637
 638     GetListsInfo(&lists);
 639     ASSERT_LE(1U, lists.size());
 640     EXPECT_EQ(test_case.test_list_name,
 641               lists[test_case.expected_list_index].name);
 642     EXPECT_EQ("1-3,7", lists[test_case.expected_list_index].adds);
 643     EXPECT_EQ("4", lists[test_case.expected_list_index].subs);
 644
 645     // Test removing all the prefixes from an add chunk.
 646     ASSERT_TRUE(database_->UpdateStarted(&lists));
 647     AddDelChunk(test_case.test_list_name, 2);
 648     database_->UpdateFinished(true);
 649
 650     EXPECT_FALSE((database_.get()->*test_case.test_list_contains_bad_url)(
 651         GURL("http://www.evil.com/notevil2.html"), &prefix_hits, &cache_hits));
 652
 653     EXPECT_FALSE((database_.get()->*test_case.test_list_contains_bad_url)(
 654         GURL("http://www.good.com/good1.html"), &prefix_hits, &cache_hits));
 655
 656     EXPECT_FALSE((database_.get()->*test_case.test_list_contains_bad_url)(
 657         GURL("http://www.good.com/good2.html"), &prefix_hits, &cache_hits));
 658
 659     GetListsInfo(&lists);
 660     ASSERT_LE(1U, lists.size());
 661     EXPECT_EQ(test_case.test_list_name,
 662               lists[test_case.expected_list_index].name);
 663     EXPECT_EQ("1,3,7", lists[test_case.expected_list_index].adds);
 664     EXPECT_EQ("4", lists[test_case.expected_list_index].subs);
 665
 666     // The adddel command exposed a bug in the transaction code where any
 667     // transaction after it would fail.  Add a dummy entry and remove it to
 668     // make sure the transcation works fine.
 669     chunks.clear();
 670     chunks.push_back(AddChunkPrefixValue(44, "www.redherring.com/index.html"));
 671     ASSERT_TRUE(database_->UpdateStarted(&lists));
 672     database_->InsertChunks(test_case.test_list_name, chunks.get());
 673
 674     // Now remove the dummy entry.  If there are any problems with the
 675     // transactions, asserts will fire.
 676     AddDelChunk(test_case.test_list_name, 44);
 677
 678     // Test the subdel command.
 679     SubDelChunk(test_case.test_list_name, 4);
 680     database_->UpdateFinished(true);
 681
 682     GetListsInfo(&lists);
 683     ASSERT_LE(1U, lists.size());
 684     EXPECT_EQ(test_case.test_list_name,
 685               lists[test_case.expected_list_index].name);
 686     EXPECT_EQ("1,3,7", lists[test_case.expected_list_index].adds);
 687     EXPECT_TRUE(lists[test_case.expected_list_index].subs.empty());
 688
 689     // Test a sub command coming in before the add.
 690     chunks.clear();
 691     chunks.push_back(SubChunkPrefix2Value(5,
 692                                           "www.notevilanymore.com/index.html",
 693                                           10,
 694                                           "www.notevilanymore.com/good.html",
 695                                           10));
 696     ASSERT_TRUE(database_->UpdateStarted(&lists));
 697     database_->InsertChunks(test_case.test_list_name, chunks.get());
 698     database_->UpdateFinished(true);
 699
 700     EXPECT_FALSE((database_.get()->*test_case.test_list_contains_bad_url)(
 701         GURL("http://www.notevilanymore.com/index.html"),
 702         &prefix_hits,
 703         &cache_hits));
 704
 705     // Now insert the tardy add chunk and we don't expect them to appear
 706     // in database because of the previous sub chunk.
 707     chunks.clear();
 708     chunks.push_back(AddChunkPrefix2Value(10,
 709                                           "www.notevilanymore.com/index.html",
 710                                           "www.notevilanymore.com/good.html"));
 711     ASSERT_TRUE(database_->UpdateStarted(&lists));
 712     database_->InsertChunks(test_case.test_list_name, chunks.get());
 713     database_->UpdateFinished(true);
 714
 715     EXPECT_FALSE((database_.get()->*test_case.test_list_contains_bad_url)(
 716         GURL("http://www.notevilanymore.com/index.html"),
 717         &prefix_hits,
 718         &cache_hits));
 719
 720     EXPECT_FALSE((database_.get()->*test_case.test_list_contains_bad_url)(
 721         GURL("http://www.notevilanymore.com/good.html"),
 722         &prefix_hits,
 723         &cache_hits));
 724
 725     // Reset and reload the database.  The database will rely on the prefix set.
 726     ResetAndReloadFullDatabase();
 727
 728     // Check that a prefix still hits.
 729     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 730         GURL("http://www.evil.com/phishing.html"), &prefix_hits, &cache_hits));
 731     ASSERT_EQ(1U, prefix_hits.size());
 732     EXPECT_EQ(SBPrefixForString("www.evil.com/phishing.html"), prefix_hits[0]);
 733
 734     // Also check that it's not just always returning true in this case.
 735     EXPECT_FALSE((database_.get()->*test_case.test_list_contains_bad_url)(
 736         GURL("http://www.evil.com/"), &prefix_hits, &cache_hits));
 737
 738     // Check that the full hash is still present.
 739     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 740         GURL("http://www.evil.com/evil.html"), &prefix_hits, &cache_hits));
 741     ASSERT_EQ(1U, prefix_hits.size());
 742     EXPECT_EQ(SBPrefixForString("www.evil.com/evil.html"), prefix_hits[0]);
 743   }
 744 }
 745
 746 // Test adding zero length chunks to the database.
 747 TEST_F(SafeBrowsingDatabaseTest, ZeroSizeChunk) {
 748   std::vector<SBListChunkRanges> lists;
 749   ScopedVector<SBChunkData> chunks;
 750
 751   // Populate with a couple of normal chunks.
 752   chunks.push_back(AddChunkPrefix2Value(1,
 753                                         "www.test.com/test1.html",
 754                                         "www.test.com/test2.html"));
 755   chunks.push_back(AddChunkPrefix2Value(10,
 756                                         "www.random.com/random1.html",
 757                                         "www.random.com/random2.html"));
 758
 759   ASSERT_TRUE(database_->UpdateStarted(&lists));
 760   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
 761   database_->UpdateFinished(true);
 762
 763   // Add an empty ADD and SUB chunk.
 764   GetListsInfo(&lists);
 765   ASSERT_LE(1U, lists.size());
 766   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
 767   EXPECT_EQ("1,10", lists[0].adds);
 768   EXPECT_TRUE(lists[0].subs.empty());
 769
 770   chunks.clear();
 771   chunks.push_back(BuildChunk(19, safe_browsing::ChunkData::ADD,
 772                               safe_browsing::ChunkData::PREFIX_4B,
 773                               NULL, 0, std::vector<int>()));
 774   chunks.push_back(BuildChunk(7, safe_browsing::ChunkData::SUB,
 775                               safe_browsing::ChunkData::PREFIX_4B,
 776                               NULL, 0, std::vector<int>()));
 777
 778   ASSERT_TRUE(database_->UpdateStarted(&lists));
 779   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
 780   database_->UpdateFinished(true);
 781
 782   GetListsInfo(&lists);
 783   ASSERT_LE(1U, lists.size());
 784   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
 785   EXPECT_EQ("1,10,19", lists[0].adds);
 786   EXPECT_EQ("7", lists[0].subs);
 787
 788   // Add an empty chunk along with a couple that contain data. This should
 789   // result in the chunk range being reduced in size.
 790   chunks.clear();
 791   chunks.push_back(AddChunkPrefixValue(20, "www.notempty.com/full1.html"));
 792   chunks.push_back(BuildChunk(21, safe_browsing::ChunkData::ADD,
 793                               safe_browsing::ChunkData::PREFIX_4B,
 794                               NULL, 0, std::vector<int>()));
 795   chunks.push_back(AddChunkPrefixValue(22, "www.notempty.com/full2.html"));
 796
 797   ASSERT_TRUE(database_->UpdateStarted(&lists));
 798   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
 799   database_->UpdateFinished(true);
 800
 801   std::vector<SBPrefix> prefix_hits;
 802   std::vector<SBFullHashResult> cache_hits;
 803   EXPECT_TRUE(database_->ContainsBrowseUrl(
 804       GURL("http://www.notempty.com/full1.html"), &prefix_hits, &cache_hits));
 805   EXPECT_TRUE(database_->ContainsBrowseUrl(
 806       GURL("http://www.notempty.com/full2.html"), &prefix_hits, &cache_hits));
 807
 808   GetListsInfo(&lists);
 809   ASSERT_LE(1U, lists.size());
 810   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
 811   EXPECT_EQ("1,10,19-22", lists[0].adds);
 812   EXPECT_EQ("7", lists[0].subs);
 813
 814   // Handle AddDel and SubDel commands for empty chunks.
 815   ASSERT_TRUE(database_->UpdateStarted(&lists));
 816   AddDelChunk(safe_browsing_util::kMalwareList, 21);
 817   database_->UpdateFinished(true);
 818
 819   GetListsInfo(&lists);
 820   ASSERT_LE(1U, lists.size());
 821   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
 822   EXPECT_EQ("1,10,19-20,22", lists[0].adds);
 823   EXPECT_EQ("7", lists[0].subs);
 824
 825   ASSERT_TRUE(database_->UpdateStarted(&lists));
 826   SubDelChunk(safe_browsing_util::kMalwareList, 7);
 827   database_->UpdateFinished(true);
 828
 829   GetListsInfo(&lists);
 830   ASSERT_LE(1U, lists.size());
 831   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
 832   EXPECT_EQ("1,10,19-20,22", lists[0].adds);
 833   EXPECT_TRUE(lists[0].subs.empty());
 834 }
 835
 836 // Utility function for setting up the database for the caching test.
 837 void SafeBrowsingDatabaseTest::PopulateDatabaseForCacheTest() {
 838   // Add a couple prefixes.
 839   ScopedVector<SBChunkData> chunks;
 840   chunks.push_back(AddChunkPrefix2Value(1,
 841                                         "www.evil.com/phishing.html",
 842                                         "www.evil.com/malware.html"));
 843
 844   std::vector<SBListChunkRanges> lists;
 845   ASSERT_TRUE(database_->UpdateStarted(&lists));
 846   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
 847   database_->UpdateFinished(true);
 848
 849   // Cache should be cleared after updating.
 850   EXPECT_TRUE(
 851       database_->GetUnsynchronizedPrefixGetHashCacheForTesting()->empty());
 852
 853   SBFullHashResult full_hash;
 854   full_hash.list_id = safe_browsing_util::MALWARE;
 855
 856   std::vector<SBFullHashResult> results;
 857   std::vector<SBPrefix> prefixes;
 858
 859   // Add a fullhash result for each prefix.
 860   full_hash.hash = SBFullHashForString("www.evil.com/phishing.html");
 861   results.push_back(full_hash);
 862   prefixes.push_back(full_hash.hash.prefix);
 863
 864   full_hash.hash = SBFullHashForString("www.evil.com/malware.html");
 865   results.push_back(full_hash);
 866   prefixes.push_back(full_hash.hash.prefix);
 867
 868   database_->CacheHashResults(prefixes, results, kCacheLifetime);
 869 }
 870
 871 TEST_F(SafeBrowsingDatabaseTest, HashCaching) {
 872   PopulateDatabaseForCacheTest();
 873
 874   // We should have both full hashes in the cache.
 875   EXPECT_EQ(2U,
 876             database_->GetUnsynchronizedPrefixGetHashCacheForTesting()->size());
 877
 878   // Test the cache lookup for the first prefix.
 879   std::vector<SBPrefix> prefix_hits;
 880   std::vector<SBFullHashResult> cache_hits;
 881   EXPECT_TRUE(database_->ContainsBrowseUrl(
 882       GURL("http://www.evil.com/phishing.html"), &prefix_hits, &cache_hits));
 883   EXPECT_TRUE(prefix_hits.empty());
 884   ASSERT_EQ(1U, cache_hits.size());
 885   EXPECT_TRUE(SBFullHashEqual(
 886       cache_hits[0].hash, SBFullHashForString("www.evil.com/phishing.html")));
 887
 888   prefix_hits.clear();
 889   cache_hits.clear();
 890
 891   // Test the cache lookup for the second prefix.
 892   EXPECT_TRUE(database_->ContainsBrowseUrl(
 893       GURL("http://www.evil.com/malware.html"), &prefix_hits, &cache_hits));
 894   EXPECT_TRUE(prefix_hits.empty());
 895   ASSERT_EQ(1U, cache_hits.size());
 896   EXPECT_TRUE(SBFullHashEqual(
 897       cache_hits[0].hash, SBFullHashForString("www.evil.com/malware.html")));
 898
 899   prefix_hits.clear();
 900   cache_hits.clear();
 901
 902   // Test removing a prefix via a sub chunk.
 903   ScopedVector<SBChunkData> chunks;
 904   chunks.push_back(SubChunkPrefixValue(2, "www.evil.com/phishing.html", 1));
 905
 906   std::vector<SBListChunkRanges> lists;
 907   ASSERT_TRUE(database_->UpdateStarted(&lists));
 908   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
 909   database_->UpdateFinished(true);
 910
 911   // This prefix should still be there, but cached fullhash should be gone.
 912   EXPECT_TRUE(database_->ContainsBrowseUrl(
 913       GURL("http://www.evil.com/malware.html"), &prefix_hits, &cache_hits));
 914   ASSERT_EQ(1U, prefix_hits.size());
 915   EXPECT_EQ(SBPrefixForString("www.evil.com/malware.html"), prefix_hits[0]);
 916   EXPECT_TRUE(cache_hits.empty());
 917   prefix_hits.clear();
 918   cache_hits.clear();
 919
 920   // This prefix should be gone.
 921   EXPECT_FALSE(database_->ContainsBrowseUrl(
 922       GURL("http://www.evil.com/phishing.html"), &prefix_hits, &cache_hits));
 923   prefix_hits.clear();
 924   cache_hits.clear();
 925
 926   // Test that an AddDel for the original chunk removes the last cached entry.
 927   ASSERT_TRUE(database_->UpdateStarted(&lists));
 928   AddDelChunk(safe_browsing_util::kMalwareList, 1);
 929   database_->UpdateFinished(true);
 930   EXPECT_FALSE(database_->ContainsBrowseUrl(
 931       GURL("http://www.evil.com/malware.html"), &prefix_hits, &cache_hits));
 932   EXPECT_TRUE(
 933       database_->GetUnsynchronizedPrefixGetHashCacheForTesting()->empty());
 934   prefix_hits.clear();
 935   cache_hits.clear();
 936
 937   // Test that the cache won't return expired values. First we have to adjust
 938   // the cached entries' received time to make them older, since the database
 939   // cache insert uses Time::Now(). First, store some entries.
 940   PopulateDatabaseForCacheTest();
 941
 942   SafeBrowsingDatabaseNew::PrefixGetHashCache* hash_cache =
 943       database_->GetUnsynchronizedPrefixGetHashCacheForTesting();
 944   EXPECT_EQ(2U, hash_cache->size());
 945
 946   // Now adjust one of the entries times to be in the past.
 947   const SBPrefix key = SBPrefixForString("www.evil.com/malware.html");
 948   SafeBrowsingDatabaseNew::PrefixGetHashCache::iterator iter =
 949       hash_cache->find(key);
 950   ASSERT_TRUE(iter != hash_cache->end());
 951   iter->second.expire_after = Time::Now() - TimeDelta::FromMinutes(1);
 952
 953   EXPECT_TRUE(database_->ContainsBrowseUrl(
 954       GURL("http://www.evil.com/malware.html"), &prefix_hits, &cache_hits));
 955   EXPECT_EQ(1U, prefix_hits.size());
 956   EXPECT_TRUE(cache_hits.empty());
 957   // Expired entry should have been removed from cache.
 958   EXPECT_EQ(1U, hash_cache->size());
 959
 960   // This entry should still exist.
 961   EXPECT_TRUE(database_->ContainsBrowseUrl(
 962       GURL("http://www.evil.com/phishing.html"), &prefix_hits, &cache_hits));
 963   EXPECT_TRUE(prefix_hits.empty());
 964   EXPECT_EQ(1U, cache_hits.size());
 965
 966   // Testing prefix miss caching. First, we clear out the existing database,
 967   // Since PopulateDatabaseForCacheTest() doesn't handle adding duplicate
 968   // chunks.
 969   ASSERT_TRUE(database_->UpdateStarted(&lists));
 970   AddDelChunk(safe_browsing_util::kMalwareList, 1);
 971   database_->UpdateFinished(true);
 972
 973   // Cache should be cleared after updating.
 974   EXPECT_TRUE(hash_cache->empty());
 975
 976   std::vector<SBPrefix> prefix_misses;
 977   std::vector<SBFullHashResult> empty_full_hash;
 978   prefix_misses.push_back(SBPrefixForString("http://www.bad.com/malware.html"));
 979   prefix_misses.push_back(
 980       SBPrefixForString("http://www.bad.com/phishing.html"));
 981   database_->CacheHashResults(prefix_misses, empty_full_hash, kCacheLifetime);
 982
 983   // Prefixes with no full results are misses.
 984   EXPECT_EQ(hash_cache->size(), prefix_misses.size());
 985   ASSERT_TRUE(
 986       hash_cache->count(SBPrefixForString("http://www.bad.com/malware.html")));
 987   EXPECT_TRUE(
 988       hash_cache->find(SBPrefixForString("http://www.bad.com/malware.html"))
 989           ->second.full_hashes.empty());
 990   ASSERT_TRUE(
 991       hash_cache->count(SBPrefixForString("http://www.bad.com/phishing.html")));
 992   EXPECT_TRUE(
 993       hash_cache->find(SBPrefixForString("http://www.bad.com/phishing.html"))
 994           ->second.full_hashes.empty());
 995
 996   // Update the database.
 997   PopulateDatabaseForCacheTest();
 998
 999   // Cache a GetHash miss for a particular prefix, and even though the prefix is
1000   // in the database, it is flagged as a miss so looking up the associated URL
1001   // will not succeed.
1002   prefix_hits.clear();
1003   cache_hits.clear();
1004   prefix_misses.clear();
1005   empty_full_hash.clear();
1006   prefix_misses.push_back(SBPrefixForString("www.evil.com/phishing.html"));
1007   database_->CacheHashResults(prefix_misses, empty_full_hash, kCacheLifetime);
1008   EXPECT_FALSE(database_->ContainsBrowseUrl(
1009       GURL("http://www.evil.com/phishing.html"), &prefix_hits, &cache_hits));
1010   prefix_hits.clear();
1011   cache_hits.clear();
1012
1013   // Test receiving a full add chunk.
1014   chunks.clear();
1015   chunks.push_back(AddChunkFullHash2Value(20,
1016                                           "www.fullevil.com/bad1.html",
1017                                           "www.fullevil.com/bad2.html"));
1018   ASSERT_TRUE(database_->UpdateStarted(&lists));
1019   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1020   database_->UpdateFinished(true);
1021
1022   EXPECT_TRUE(database_->ContainsBrowseUrl(
1023       GURL("http://www.fullevil.com/bad1.html"), &prefix_hits, &cache_hits));
1024   ASSERT_EQ(1U, prefix_hits.size());
1025   EXPECT_EQ(SBPrefixForString("www.fullevil.com/bad1.html"), prefix_hits[0]);
1026   EXPECT_TRUE(cache_hits.empty());
1027   prefix_hits.clear();
1028   cache_hits.clear();
1029
1030   EXPECT_TRUE(database_->ContainsBrowseUrl(
1031       GURL("http://www.fullevil.com/bad2.html"), &prefix_hits, &cache_hits));
1032   ASSERT_EQ(1U, prefix_hits.size());
1033   EXPECT_EQ(SBPrefixForString("www.fullevil.com/bad2.html"), prefix_hits[0]);
1034   EXPECT_TRUE(cache_hits.empty());
1035   prefix_hits.clear();
1036   cache_hits.clear();
1037
1038   // Test receiving a full sub chunk, which will remove one of the full adds.
1039   chunks.clear();
1040   chunks.push_back(SubChunkFullHashValue(200,
1041                                          "www.fullevil.com/bad1.html",
1042                                          20));
1043   ASSERT_TRUE(database_->UpdateStarted(&lists));
1044   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1045   database_->UpdateFinished(true);
1046
1047   EXPECT_FALSE(database_->ContainsBrowseUrl(
1048       GURL("http://www.fullevil.com/bad1.html"), &prefix_hits, &cache_hits));
1049
1050   // There should be one remaining full add.
1051   EXPECT_TRUE(database_->ContainsBrowseUrl(
1052       GURL("http://www.fullevil.com/bad2.html"), &prefix_hits, &cache_hits));
1053   ASSERT_EQ(1U, prefix_hits.size());
1054   EXPECT_EQ(SBPrefixForString("www.fullevil.com/bad2.html"), prefix_hits[0]);
1055   EXPECT_TRUE(cache_hits.empty());
1056   prefix_hits.clear();
1057   cache_hits.clear();
1058
1059   // Now test an AddDel for the remaining full add.
1060   ASSERT_TRUE(database_->UpdateStarted(&lists));
1061   AddDelChunk(safe_browsing_util::kMalwareList, 20);
1062   database_->UpdateFinished(true);
1063
1064   EXPECT_FALSE(database_->ContainsBrowseUrl(
1065       GURL("http://www.fullevil.com/bad1.html"), &prefix_hits, &cache_hits));
1066   EXPECT_FALSE(database_->ContainsBrowseUrl(
1067       GURL("http://www.fullevil.com/bad2.html"), &prefix_hits, &cache_hits));
1068
1069   // Add a fullhash which has a prefix collision for a known url.
1070   static const char kExampleFine[] = "www.example.com/fine.html";
1071   static const char kExampleCollision[] =
1072       "www.example.com/3123364814/malware.htm";
1073   ASSERT_EQ(SBPrefixForString(kExampleFine),
1074             SBPrefixForString(kExampleCollision));
1075   ASSERT_TRUE(database_->UpdateStarted(&lists));
1076   {
1077     ScopedVector<SBChunkData> chunks;
1078     chunks.push_back(AddChunkPrefixValue(21, kExampleCollision));
1079     database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1080   }
1081   database_->UpdateFinished(true);
1082
1083   // Expect a prefix hit due to the collision between |kExampleFine| and
1084   // |kExampleCollision|.
1085   EXPECT_TRUE(database_->ContainsBrowseUrl(
1086       GURL(std::string("http://") + kExampleFine), &prefix_hits, &cache_hits));
1087   ASSERT_EQ(1U, prefix_hits.size());
1088   EXPECT_EQ(SBPrefixForString(kExampleFine), prefix_hits[0]);
1089   EXPECT_TRUE(cache_hits.empty());
1090
1091   // Cache gethash response for |kExampleCollision|.
1092   {
1093     SBFullHashResult result;
1094     result.hash = SBFullHashForString(kExampleCollision);
1095     result.list_id = safe_browsing_util::MALWARE;
1096     database_->CacheHashResults(std::vector<SBPrefix>(1, result.hash.prefix),
1097                                 std::vector<SBFullHashResult>(1, result),
1098                                 kCacheLifetime);
1099   }
1100
1101   // The cached response means the collision no longer causes a hit.
1102   EXPECT_FALSE(database_->ContainsBrowseUrl(
1103       GURL(std::string("http://") + kExampleFine), &prefix_hits, &cache_hits));
1104 }
1105
1106 // Test that corrupt databases are appropriately handled, even if the
1107 // corruption is detected in the midst of the update.
1108 // TODO(shess): Disabled until ScopedLogMessageIgnorer resolved.
1109 // http://crbug.com/56448
1110 TEST_F(SafeBrowsingDatabaseTest, DISABLED_FileCorruptionHandling) {
1111   // Re-create the database in a captive message loop so that we can
1112   // influence task-posting.  Database specifically needs to the
1113   // file-backed.
1114   database_.reset();
1115   base::MessageLoop loop;
1116   SafeBrowsingStoreFile* store = new SafeBrowsingStoreFile(task_runner_);
1117   database_.reset(new SafeBrowsingDatabaseNew(
1118       task_runner_, store, NULL, NULL, NULL, NULL, NULL, NULL, NULL));
1119   database_->Init(database_filename_);
1120
1121   // This will cause an empty database to be created.
1122   std::vector<SBListChunkRanges> lists;
1123   ASSERT_TRUE(database_->UpdateStarted(&lists));
1124   database_->UpdateFinished(true);
1125
1126   // Create a sub chunk to insert.
1127   ScopedVector<SBChunkData> chunks;
1128   chunks.push_back(SubChunkPrefixValue(7,
1129                                        "www.subbed.com/notevil1.html",
1130                                        19));
1131
1132   // Corrupt the file by corrupting the checksum, which is not checked
1133   // until the entire table is read in |UpdateFinished()|.
1134   FILE* fp = base::OpenFile(database_filename_, "r+");
1135   ASSERT_TRUE(fp);
1136   ASSERT_NE(-1, fseek(fp, -8, SEEK_END));
1137   for (size_t i = 0; i < 8; ++i) {
1138     fputc('!', fp);
1139   }
1140   fclose(fp);
1141
1142   {
1143     // The following code will cause DCHECKs, so suppress the crashes.
1144     ScopedLogMessageIgnorer ignorer;
1145
1146     // Start an update.  The insert will fail due to corruption.
1147     ASSERT_TRUE(database_->UpdateStarted(&lists));
1148     database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1149     database_->UpdateFinished(true);
1150
1151     // Database file still exists until the corruption handler has run.
1152     EXPECT_TRUE(base::PathExists(database_filename_));
1153
1154     // Flush through the corruption-handler task.
1155     DVLOG(1) << "Expect failed check on: SafeBrowsing database reset";
1156     base::MessageLoop::current()->RunUntilIdle();
1157   }
1158
1159   // Database file should not exist.
1160   EXPECT_FALSE(base::PathExists(database_filename_));
1161
1162   // Run the update again successfully.
1163   ASSERT_TRUE(database_->UpdateStarted(&lists));
1164   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1165   database_->UpdateFinished(true);
1166   EXPECT_TRUE(base::PathExists(database_filename_));
1167
1168   database_.reset();
1169 }
1170
1171 // Checks database reading and writing.
1172 TEST_F(SafeBrowsingDatabaseTest, ContainsDownloadUrlPrefixes) {
1173   const char kEvil1Url1[] = "www.evil1.com/download1/";
1174   const char kEvil1Url2[] = "www.evil1.com/download2.html";
1175
1176   // Add a simple chunk with one hostkey for download url list.
1177   ScopedVector<SBChunkData> chunks;
1178   chunks.push_back(AddChunkPrefix2Value(1, kEvil1Url1, kEvil1Url2));
1179
1180   std::vector<SBListChunkRanges> lists;
1181   ASSERT_TRUE(database_->UpdateStarted(&lists));
1182   database_->InsertChunks(safe_browsing_util::kBinUrlList, chunks.get());
1183   database_->UpdateFinished(true);
1184
1185   std::vector<SBPrefix> prefix_hits;
1186   std::vector<GURL> urls(1);
1187
1188   urls[0] = GURL(std::string("http://") + kEvil1Url1);
1189   EXPECT_TRUE(ContainsDownloadUrl(urls, &prefix_hits));
1190   ASSERT_EQ(1U, prefix_hits.size());
1191   EXPECT_EQ(SBPrefixForString(kEvil1Url1), prefix_hits[0]);
1192
1193   urls[0] = GURL(std::string("http://") + kEvil1Url2);
1194   EXPECT_TRUE(ContainsDownloadUrl(urls, &prefix_hits));
1195   ASSERT_EQ(1U, prefix_hits.size());
1196   EXPECT_EQ(SBPrefixForString(kEvil1Url2), prefix_hits[0]);
1197
1198   urls[0] = GURL(std::string("https://") + kEvil1Url2);
1199   EXPECT_TRUE(ContainsDownloadUrl(urls, &prefix_hits));
1200   ASSERT_EQ(1U, prefix_hits.size());
1201   EXPECT_EQ(SBPrefixForString(kEvil1Url2), prefix_hits[0]);
1202
1203   urls[0] = GURL(std::string("ftp://") + kEvil1Url2);
1204   EXPECT_TRUE(ContainsDownloadUrl(urls, &prefix_hits));
1205   ASSERT_EQ(1U, prefix_hits.size());
1206   EXPECT_EQ(SBPrefixForString(kEvil1Url2), prefix_hits[0]);
1207
1208   urls[0] = GURL("http://www.randomevil.com");
1209   EXPECT_FALSE(ContainsDownloadUrl(urls, &prefix_hits));
1210
1211   // Should match with query args stripped.
1212   urls[0] = GURL(std::string("http://") + kEvil1Url2 + "?blah");
1213   EXPECT_TRUE(ContainsDownloadUrl(urls, &prefix_hits));
1214   ASSERT_EQ(1U, prefix_hits.size());
1215   EXPECT_EQ(SBPrefixForString(kEvil1Url2), prefix_hits[0]);
1216
1217   // Should match with extra path stuff and query args stripped.
1218   urls[0] = GURL(std::string("http://") + kEvil1Url1 + "foo/bar?blah");
1219   EXPECT_TRUE(ContainsDownloadUrl(urls, &prefix_hits));
1220   ASSERT_EQ(1U, prefix_hits.size());
1221   EXPECT_EQ(SBPrefixForString(kEvil1Url1), prefix_hits[0]);
1222
1223   // First hit in redirect chain is malware.
1224   urls.clear();
1225   urls.push_back(GURL(std::string("http://") + kEvil1Url1));
1226   urls.push_back(GURL("http://www.randomevil.com"));
1227   EXPECT_TRUE(ContainsDownloadUrl(urls, &prefix_hits));
1228   ASSERT_EQ(1U, prefix_hits.size());
1229   EXPECT_EQ(SBPrefixForString(kEvil1Url1), prefix_hits[0]);
1230
1231   // Middle hit in redirect chain is malware.
1232   urls.clear();
1233   urls.push_back(GURL("http://www.randomevil.com"));
1234   urls.push_back(GURL(std::string("http://") + kEvil1Url1));
1235   urls.push_back(GURL("http://www.randomevil2.com"));
1236   EXPECT_TRUE(ContainsDownloadUrl(urls, &prefix_hits));
1237   ASSERT_EQ(1U, prefix_hits.size());
1238   EXPECT_EQ(SBPrefixForString(kEvil1Url1), prefix_hits[0]);
1239
1240   // Final hit in redirect chain is malware.
1241   urls.clear();
1242   urls.push_back(GURL("http://www.randomevil.com"));
1243   urls.push_back(GURL(std::string("http://") + kEvil1Url1));
1244   EXPECT_TRUE(ContainsDownloadUrl(urls, &prefix_hits));
1245   ASSERT_EQ(1U, prefix_hits.size());
1246   EXPECT_EQ(SBPrefixForString(kEvil1Url1), prefix_hits[0]);
1247
1248   // Multiple hits in redirect chain are in malware list.
1249   urls.clear();
1250   urls.push_back(GURL(std::string("http://") + kEvil1Url1));
1251   urls.push_back(GURL(std::string("https://") + kEvil1Url2));
1252   EXPECT_TRUE(ContainsDownloadUrl(urls, &prefix_hits));
1253   ASSERT_EQ(2U, prefix_hits.size());
1254   EXPECT_EQ(SBPrefixForString(kEvil1Url1), prefix_hits[0]);
1255   EXPECT_EQ(SBPrefixForString(kEvil1Url2), prefix_hits[1]);
1256   database_.reset();
1257 }
1258
1259 // Checks that the whitelists are handled properly.
1260 TEST_F(SafeBrowsingDatabaseTest, Whitelists) {
1261   struct TestCase {
1262     using TestListContainsWhitelistedUrl =
1263         bool (SafeBrowsingDatabase::*)(const GURL& url);
1264     using TestListContainsWhitelistedString =
1265         bool (SafeBrowsingDatabase::*)(const std::string& str);
1266
1267     // Returns true if strings should be tested in this test case (i.e.
1268     // |test_list_contains_whitelisted_string| is not null).
1269     bool TestStrings() const {
1270       return test_list_contains_whitelisted_string != nullptr;
1271     }
1272
1273     const char* test_list_name;
1274     TestListContainsWhitelistedUrl test_list_contains_whitelisted_url;
1275     // Optional test case field, if set the tested whitelist will also be tested
1276     // for strings.
1277     TestListContainsWhitelistedString test_list_contains_whitelisted_string;
1278   } const kTestCases[]{
1279       {safe_browsing_util::kCsdWhiteList,
1280        &SafeBrowsingDatabase::ContainsCsdWhitelistedUrl,
1281        nullptr},
1282       {safe_browsing_util::kDownloadWhiteList,
1283        &SafeBrowsingDatabase::ContainsDownloadWhitelistedUrl,
1284        &SafeBrowsingDatabase::ContainsDownloadWhitelistedString},
1285       {safe_browsing_util::kInclusionWhitelist,
1286        &SafeBrowsingDatabase::ContainsInclusionWhitelistedUrl,
1287        nullptr},
1288   };
1289
1290   // If the whitelist is disabled everything should match the whitelist.
1291   database_.reset(new SafeBrowsingDatabaseNew(
1292       task_runner_, new SafeBrowsingStoreFile(task_runner_), NULL, NULL, NULL,
1293       NULL, NULL, NULL, NULL));
1294   database_->Init(database_filename_);
1295   for (const auto& test_case : kTestCases) {
1296     SCOPED_TRACE(std::string("Tested list at fault => ") +
1297                  test_case.test_list_name);
1298
1299     EXPECT_TRUE(
1300         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1301             GURL(std::string("http://www.phishing.com/"))));
1302     if (test_case.TestStrings()) {
1303       EXPECT_TRUE(
1304           (database_.get()->*test_case.test_list_contains_whitelisted_string)(
1305               "asdf"));
1306     }
1307   }
1308   ResetAndReloadFullDatabase();
1309
1310   // Now test every whitelist one-by-one; intentionally not resetting the
1311   // database in-between to further stress potential inter-dependencies.
1312   for (const auto& test_case : kTestCases) {
1313     SCOPED_TRACE(std::string("Tested list at fault => ") +
1314                  test_case.test_list_name);
1315
1316     const char kGood1Host[] = "www.good1.com/";
1317     const char kGood1Url1[] = "www.good1.com/a/b.html";
1318     const char kGood1Url2[] = "www.good1.com/b/";
1319
1320     const char kGood2Url1[] = "www.good2.com/c";  // Should match '/c/bla'.
1321
1322     // good3.com/a/b/c/d/e/f/g/ should match because it's a whitelist.
1323     const char kGood3Url1[] = "good3.com/";
1324
1325     const char kGoodString[] = "good_string";
1326
1327     // Nothing should be whitelisted before the database receives the chunks.
1328     EXPECT_FALSE(
1329         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1330             GURL(std::string("http://") + kGood1Host)));
1331     EXPECT_FALSE(
1332         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1333             GURL(std::string("http://") + kGood1Url1)));
1334     EXPECT_FALSE(
1335         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1336             GURL(std::string("http://") + kGood1Url2)));
1337     EXPECT_FALSE(
1338         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1339             GURL(std::string("http://") + kGood2Url1)));
1340     EXPECT_FALSE(
1341         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1342             GURL(std::string("http://") + kGood3Url1)));
1343     if (test_case.TestStrings()) {
1344       EXPECT_FALSE(
1345           (database_.get()->*test_case.test_list_contains_whitelisted_string)(
1346               kGoodString));
1347     }
1348
1349     ScopedVector<SBChunkData> chunks;
1350
1351     // Add a few test chunks to the whitelist under test.
1352     chunks.push_back(AddChunkFullHash2Value(1, kGood1Url1, kGood1Url2));
1353     chunks.push_back(AddChunkFullHashValue(2, kGood2Url1));
1354     if (test_case.TestStrings())
1355       chunks.push_back(AddChunkFullHashValue(3, kGoodString));
1356     chunks.push_back(AddChunkFullHashValue(4, kGood3Url1));
1357
1358     std::vector<SBListChunkRanges> lists;
1359     ASSERT_TRUE(database_->UpdateStarted(&lists));
1360     database_->InsertChunks(test_case.test_list_name, chunks.get());
1361     database_->UpdateFinished(true);
1362
1363     EXPECT_FALSE(
1364         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1365             GURL(std::string("http://") + kGood1Host)));
1366
1367     EXPECT_TRUE(
1368         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1369             GURL(std::string("http://") + kGood1Url1)));
1370     EXPECT_TRUE(
1371         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1372             GURL(std::string("http://") + kGood1Url1 + "?a=b")));
1373
1374     EXPECT_TRUE(
1375         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1376             GURL(std::string("http://") + kGood1Url2)));
1377     EXPECT_TRUE(
1378         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1379             GURL(std::string("http://") + kGood1Url2 + "/c.html")));
1380
1381     EXPECT_TRUE(
1382         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1383             GURL(std::string("https://") + kGood1Url2 + "/c.html")));
1384
1385     EXPECT_TRUE(
1386         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1387             GURL(std::string("http://") + kGood2Url1 + "/c")));
1388     EXPECT_TRUE(
1389         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1390             GURL(std::string("http://") + kGood2Url1 + "/c?bla")));
1391     EXPECT_TRUE(
1392         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1393             GURL(std::string("http://") + kGood2Url1 + "/c/bla")));
1394
1395     EXPECT_FALSE(
1396         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1397             GURL(std::string("http://www.google.com/"))));
1398
1399     EXPECT_TRUE(
1400         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1401             GURL(std::string("http://") + kGood3Url1 + "a/b/c/d/e/f/g/")));
1402     EXPECT_TRUE(
1403         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1404             GURL(std::string("http://a.b.") + kGood3Url1)));
1405
1406     if (test_case.TestStrings()) {
1407       EXPECT_FALSE(
1408           (database_.get()->*test_case.test_list_contains_whitelisted_string)(
1409               "asdf"));
1410       EXPECT_TRUE(
1411           (database_.get()->*test_case.test_list_contains_whitelisted_string)(
1412               kGoodString));
1413     }
1414
1415     EXPECT_FALSE(
1416         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1417             GURL(std::string("http://www.google.com/"))));
1418
1419     // The malware kill switch is for the CSD whitelist only.
1420     if (test_case.test_list_name == safe_browsing_util::kCsdWhiteList) {
1421       // The CSD whitelist killswitch is not present.
1422       EXPECT_FALSE(database_->IsCsdWhitelistKillSwitchOn());
1423
1424       // Test only add the malware IP killswitch
1425       chunks.clear();
1426       chunks.push_back(AddChunkFullHashValue(
1427           15, "sb-ssl.google.com/safebrowsing/csd/killswitch_malware"));
1428
1429       ASSERT_TRUE(database_->UpdateStarted(&lists));
1430       database_->InsertChunks(safe_browsing_util::kCsdWhiteList, chunks.get());
1431       database_->UpdateFinished(true);
1432
1433       EXPECT_TRUE(database_->IsMalwareIPMatchKillSwitchOn());
1434       // The CSD whitelist killswitch is not present.
1435       EXPECT_FALSE(database_->IsCsdWhitelistKillSwitchOn());
1436     }
1437
1438     // Test that the generic whitelist kill-switch works as intended.
1439     chunks.clear();
1440     lists.clear();
1441     chunks.push_back(AddChunkFullHashValue(
1442         5, "sb-ssl.google.com/safebrowsing/csd/killswitch"));
1443
1444     ASSERT_TRUE(database_->UpdateStarted(&lists));
1445     database_->InsertChunks(test_case.test_list_name, chunks.get());
1446     database_->UpdateFinished(true);
1447
1448     // Test CSD whitelist specific methods.
1449     if (test_case.test_list_name == safe_browsing_util::kCsdWhiteList) {
1450       // The CSD whitelist killswitch is present.
1451       EXPECT_TRUE(database_->IsCsdWhitelistKillSwitchOn());
1452       EXPECT_TRUE(database_->IsMalwareIPMatchKillSwitchOn());
1453     }
1454
1455     EXPECT_TRUE(
1456         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1457             GURL(std::string("https://") + kGood1Url2 + "/c.html")));
1458     EXPECT_TRUE(
1459         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1460             GURL(std::string("http://www.google.com/"))));
1461     EXPECT_TRUE(
1462         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1463             GURL(std::string("http://www.phishing_url.com/"))));
1464
1465     if (test_case.TestStrings()) {
1466       EXPECT_TRUE(
1467           (database_.get()->*test_case.test_list_contains_whitelisted_string)(
1468               "asdf"));
1469       EXPECT_TRUE(
1470           (database_.get()->*test_case.test_list_contains_whitelisted_string)(
1471               kGoodString));
1472     }
1473
1474     // Remove the kill-switch and verify that we can recover.
1475     chunks.clear();
1476     lists.clear();
1477     chunks.push_back(SubChunkFullHashValue(
1478         1, "sb-ssl.google.com/safebrowsing/csd/killswitch", 5));
1479     if (test_case.test_list_name == safe_browsing_util::kCsdWhiteList) {
1480       chunks.push_back(SubChunkFullHashValue(
1481           10, "sb-ssl.google.com/safebrowsing/csd/killswitch_malware", 15));
1482     }
1483
1484     ASSERT_TRUE(database_->UpdateStarted(&lists));
1485     database_->InsertChunks(test_case.test_list_name, chunks.get());
1486     database_->UpdateFinished(true);
1487
1488     if (test_case.test_list_name == safe_browsing_util::kCsdWhiteList) {
1489       EXPECT_FALSE(database_->IsMalwareIPMatchKillSwitchOn());
1490       EXPECT_FALSE(database_->IsCsdWhitelistKillSwitchOn());
1491     }
1492     EXPECT_TRUE(
1493         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1494             GURL(std::string("https://") + kGood1Url2 + "/c.html")));
1495     EXPECT_TRUE(
1496         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1497             GURL(std::string("https://") + kGood2Url1 + "/c/bla")));
1498     EXPECT_TRUE(
1499         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1500             GURL(std::string("https://") + kGood3Url1)));
1501     EXPECT_FALSE(
1502         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1503             GURL(std::string("http://www.google.com/"))));
1504     EXPECT_FALSE(
1505         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1506             GURL(std::string("http://www.phishing_url.com/"))));
1507     if (test_case.TestStrings()) {
1508       EXPECT_TRUE(
1509           (database_.get()->*test_case.test_list_contains_whitelisted_string)(
1510               kGoodString));
1511       EXPECT_FALSE(
1512           (database_.get()->*test_case.test_list_contains_whitelisted_string)(
1513               "asdf"));
1514     }
1515   }
1516 }
1517
1518 // Test to make sure we could insert chunk list that
1519 // contains entries for the same host.
1520 TEST_F(SafeBrowsingDatabaseTest, SameHostEntriesOkay) {
1521   ScopedVector<SBChunkData> chunks;
1522
1523   // Add a malware add chunk with two entries of the same host.
1524   chunks.push_back(AddChunkPrefix2Value(1,
1525                                         "www.evil.com/malware1.html",
1526                                         "www.evil.com/malware2.html"));
1527
1528   // Insert the testing chunks into database.
1529   std::vector<SBListChunkRanges> lists;
1530   ASSERT_TRUE(database_->UpdateStarted(&lists));
1531   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1532   database_->UpdateFinished(true);
1533
1534   GetListsInfo(&lists);
1535   ASSERT_LE(1U, lists.size());
1536   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
1537   EXPECT_EQ("1", lists[0].adds);
1538   EXPECT_TRUE(lists[0].subs.empty());
1539
1540   // Add a phishing add chunk with two entries of the same host.
1541   chunks.clear();
1542   chunks.push_back(AddChunkPrefix2Value(47,
1543                                         "www.evil.com/phishing1.html",
1544                                         "www.evil.com/phishing2.html"));
1545
1546   ASSERT_TRUE(database_->UpdateStarted(&lists));
1547   database_->InsertChunks(safe_browsing_util::kPhishingList, chunks.get());
1548   database_->UpdateFinished(true);
1549
1550   GetListsInfo(&lists);
1551   ASSERT_LE(2U, lists.size());
1552   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
1553   EXPECT_EQ("1", lists[0].adds);
1554   EXPECT_TRUE(lists[0].subs.empty());
1555   EXPECT_EQ(safe_browsing_util::kPhishingList, lists[1].name);
1556   EXPECT_EQ("47", lists[1].adds);
1557   EXPECT_TRUE(lists[1].subs.empty());
1558
1559   std::vector<SBPrefix> prefix_hits;
1560   std::vector<SBFullHashResult> cache_hits;
1561
1562   EXPECT_TRUE(database_->ContainsBrowseUrl(
1563       GURL("http://www.evil.com/malware1.html"), &prefix_hits, &cache_hits));
1564   EXPECT_TRUE(database_->ContainsBrowseUrl(
1565       GURL("http://www.evil.com/malware2.html"), &prefix_hits, &cache_hits));
1566   EXPECT_TRUE(database_->ContainsBrowseUrl(
1567       GURL("http://www.evil.com/phishing1.html"), &prefix_hits, &cache_hits));
1568   EXPECT_TRUE(database_->ContainsBrowseUrl(
1569       GURL("http://www.evil.com/phishing2.html"), &prefix_hits, &cache_hits));
1570
1571   // Test removing a single prefix from the add chunk.
1572   // Remove the prefix that added first.
1573   chunks.clear();
1574   chunks.push_back(SubChunkPrefixValue(4, "www.evil.com/malware1.html", 1));
1575   ASSERT_TRUE(database_->UpdateStarted(&lists));
1576   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1577   database_->UpdateFinished(true);
1578
1579   // Remove the prefix that added last.
1580   chunks.clear();
1581   chunks.push_back(SubChunkPrefixValue(5, "www.evil.com/phishing2.html", 47));
1582   ASSERT_TRUE(database_->UpdateStarted(&lists));
1583   database_->InsertChunks(safe_browsing_util::kPhishingList, chunks.get());
1584   database_->UpdateFinished(true);
1585
1586   // Verify that the database contains urls expected.
1587   EXPECT_FALSE(database_->ContainsBrowseUrl(
1588       GURL("http://www.evil.com/malware1.html"), &prefix_hits, &cache_hits));
1589   EXPECT_TRUE(database_->ContainsBrowseUrl(
1590       GURL("http://www.evil.com/malware2.html"), &prefix_hits, &cache_hits));
1591   EXPECT_TRUE(database_->ContainsBrowseUrl(
1592       GURL("http://www.evil.com/phishing1.html"), &prefix_hits, &cache_hits));
1593   EXPECT_FALSE(database_->ContainsBrowseUrl(
1594       GURL("http://www.evil.com/phishing2.html"), &prefix_hits, &cache_hits));
1595 }
1596
1597 // Test that an empty update doesn't actually update the database.
1598 // This isn't a functionality requirement, but it is a useful
1599 // optimization.
1600 TEST_F(SafeBrowsingDatabaseTest, EmptyUpdate) {
1601   ScopedVector<SBChunkData> chunks;
1602
1603   base::FilePath filename = database_->BrowseDBFilename(database_filename_);
1604
1605   // Prime the database.
1606   std::vector<SBListChunkRanges> lists;
1607   ASSERT_TRUE(database_->UpdateStarted(&lists));
1608   chunks.push_back(AddChunkPrefixValue(1, "www.evil.com/malware.html"));
1609   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1610   database_->UpdateFinished(true);
1611
1612   // Get an older time to reset the lastmod time for detecting whether
1613   // the file has been updated.
1614   base::File::Info before_info, after_info;
1615   ASSERT_TRUE(base::GetFileInfo(filename, &before_info));
1616   const Time old_last_modified =
1617       before_info.last_modified - TimeDelta::FromSeconds(10);
1618
1619   // Inserting another chunk updates the database file.  The sleep is
1620   // needed because otherwise the entire test can finish w/in the
1621   // resolution of the lastmod time.
1622   ASSERT_TRUE(base::TouchFile(filename, old_last_modified, old_last_modified));
1623   ASSERT_TRUE(base::GetFileInfo(filename, &before_info));
1624   ASSERT_TRUE(database_->UpdateStarted(&lists));
1625   chunks.push_back(AddChunkPrefixValue(2, "www.foo.com/malware.html"));
1626   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1627   database_->UpdateFinished(true);
1628   ASSERT_TRUE(base::GetFileInfo(filename, &after_info));
1629   EXPECT_LT(before_info.last_modified, after_info.last_modified);
1630
1631   // Deleting a chunk updates the database file.
1632   ASSERT_TRUE(base::TouchFile(filename, old_last_modified, old_last_modified));
1633   ASSERT_TRUE(base::GetFileInfo(filename, &before_info));
1634   ASSERT_TRUE(database_->UpdateStarted(&lists));
1635   AddDelChunk(safe_browsing_util::kMalwareList, 2);
1636   database_->UpdateFinished(true);
1637   ASSERT_TRUE(base::GetFileInfo(filename, &after_info));
1638   EXPECT_LT(before_info.last_modified, after_info.last_modified);
1639
1640   // Simply calling |UpdateStarted()| then |UpdateFinished()| does not
1641   // update the database file.
1642   ASSERT_TRUE(base::TouchFile(filename, old_last_modified, old_last_modified));
1643   ASSERT_TRUE(base::GetFileInfo(filename, &before_info));
1644   ASSERT_TRUE(database_->UpdateStarted(&lists));
1645   database_->UpdateFinished(true);
1646   ASSERT_TRUE(base::GetFileInfo(filename, &after_info));
1647   EXPECT_EQ(before_info.last_modified, after_info.last_modified);
1648 }
1649
1650 // Test that a filter file is written out during update and read back
1651 // in during setup.
1652 TEST_F(SafeBrowsingDatabaseTest, FilterFile) {
1653   // Create a database with trivial example data and write it out.
1654   {
1655     // Prime the database.
1656     std::vector<SBListChunkRanges> lists;
1657     ASSERT_TRUE(database_->UpdateStarted(&lists));
1658
1659     ScopedVector<SBChunkData> chunks;
1660     chunks.push_back(AddChunkPrefixValue(1, "www.evil.com/malware.html"));
1661     database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1662     database_->UpdateFinished(true);
1663   }
1664
1665   // Find the malware url in the database, don't find a good url.
1666   std::vector<SBPrefix> prefix_hits;
1667   std::vector<SBFullHashResult> cache_hits;
1668   EXPECT_TRUE(database_->ContainsBrowseUrl(
1669       GURL("http://www.evil.com/malware.html"), &prefix_hits, &cache_hits));
1670   EXPECT_FALSE(database_->ContainsBrowseUrl(
1671       GURL("http://www.good.com/goodware.html"), &prefix_hits, &cache_hits));
1672
1673   base::FilePath filter_file = database_->PrefixSetForFilename(
1674       database_->BrowseDBFilename(database_filename_));
1675
1676   // After re-creating the database, it should have a filter read from
1677   // a file, so it should find the same results.
1678   ASSERT_TRUE(base::PathExists(filter_file));
1679   ResetAndReloadFullDatabase();
1680   EXPECT_TRUE(database_->ContainsBrowseUrl(
1681       GURL("http://www.evil.com/malware.html"), &prefix_hits, &cache_hits));
1682   EXPECT_FALSE(database_->ContainsBrowseUrl(
1683       GURL("http://www.good.com/goodware.html"), &prefix_hits, &cache_hits));
1684
1685   // If there is no filter file, the database cannot find malware urls.
1686   base::DeleteFile(filter_file, false);
1687   ASSERT_FALSE(base::PathExists(filter_file));
1688   ResetAndReloadFullDatabase();
1689   EXPECT_FALSE(database_->ContainsBrowseUrl(
1690       GURL("http://www.evil.com/malware.html"), &prefix_hits, &cache_hits));
1691   EXPECT_FALSE(database_->ContainsBrowseUrl(
1692       GURL("http://www.good.com/goodware.html"), &prefix_hits, &cache_hits));
1693 }
1694
1695 TEST_F(SafeBrowsingDatabaseTest, CachedFullMiss) {
1696   const SBPrefix kPrefix1 = 1001U;
1697   const SBFullHash kFullHash1_1 =
1698       SBFullHashForPrefixAndSuffix(kPrefix1, "\x01");
1699
1700   const SBPrefix kPrefix2 = 1002U;
1701   const SBFullHash kFullHash2_1 =
1702       SBFullHashForPrefixAndSuffix(kPrefix2, "\x01");
1703
1704   // Insert prefix kPrefix1 and kPrefix2 into database.
1705   ScopedVector<SBChunkData> chunks;
1706   chunks.push_back(AddChunkPrefix(1, kPrefix1));
1707   chunks.push_back(AddChunkPrefix(2, kPrefix2));
1708
1709   std::vector<SBListChunkRanges> lists;
1710   ASSERT_TRUE(database_->UpdateStarted(&lists));
1711   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1712   database_->UpdateFinished(true);
1713
1714   {
1715     // Cache a full miss result for kPrefix1.
1716     std::vector<SBPrefix> prefixes(1, kPrefix1);
1717     std::vector<SBFullHashResult> cache_results;
1718     database_->CacheHashResults(prefixes, cache_results, kCacheLifetime);
1719   }
1720
1721   {
1722     // kFullHash1_1 gets no prefix hit because of the cached item, and also does
1723     // not have a cache hit.
1724     std::vector<SBFullHash> full_hashes(1, kFullHash1_1);
1725     std::vector<SBPrefix> prefix_hits;
1726     std::vector<SBFullHashResult> cache_hits;
1727     EXPECT_FALSE(database_->ContainsBrowseUrlHashesForTesting(
1728         full_hashes, &prefix_hits, &cache_hits));
1729
1730     // kFullHash2_1 gets a hit from the prefix in the database.
1731     full_hashes.push_back(kFullHash2_1);
1732     prefix_hits.clear();
1733     cache_hits.clear();
1734     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
1735         full_hashes, &prefix_hits, &cache_hits));
1736     ASSERT_EQ(1U, prefix_hits.size());
1737     EXPECT_EQ(kPrefix2, prefix_hits[0]);
1738     EXPECT_TRUE(cache_hits.empty());
1739   }
1740 }
1741
1742 TEST_F(SafeBrowsingDatabaseTest, CachedPrefixHitFullMiss) {
1743   const SBPrefix kPrefix1 = 1001U;
1744   const SBFullHash kFullHash1_1 =
1745       SBFullHashForPrefixAndSuffix(kPrefix1, "\x01");
1746   const SBFullHash kFullHash1_2 =
1747       SBFullHashForPrefixAndSuffix(kPrefix1, "\x02");
1748   const SBFullHash kFullHash1_3 =
1749       SBFullHashForPrefixAndSuffix(kPrefix1, "\x03");
1750
1751   const SBPrefix kPrefix2 = 1002U;
1752   const SBFullHash kFullHash2_1 =
1753       SBFullHashForPrefixAndSuffix(kPrefix2, "\x01");
1754
1755   const SBPrefix kPrefix3 = 1003U;
1756   const SBFullHash kFullHash3_1 =
1757       SBFullHashForPrefixAndSuffix(kPrefix3, "\x01");
1758
1759   // Insert prefix kPrefix1 and kPrefix2 into database.
1760   ScopedVector<SBChunkData> chunks;
1761   chunks.push_back(AddChunkPrefix(1, kPrefix1));
1762   chunks.push_back(AddChunkPrefix(2, kPrefix2));
1763
1764   std::vector<SBListChunkRanges> lists;
1765   ASSERT_TRUE(database_->UpdateStarted(&lists));
1766   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1767   database_->UpdateFinished(true);
1768
1769   {
1770     // kFullHash1_1 has a prefix hit of kPrefix1.
1771     std::vector<SBFullHash> full_hashes;
1772     full_hashes.push_back(kFullHash1_1);
1773     std::vector<SBPrefix> prefix_hits;
1774     std::vector<SBFullHashResult> cache_hits;
1775     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
1776         full_hashes, &prefix_hits, &cache_hits));
1777     ASSERT_EQ(1U, prefix_hits.size());
1778     EXPECT_EQ(kPrefix1, prefix_hits[0]);
1779     EXPECT_TRUE(cache_hits.empty());
1780
1781     // kFullHash2_1 has a prefix hit of kPrefix2.
1782     full_hashes.push_back(kFullHash2_1);
1783     prefix_hits.clear();
1784     cache_hits.clear();
1785     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
1786         full_hashes, &prefix_hits, &cache_hits));
1787     ASSERT_EQ(2U, prefix_hits.size());
1788     EXPECT_EQ(kPrefix1, prefix_hits[0]);
1789     EXPECT_EQ(kPrefix2, prefix_hits[1]);
1790     EXPECT_TRUE(cache_hits.empty());
1791
1792     // kFullHash3_1 has no hits.
1793     full_hashes.push_back(kFullHash3_1);
1794     prefix_hits.clear();
1795     cache_hits.clear();
1796     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
1797         full_hashes, &prefix_hits, &cache_hits));
1798     ASSERT_EQ(2U, prefix_hits.size());
1799     EXPECT_EQ(kPrefix1, prefix_hits[0]);
1800     EXPECT_EQ(kPrefix2, prefix_hits[1]);
1801     EXPECT_TRUE(cache_hits.empty());
1802   }
1803
1804   {
1805     // Cache a fullhash result for two kPrefix1 full hashes.
1806     std::vector<SBPrefix> prefixes(1, kPrefix1);
1807     std::vector<SBFullHashResult> cache_results;
1808
1809     SBFullHashResult full_hash_result;
1810     full_hash_result.list_id = safe_browsing_util::MALWARE;
1811
1812     full_hash_result.hash = kFullHash1_1;
1813     cache_results.push_back(full_hash_result);
1814
1815     full_hash_result.hash = kFullHash1_3;
1816     cache_results.push_back(full_hash_result);
1817
1818     database_->CacheHashResults(prefixes, cache_results, kCacheLifetime);
1819   }
1820
1821   {
1822     // kFullHash1_1 should now see a cache hit.
1823     std::vector<SBFullHash> full_hashes(1, kFullHash1_1);
1824     std::vector<SBPrefix> prefix_hits;
1825     std::vector<SBFullHashResult> cache_hits;
1826     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
1827         full_hashes, &prefix_hits, &cache_hits));
1828     EXPECT_TRUE(prefix_hits.empty());
1829     ASSERT_EQ(1U, cache_hits.size());
1830     EXPECT_TRUE(SBFullHashEqual(kFullHash1_1, cache_hits[0].hash));
1831
1832     // Adding kFullHash2_1 will see the existing cache hit plus the prefix hit
1833     // for kPrefix2.
1834     full_hashes.push_back(kFullHash2_1);
1835     prefix_hits.clear();
1836     cache_hits.clear();
1837     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
1838         full_hashes, &prefix_hits, &cache_hits));
1839     ASSERT_EQ(1U, prefix_hits.size());
1840     EXPECT_EQ(kPrefix2, prefix_hits[0]);
1841     ASSERT_EQ(1U, cache_hits.size());
1842     EXPECT_TRUE(SBFullHashEqual(kFullHash1_1, cache_hits[0].hash));
1843
1844     // kFullHash1_3 also gets a cache hit.
1845     full_hashes.push_back(kFullHash1_3);
1846     prefix_hits.clear();
1847     cache_hits.clear();
1848     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
1849         full_hashes, &prefix_hits, &cache_hits));
1850     ASSERT_EQ(1U, prefix_hits.size());
1851     EXPECT_EQ(kPrefix2, prefix_hits[0]);
1852     ASSERT_EQ(2U, cache_hits.size());
1853     EXPECT_TRUE(SBFullHashEqual(kFullHash1_1, cache_hits[0].hash));
1854     EXPECT_TRUE(SBFullHashEqual(kFullHash1_3, cache_hits[1].hash));
1855   }
1856
1857   {
1858     // Check if DB contains only kFullHash1_3. Should return a cache hit.
1859     std::vector<SBFullHash> full_hashes(1, kFullHash1_3);
1860     std::vector<SBPrefix> prefix_hits;
1861     std::vector<SBFullHashResult> cache_hits;
1862     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
1863         full_hashes, &prefix_hits, &cache_hits));
1864     EXPECT_TRUE(prefix_hits.empty());
1865     ASSERT_EQ(1U, cache_hits.size());
1866     EXPECT_TRUE(SBFullHashEqual(kFullHash1_3, cache_hits[0].hash));
1867   }
1868
1869   {
1870     // kFullHash1_2 has no cache hit, and no prefix hit because of the cache for
1871     // kPrefix1.
1872     std::vector<SBFullHash> full_hashes(1, kFullHash1_2);
1873     std::vector<SBPrefix> prefix_hits;
1874     std::vector<SBFullHashResult> cache_hits;
1875     EXPECT_FALSE(database_->ContainsBrowseUrlHashesForTesting(
1876         full_hashes, &prefix_hits, &cache_hits));
1877
1878     // Other prefix hits possible when kFullHash1_2 hits nothing.
1879     full_hashes.push_back(kFullHash2_1);
1880     prefix_hits.clear();
1881     cache_hits.clear();
1882     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
1883         full_hashes, &prefix_hits, &cache_hits));
1884     ASSERT_EQ(1U, prefix_hits.size());
1885     EXPECT_EQ(kPrefix2, prefix_hits[0]);
1886     EXPECT_TRUE(cache_hits.empty());
1887   }
1888 }
1889
1890 TEST_F(SafeBrowsingDatabaseTest, BrowseFullHashMatching) {
1891   const SBPrefix kPrefix1 = 1001U;
1892   const SBFullHash kFullHash1_1 =
1893       SBFullHashForPrefixAndSuffix(kPrefix1, "\x01");
1894   const SBFullHash kFullHash1_2 =
1895       SBFullHashForPrefixAndSuffix(kPrefix1, "\x02");
1896   const SBFullHash kFullHash1_3 =
1897       SBFullHashForPrefixAndSuffix(kPrefix1, "\x03");
1898
1899   // Insert two full hashes with a shared prefix.
1900   ScopedVector<SBChunkData> chunks;
1901   chunks.push_back(AddChunkFullHash(1, kFullHash1_1));
1902   chunks.push_back(AddChunkFullHash(2, kFullHash1_2));
1903
1904   std::vector<SBListChunkRanges> lists;
1905   ASSERT_TRUE(database_->UpdateStarted(&lists));
1906   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1907   database_->UpdateFinished(true);
1908
1909   {
1910     // Check a full hash which isn't present.
1911     std::vector<SBFullHash> full_hashes(1, kFullHash1_3);
1912     std::vector<SBPrefix> prefix_hits;
1913     std::vector<SBFullHashResult> cache_hits;
1914     EXPECT_FALSE(database_->ContainsBrowseUrlHashesForTesting(
1915         full_hashes, &prefix_hits, &cache_hits));
1916
1917     // Also one which is present, should have a prefix hit.
1918     full_hashes.push_back(kFullHash1_1);
1919     prefix_hits.clear();
1920     cache_hits.clear();
1921     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
1922         full_hashes, &prefix_hits, &cache_hits));
1923     ASSERT_EQ(1U, prefix_hits.size());
1924     EXPECT_EQ(kPrefix1, prefix_hits[0]);
1925     EXPECT_TRUE(cache_hits.empty());
1926
1927     // Two full hash matches with the same prefix should return one prefix hit.
1928     full_hashes.push_back(kFullHash1_2);
1929     prefix_hits.clear();
1930     cache_hits.clear();
1931     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
1932         full_hashes, &prefix_hits, &cache_hits));
1933     ASSERT_EQ(1U, prefix_hits.size());
1934     EXPECT_EQ(kPrefix1, prefix_hits[0]);
1935     EXPECT_TRUE(cache_hits.empty());
1936   }
1937
1938   {
1939     // Cache a gethash result for kFullHash1_2.
1940     SBFullHashResult full_hash_result;
1941     full_hash_result.list_id = safe_browsing_util::MALWARE;
1942     full_hash_result.hash = kFullHash1_2;
1943
1944     std::vector<SBPrefix> prefixes(1, kPrefix1);
1945     std::vector<SBFullHashResult> cache_results(1, full_hash_result);
1946
1947     database_->CacheHashResults(prefixes, cache_results, kCacheLifetime);
1948   }
1949
1950   {
1951     // kFullHash1_3 should still return false, because the cached
1952     // result for kPrefix1 doesn't contain kFullHash1_3.
1953     std::vector<SBFullHash> full_hashes(1, kFullHash1_3);
1954     std::vector<SBPrefix> prefix_hits;
1955     std::vector<SBFullHashResult> cache_hits;
1956     EXPECT_FALSE(database_->ContainsBrowseUrlHashesForTesting(
1957         full_hashes, &prefix_hits, &cache_hits));
1958
1959     // kFullHash1_1 is also not in the cached result, which takes
1960     // priority over the database.
1961     prefix_hits.clear();
1962     full_hashes.push_back(kFullHash1_1);
1963     cache_hits.clear();
1964     EXPECT_FALSE(database_->ContainsBrowseUrlHashesForTesting(
1965         full_hashes, &prefix_hits, &cache_hits));
1966
1967     // kFullHash1_2 is in the cached result.
1968     full_hashes.push_back(kFullHash1_2);
1969     prefix_hits.clear();
1970     cache_hits.clear();
1971     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
1972         full_hashes, &prefix_hits, &cache_hits));
1973     EXPECT_TRUE(prefix_hits.empty());
1974     ASSERT_EQ(1U, cache_hits.size());
1975     EXPECT_TRUE(SBFullHashEqual(kFullHash1_2, cache_hits[0].hash));
1976   }
1977
1978   // Remove kFullHash1_1 from the database.
1979   chunks.clear();
1980   chunks.push_back(SubChunkFullHash(11, kFullHash1_1, 1));
1981
1982   ASSERT_TRUE(database_->UpdateStarted(&lists));
1983   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1984   database_->UpdateFinished(true);
1985
1986   // Cache should be cleared after updating.
1987   EXPECT_TRUE(
1988       database_->GetUnsynchronizedPrefixGetHashCacheForTesting()->empty());
1989
1990   {
1991     // Now the database doesn't contain kFullHash1_1.
1992     std::vector<SBFullHash> full_hashes(1, kFullHash1_1);
1993     std::vector<SBPrefix> prefix_hits;
1994     std::vector<SBFullHashResult> cache_hits;
1995     EXPECT_FALSE(database_->ContainsBrowseUrlHashesForTesting(
1996         full_hashes, &prefix_hits, &cache_hits));
1997
1998     // Nor kFullHash1_3.
1999     full_hashes.push_back(kFullHash1_3);
2000     prefix_hits.clear();
2001     cache_hits.clear();
2002     EXPECT_FALSE(database_->ContainsBrowseUrlHashesForTesting(
2003         full_hashes, &prefix_hits, &cache_hits));
2004
2005     // Still has kFullHash1_2.
2006     full_hashes.push_back(kFullHash1_2);
2007     prefix_hits.clear();
2008     cache_hits.clear();
2009     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
2010         full_hashes, &prefix_hits, &cache_hits));
2011     ASSERT_EQ(1U, prefix_hits.size());
2012     EXPECT_EQ(kPrefix1, prefix_hits[0]);
2013     EXPECT_TRUE(cache_hits.empty());
2014   }
2015
2016   // Remove kFullHash1_2 from the database.
2017   chunks.clear();
2018   chunks.push_back(SubChunkFullHash(12, kFullHash1_2, 2));
2019
2020   ASSERT_TRUE(database_->UpdateStarted(&lists));
2021   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
2022   database_->UpdateFinished(true);
2023
2024   // Cache should be cleared after updating.
2025   EXPECT_TRUE(
2026       database_->GetUnsynchronizedPrefixGetHashCacheForTesting()->empty());
2027
2028   {
2029     // None are present.
2030     std::vector<SBFullHash> full_hashes;
2031     std::vector<SBPrefix> prefix_hits;
2032     std::vector<SBFullHashResult> cache_hits;
2033     full_hashes.push_back(kFullHash1_1);
2034     full_hashes.push_back(kFullHash1_2);
2035     full_hashes.push_back(kFullHash1_3);
2036     EXPECT_FALSE(database_->ContainsBrowseUrlHashesForTesting(
2037         full_hashes, &prefix_hits, &cache_hits));
2038   }
2039 }
2040
2041 TEST_F(SafeBrowsingDatabaseTest, BrowseFullHashAndPrefixMatching) {
2042   const SBPrefix kPrefix1 = 1001U;
2043   const SBFullHash kFullHash1_1 =
2044       SBFullHashForPrefixAndSuffix(kPrefix1, "\x01");
2045   const SBFullHash kFullHash1_2 =
2046       SBFullHashForPrefixAndSuffix(kPrefix1, "\x02");
2047
2048   ScopedVector<SBChunkData> chunks;
2049   chunks.push_back(AddChunkFullHash(1, kFullHash1_1));
2050
2051   std::vector<SBListChunkRanges> lists;
2052   ASSERT_TRUE(database_->UpdateStarted(&lists));
2053   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
2054   database_->UpdateFinished(true);
2055
2056   {
2057     // kFullHash1_2 does not match kFullHash1_1.
2058     std::vector<SBFullHash> full_hashes(1, kFullHash1_2);
2059     std::vector<SBPrefix> prefix_hits;
2060     std::vector<SBFullHashResult> cache_hits;
2061     EXPECT_FALSE(database_->ContainsBrowseUrlHashesForTesting(
2062         full_hashes, &prefix_hits, &cache_hits));
2063   }
2064
2065   // Add a prefix match.
2066   chunks.clear();
2067   chunks.push_back(AddChunkPrefix(2, kPrefix1));
2068
2069   ASSERT_TRUE(database_->UpdateStarted(&lists));
2070   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
2071   database_->UpdateFinished(true);
2072
2073   {
2074     // kFullHash1_2 does match kPrefix1.
2075     std::vector<SBFullHash> full_hashes(1, kFullHash1_2);
2076     std::vector<SBPrefix> prefix_hits;
2077     std::vector<SBFullHashResult> cache_hits;
2078     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
2079         full_hashes, &prefix_hits, &cache_hits));
2080     ASSERT_EQ(1U, prefix_hits.size());
2081     EXPECT_EQ(kPrefix1, prefix_hits[0]);
2082     EXPECT_TRUE(cache_hits.empty());
2083   }
2084
2085   // Remove the full hash.
2086   chunks.clear();
2087   chunks.push_back(SubChunkFullHash(11, kFullHash1_1, 1));
2088
2089   ASSERT_TRUE(database_->UpdateStarted(&lists));
2090   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
2091   database_->UpdateFinished(true);
2092
2093   {
2094     // kFullHash1_2 still returns true due to the prefix hit.
2095     std::vector<SBFullHash> full_hashes(1, kFullHash1_2);
2096     std::vector<SBPrefix> prefix_hits;
2097     std::vector<SBFullHashResult> cache_hits;
2098     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
2099         full_hashes, &prefix_hits, &cache_hits));
2100     ASSERT_EQ(1U, prefix_hits.size());
2101     EXPECT_EQ(kPrefix1, prefix_hits[0]);
2102     EXPECT_TRUE(cache_hits.empty());
2103   }
2104 }
2105
2106 TEST_F(SafeBrowsingDatabaseTest, MalwareIpBlacklist) {
2107   std::vector<SBListChunkRanges> lists;
2108   ASSERT_TRUE(database_->UpdateStarted(&lists));
2109
2110   ScopedVector<SBChunkData> chunks;
2111
2112   // IPv4 prefix match for ::ffff:192.168.1.0/120.
2113   chunks.push_back(AddChunkHashedIpValue(1, "::ffff:192.168.1.0", 120));
2114
2115   // IPv4 exact match for ::ffff:192.1.1.1.
2116   chunks.push_back(AddChunkHashedIpValue(2, "::ffff:192.1.1.1", 128));
2117
2118   // IPv6 exact match for: fe80::31a:a0ff:fe10:786e/128.
2119   chunks.push_back(AddChunkHashedIpValue(3, "fe80::31a:a0ff:fe10:786e", 128));
2120
2121   // IPv6 prefix match for: 2620:0:1000:3103::/64.
2122   chunks.push_back(AddChunkHashedIpValue(4, "2620:0:1000:3103::", 64));
2123
2124   // IPv4 prefix match for ::ffff:192.1.122.0/119.
2125   chunks.push_back(AddChunkHashedIpValue(5, "::ffff:192.1.122.0", 119));
2126
2127   // IPv4 prefix match for ::ffff:192.1.128.0/113.
2128   chunks.push_back(AddChunkHashedIpValue(6, "::ffff:192.1.128.0", 113));
2129
2130   database_->InsertChunks(safe_browsing_util::kIPBlacklist, chunks.get());
2131   database_->UpdateFinished(true);
2132
2133   EXPECT_FALSE(database_->ContainsMalwareIP("192.168.0.255"));
2134   EXPECT_TRUE(database_->ContainsMalwareIP("192.168.1.0"));
2135   EXPECT_TRUE(database_->ContainsMalwareIP("192.168.1.255"));
2136   EXPECT_TRUE(database_->ContainsMalwareIP("192.168.1.10"));
2137   EXPECT_TRUE(database_->ContainsMalwareIP("::ffff:192.168.1.2"));
2138   EXPECT_FALSE(database_->ContainsMalwareIP("192.168.2.0"));
2139
2140   EXPECT_FALSE(database_->ContainsMalwareIP("192.1.1.0"));
2141   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.1.1"));
2142   EXPECT_FALSE(database_->ContainsMalwareIP("192.1.1.2"));
2143
2144   EXPECT_FALSE(database_->ContainsMalwareIP(
2145       "2620:0:1000:3102:ffff:ffff:ffff:ffff"));
2146   EXPECT_TRUE(database_->ContainsMalwareIP("2620:0:1000:3103::"));
2147   EXPECT_TRUE(database_->ContainsMalwareIP(
2148       "2620:0:1000:3103:ffff:ffff:ffff:ffff"));
2149   EXPECT_FALSE(database_->ContainsMalwareIP("2620:0:1000:3104::"));
2150
2151   EXPECT_FALSE(database_->ContainsMalwareIP("fe80::21a:a0ff:fe10:786d"));
2152   EXPECT_TRUE(database_->ContainsMalwareIP("fe80::31a:a0ff:fe10:786e"));
2153   EXPECT_FALSE(database_->ContainsMalwareIP("fe80::21a:a0ff:fe10:786f"));
2154
2155   EXPECT_FALSE(database_->ContainsMalwareIP("192.1.121.255"));
2156   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.122.0"));
2157   EXPECT_TRUE(database_->ContainsMalwareIP("::ffff:192.1.122.1"));
2158   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.122.255"));
2159   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.123.0"));
2160   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.123.255"));
2161   EXPECT_FALSE(database_->ContainsMalwareIP("192.1.124.0"));
2162
2163   EXPECT_FALSE(database_->ContainsMalwareIP("192.1.127.255"));
2164   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.128.0"));
2165   EXPECT_TRUE(database_->ContainsMalwareIP("::ffff:192.1.128.1"));
2166   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.128.255"));
2167   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.255.0"));
2168   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.255.255"));
2169   EXPECT_FALSE(database_->ContainsMalwareIP("192.2.0.0"));
2170 }
2171
2172 TEST_F(SafeBrowsingDatabaseTest, ContainsBrowseURL) {
2173   std::vector<SBListChunkRanges> lists;
2174   ASSERT_TRUE(database_->UpdateStarted(&lists));
2175
2176   // Add a host-level hit.
2177   {
2178     ScopedVector<SBChunkData> chunks;
2179     chunks.push_back(AddChunkPrefixValue(1, "www.evil.com/"));
2180     database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
2181   }
2182
2183   // Add a specific fullhash.
2184   static const char kWhateverMalware[] = "www.whatever.com/malware.html";
2185   {
2186     ScopedVector<SBChunkData> chunks;
2187     chunks.push_back(AddChunkFullHashValue(2, kWhateverMalware));
2188     database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
2189   }
2190
2191   // Add a fullhash which has a prefix collision for a known url.
2192   static const char kExampleFine[] = "www.example.com/fine.html";
2193   static const char kExampleCollision[] =
2194       "www.example.com/3123364814/malware.htm";
2195   ASSERT_EQ(SBPrefixForString(kExampleFine),
2196             SBPrefixForString(kExampleCollision));
2197   {
2198     ScopedVector<SBChunkData> chunks;
2199     chunks.push_back(AddChunkFullHashValue(3, kExampleCollision));
2200     database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
2201   }
2202
2203   database_->UpdateFinished(true);
2204
2205   std::vector<SBPrefix> prefix_hits;
2206   std::vector<SBFullHashResult> cache_hits;
2207
2208   // Anything will hit the host prefix.
2209   EXPECT_TRUE(database_->ContainsBrowseUrl(
2210       GURL("http://www.evil.com/malware.html"), &prefix_hits, &cache_hits));
2211   ASSERT_EQ(1U, prefix_hits.size());
2212   EXPECT_EQ(SBPrefixForString("www.evil.com/"), prefix_hits[0]);
2213   EXPECT_TRUE(cache_hits.empty());
2214
2215   // Hit the specific URL prefix.
2216   EXPECT_TRUE(database_->ContainsBrowseUrl(
2217       GURL(std::string("http://") + kWhateverMalware),
2218       &prefix_hits, &cache_hits));
2219   ASSERT_EQ(1U, prefix_hits.size());
2220   EXPECT_EQ(SBPrefixForString(kWhateverMalware), prefix_hits[0]);
2221   EXPECT_TRUE(cache_hits.empty());
2222
2223   // Other URLs at that host are fine.
2224   EXPECT_FALSE(database_->ContainsBrowseUrl(
2225       GURL("http://www.whatever.com/fine.html"), &prefix_hits, &cache_hits));
2226   EXPECT_TRUE(prefix_hits.empty());
2227   EXPECT_TRUE(cache_hits.empty());
2228
2229   // Hit the specific URL full hash.
2230   EXPECT_TRUE(database_->ContainsBrowseUrl(
2231       GURL(std::string("http://") + kExampleCollision),
2232       &prefix_hits, &cache_hits));
2233   ASSERT_EQ(1U, prefix_hits.size());
2234   EXPECT_EQ(SBPrefixForString(kExampleCollision), prefix_hits[0]);
2235   EXPECT_TRUE(cache_hits.empty());
2236
2237   // This prefix collides, but no full hash match.
2238   EXPECT_FALSE(database_->ContainsBrowseUrl(
2239       GURL(std::string("http://") + kExampleFine), &prefix_hits, &cache_hits));
2240 }