chrome/browser/safe_browsing/safe_browsing_database_unittest.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4 //
   5 // Unit tests for the SafeBrowsing storage system.
   6
   7 #include "chrome/browser/safe_browsing/safe_browsing_database.h"
   8
   9 #include "base/files/file_util.h"
  10 #include "base/files/scoped_temp_dir.h"
  11 #include "base/logging.h"
  12 #include "base/memory/scoped_vector.h"
  13 #include "base/message_loop/message_loop.h"
  14 #include "base/sha1.h"
  15 #include "base/strings/string_number_conversions.h"
  16 #include "base/strings/string_split.h"
  17 #include "base/test/test_simple_task_runner.h"
  18 #include "base/time/time.h"
  19 #include "chrome/browser/safe_browsing/chunk.pb.h"
  20 #include "chrome/browser/safe_browsing/safe_browsing_store_file.h"
  21 #include "crypto/sha2.h"
  22 #include "net/base/net_util.h"
  23 #include "sql/connection.h"
  24 #include "sql/statement.h"
  25 #include "testing/gtest/include/gtest/gtest.h"
  26 #include "testing/platform_test.h"
  27 #include "url/gurl.h"
  28
  29 using base::Time;
  30 using base::TimeDelta;
  31
  32 namespace {
  33
  34 const TimeDelta kCacheLifetime = TimeDelta::FromMinutes(45);
  35
  36 SBPrefix SBPrefixForString(const std::string& str) {
  37   return SBFullHashForString(str).prefix;
  38 }
  39
  40 // Construct a full hash which has the given prefix, with the given
  41 // suffix data coming after the prefix.
  42 SBFullHash SBFullHashForPrefixAndSuffix(SBPrefix prefix,
  43                                         const base::StringPiece& suffix) {
  44   SBFullHash full_hash;
  45   memset(&full_hash, 0, sizeof(SBFullHash));
  46   full_hash.prefix = prefix;
  47   CHECK_LE(suffix.size() + sizeof(SBPrefix), sizeof(SBFullHash));
  48   memcpy(full_hash.full_hash + sizeof(SBPrefix), suffix.data(), suffix.size());
  49   return full_hash;
  50 }
  51
  52 std::string HashedIpPrefix(const std::string& ip_prefix, size_t prefix_size) {
  53   net::IPAddressNumber ip_number;
  54   EXPECT_TRUE(net::ParseIPLiteralToNumber(ip_prefix, &ip_number));
  55   EXPECT_EQ(net::kIPv6AddressSize, ip_number.size());
  56   const std::string hashed_ip_prefix = base::SHA1HashString(
  57       net::IPAddressToPackedString(ip_number));
  58   std::string hash(crypto::kSHA256Length, '\0');
  59   hash.replace(0, hashed_ip_prefix.size(), hashed_ip_prefix);
  60   hash[base::kSHA1Length] = static_cast<char>(prefix_size);
  61   return hash;
  62 }
  63
  64 // Helper to build a chunk.  Caller takes ownership.
  65 SBChunkData* BuildChunk(int chunk_number,
  66                         safe_browsing::ChunkData::ChunkType chunk_type,
  67                         safe_browsing::ChunkData::PrefixType prefix_type,
  68                         const void* data, size_t data_size,
  69                         const std::vector<int>& add_chunk_numbers) {
  70   scoped_ptr<safe_browsing::ChunkData> raw_data(new safe_browsing::ChunkData);
  71   raw_data->set_chunk_number(chunk_number);
  72   raw_data->set_chunk_type(chunk_type);
  73   raw_data->set_prefix_type(prefix_type);
  74   raw_data->set_hashes(data, data_size);
  75   raw_data->clear_add_numbers();
  76   for (size_t i = 0; i < add_chunk_numbers.size(); ++i) {
  77     raw_data->add_add_numbers(add_chunk_numbers[i]);
  78   }
  79
  80   return new SBChunkData(raw_data.release());
  81 }
  82
  83 // Create add chunk with a single prefix.
  84 SBChunkData* AddChunkPrefix(int chunk_number, SBPrefix prefix) {
  85   return BuildChunk(chunk_number, safe_browsing::ChunkData::ADD,
  86                     safe_browsing::ChunkData::PREFIX_4B,
  87                     &prefix, sizeof(prefix),
  88                     std::vector<int>());
  89 }
  90
  91 // Create add chunk with a single prefix generated from |value|.
  92 SBChunkData* AddChunkPrefixValue(int chunk_number,
  93                                  const std::string& value) {
  94   return AddChunkPrefix(chunk_number, SBPrefixForString(value));
  95 }
  96
  97 // Generate an add chunk with two prefixes.
  98 SBChunkData* AddChunkPrefix2Value(int chunk_number,
  99                                   const std::string& value1,
 100                                   const std::string& value2) {
 101   const SBPrefix prefixes[2] = {
 102     SBPrefixForString(value1),
 103     SBPrefixForString(value2),
 104   };
 105   return BuildChunk(chunk_number, safe_browsing::ChunkData::ADD,
 106                     safe_browsing::ChunkData::PREFIX_4B,
 107                     &prefixes[0], sizeof(prefixes),
 108                     std::vector<int>());
 109 }
 110
 111 // Generate an add chunk with four prefixes.
 112 SBChunkData* AddChunkPrefix4Value(int chunk_number,
 113                                   const std::string& value1,
 114                                   const std::string& value2,
 115                                   const std::string& value3,
 116                                   const std::string& value4) {
 117   const SBPrefix prefixes[4] = {
 118     SBPrefixForString(value1),
 119     SBPrefixForString(value2),
 120     SBPrefixForString(value3),
 121     SBPrefixForString(value4),
 122   };
 123   return BuildChunk(chunk_number, safe_browsing::ChunkData::ADD,
 124                     safe_browsing::ChunkData::PREFIX_4B,
 125                     &prefixes[0], sizeof(prefixes),
 126                     std::vector<int>());
 127 }
 128
 129 // Generate an add chunk with a full hash.
 130 SBChunkData* AddChunkFullHash(int chunk_number, SBFullHash full_hash) {
 131   return BuildChunk(chunk_number, safe_browsing::ChunkData::ADD,
 132                     safe_browsing::ChunkData::FULL_32B,
 133                     &full_hash, sizeof(full_hash),
 134                     std::vector<int>());
 135 }
 136
 137 // Generate an add chunk with a full hash generated from |value|.
 138 SBChunkData* AddChunkFullHashValue(int chunk_number,
 139                                    const std::string& value) {
 140   return AddChunkFullHash(chunk_number, SBFullHashForString(value));
 141 }
 142
 143 // Generate an add chunk with two full hashes.
 144 SBChunkData* AddChunkFullHash2Value(int chunk_number,
 145                                    const std::string& value1,
 146                                    const std::string& value2) {
 147   const SBFullHash full_hashes[2] = {
 148     SBFullHashForString(value1),
 149     SBFullHashForString(value2),
 150   };
 151   return BuildChunk(chunk_number, safe_browsing::ChunkData::ADD,
 152                     safe_browsing::ChunkData::FULL_32B,
 153                     &full_hashes[0], sizeof(full_hashes),
 154                     std::vector<int>());
 155 }
 156
 157 // Generate a sub chunk with a prefix generated from |value|.
 158 SBChunkData* SubChunkPrefixValue(int chunk_number,
 159                                  const std::string& value,
 160                                  int add_chunk_number) {
 161   const SBPrefix prefix = SBPrefixForString(value);
 162   return BuildChunk(chunk_number, safe_browsing::ChunkData::SUB,
 163                     safe_browsing::ChunkData::PREFIX_4B,
 164                     &prefix, sizeof(prefix),
 165                     std::vector<int>(1, add_chunk_number));
 166 }
 167
 168 // Generate a sub chunk with two prefixes.
 169 SBChunkData* SubChunkPrefix2Value(int chunk_number,
 170                                   const std::string& value1,
 171                                   int add_chunk_number1,
 172                                   const std::string& value2,
 173                                   int add_chunk_number2) {
 174   const SBPrefix prefixes[2] = {
 175     SBPrefixForString(value1),
 176     SBPrefixForString(value2),
 177   };
 178   std::vector<int> add_chunk_numbers;
 179   add_chunk_numbers.push_back(add_chunk_number1);
 180   add_chunk_numbers.push_back(add_chunk_number2);
 181   return BuildChunk(chunk_number, safe_browsing::ChunkData::SUB,
 182                     safe_browsing::ChunkData::PREFIX_4B,
 183                     &prefixes[0], sizeof(prefixes),
 184                     add_chunk_numbers);
 185 }
 186
 187 // Generate a sub chunk with a full hash.
 188 SBChunkData* SubChunkFullHash(int chunk_number,
 189                               SBFullHash full_hash,
 190                               int add_chunk_number) {
 191   return BuildChunk(chunk_number, safe_browsing::ChunkData::SUB,
 192                     safe_browsing::ChunkData::FULL_32B,
 193                     &full_hash, sizeof(full_hash),
 194                     std::vector<int>(1, add_chunk_number));
 195 }
 196
 197 // Generate a sub chunk with a full hash generated from |value|.
 198 SBChunkData* SubChunkFullHashValue(int chunk_number,
 199                                    const std::string& value,
 200                                    int add_chunk_number) {
 201   return SubChunkFullHash(chunk_number,
 202                           SBFullHashForString(value),
 203                           add_chunk_number);
 204 }
 205
 206 // Generate an add chunk with a single full hash for the ip blacklist.
 207 SBChunkData* AddChunkHashedIpValue(int chunk_number,
 208                                    const std::string& ip_str,
 209                                    size_t prefix_size) {
 210   const std::string full_hash_str = HashedIpPrefix(ip_str, prefix_size);
 211   EXPECT_EQ(sizeof(SBFullHash), full_hash_str.size());
 212   SBFullHash full_hash;
 213   std::memcpy(&(full_hash.full_hash), full_hash_str.data(), sizeof(SBFullHash));
 214   return BuildChunk(chunk_number, safe_browsing::ChunkData::ADD,
 215                     safe_browsing::ChunkData::FULL_32B,
 216                     &full_hash, sizeof(full_hash),
 217                     std::vector<int>());
 218 }
 219
 220 // Prevent DCHECK from killing tests.
 221 // TODO(shess): Pawel disputes the use of this, so the test which uses
 222 // it is DISABLED.  http://crbug.com/56448
 223 class ScopedLogMessageIgnorer {
 224  public:
 225   ScopedLogMessageIgnorer() {
 226     logging::SetLogMessageHandler(&LogMessageIgnorer);
 227   }
 228   ~ScopedLogMessageIgnorer() {
 229     // TODO(shess): Would be better to verify whether anyone else
 230     // changed it, and then restore it to the previous value.
 231     logging::SetLogMessageHandler(NULL);
 232   }
 233
 234  private:
 235   static bool LogMessageIgnorer(int severity, const char* file, int line,
 236       size_t message_start, const std::string& str) {
 237     // Intercept FATAL, strip the stack backtrace, and log it without
 238     // the crash part.
 239     if (severity == logging::LOG_FATAL) {
 240       size_t newline = str.find('\n');
 241       if (newline != std::string::npos) {
 242         const std::string msg = str.substr(0, newline + 1);
 243         fprintf(stderr, "%s", msg.c_str());
 244         fflush(stderr);
 245       }
 246       return true;
 247     }
 248
 249     return false;
 250   }
 251 };
 252
 253 }  // namespace
 254
 255 class SafeBrowsingDatabaseTest : public PlatformTest {
 256  public:
 257   SafeBrowsingDatabaseTest() : task_runner_(new base::TestSimpleTaskRunner) {}
 258
 259   void SetUp() override {
 260     PlatformTest::SetUp();
 261
 262     // Setup a database in a temporary directory.
 263     ASSERT_TRUE(temp_dir_.CreateUniqueTempDir());
 264     database_filename_ =
 265         temp_dir_.path().AppendASCII("SafeBrowsingTestDatabase");
 266
 267     ResetAndReloadFullDatabase();
 268   }
 269
 270   void TearDown() override {
 271     database_.reset();
 272
 273     PlatformTest::TearDown();
 274   }
 275
 276   // Reloads the |database_| in a new SafeBrowsingDatabaseNew object with all
 277   // stores enabled.
 278   void ResetAndReloadFullDatabase() {
 279     SafeBrowsingStoreFile* browse_store =
 280         new SafeBrowsingStoreFile(task_runner_);
 281     SafeBrowsingStoreFile* download_store =
 282         new SafeBrowsingStoreFile(task_runner_);
 283     SafeBrowsingStoreFile* csd_whitelist_store =
 284         new SafeBrowsingStoreFile(task_runner_);
 285     SafeBrowsingStoreFile* download_whitelist_store =
 286         new SafeBrowsingStoreFile(task_runner_);
 287     SafeBrowsingStoreFile* inclusion_whitelist_store =
 288         new SafeBrowsingStoreFile(task_runner_);
 289     SafeBrowsingStoreFile* extension_blacklist_store =
 290         new SafeBrowsingStoreFile(task_runner_);
 291     SafeBrowsingStoreFile* side_effect_free_whitelist_store =
 292         new SafeBrowsingStoreFile(task_runner_);
 293     SafeBrowsingStoreFile* ip_blacklist_store =
 294         new SafeBrowsingStoreFile(task_runner_);
 295     SafeBrowsingStoreFile* unwanted_software_store =
 296         new SafeBrowsingStoreFile(task_runner_);
 297     database_.reset(new SafeBrowsingDatabaseNew(
 298         task_runner_, browse_store, download_store, csd_whitelist_store,
 299         download_whitelist_store, inclusion_whitelist_store,
 300         extension_blacklist_store, side_effect_free_whitelist_store,
 301         ip_blacklist_store, unwanted_software_store));
 302     database_->Init(database_filename_);
 303   }
 304
 305   void GetListsInfo(std::vector<SBListChunkRanges>* lists) {
 306     lists->clear();
 307     ASSERT_TRUE(database_->UpdateStarted(lists));
 308     database_->UpdateFinished(true);
 309   }
 310
 311   // Helper function to do an AddDel or SubDel command.
 312   void DelChunk(const std::string& list,
 313                 int chunk_id,
 314                 bool is_sub_del) {
 315     std::vector<SBChunkDelete> deletes;
 316     SBChunkDelete chunk_delete;
 317     chunk_delete.list_name = list;
 318     chunk_delete.is_sub_del = is_sub_del;
 319     chunk_delete.chunk_del.push_back(ChunkRange(chunk_id));
 320     deletes.push_back(chunk_delete);
 321     database_->DeleteChunks(deletes);
 322   }
 323
 324   void AddDelChunk(const std::string& list, int chunk_id) {
 325     DelChunk(list, chunk_id, false);
 326   }
 327
 328   void SubDelChunk(const std::string& list, int chunk_id) {
 329     DelChunk(list, chunk_id, true);
 330   }
 331
 332   // Utility function for setting up the database for the caching test.
 333   void PopulateDatabaseForCacheTest();
 334
 335   scoped_refptr<base::TestSimpleTaskRunner> task_runner_;
 336   scoped_ptr<SafeBrowsingDatabaseNew> database_;
 337   base::FilePath database_filename_;
 338   base::ScopedTempDir temp_dir_;
 339 };
 340
 341 // Tests retrieving list name information.
 342 TEST_F(SafeBrowsingDatabaseTest, BrowseListsInfo) {
 343   std::vector<SBListChunkRanges> lists;
 344   ScopedVector<SBChunkData> chunks;
 345
 346   chunks.push_back(AddChunkPrefixValue(1, "www.evil.com/malware.html"));
 347   chunks.push_back(AddChunkPrefixValue(2, "www.foo.com/malware.html"));
 348   chunks.push_back(AddChunkPrefixValue(3, "www.whatever.com/malware.html"));
 349
 350   ASSERT_TRUE(database_->UpdateStarted(&lists));
 351   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
 352   database_->UpdateFinished(true);
 353
 354   GetListsInfo(&lists);
 355   ASSERT_LE(1U, lists.size());
 356   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
 357   EXPECT_EQ("1-3", lists[0].adds);
 358   EXPECT_TRUE(lists[0].subs.empty());
 359
 360   // Insert a malware sub chunk.
 361   chunks.clear();
 362   chunks.push_back(SubChunkPrefixValue(7, "www.subbed.com/noteveil1.html", 19));
 363
 364   ASSERT_TRUE(database_->UpdateStarted(&lists));
 365   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
 366   database_->UpdateFinished(true);
 367
 368   GetListsInfo(&lists);
 369   ASSERT_LE(1U, lists.size());
 370   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
 371   EXPECT_EQ("1-3", lists[0].adds);
 372   EXPECT_EQ("7", lists[0].subs);
 373   if (lists.size() == 2) {
 374     // Old style database won't have the second entry since it creates the lists
 375     // when it receives an update containing that list. The filter-based
 376     // database has these values hard coded.
 377     EXPECT_EQ(safe_browsing_util::kPhishingList, lists[1].name);
 378     EXPECT_TRUE(lists[1].adds.empty());
 379     EXPECT_TRUE(lists[1].subs.empty());
 380   }
 381
 382   // Add phishing chunks.
 383   chunks.clear();
 384   chunks.push_back(AddChunkPrefixValue(47, "www.evil.com/phishing.html"));
 385   chunks.push_back(
 386       SubChunkPrefixValue(200, "www.phishy.com/notevil1.html", 1999));
 387   chunks.push_back(
 388       SubChunkPrefixValue(201, "www.phishy2.com/notevil1.html", 1999));
 389
 390   ASSERT_TRUE(database_->UpdateStarted(&lists));
 391   database_->InsertChunks(safe_browsing_util::kPhishingList, chunks.get());
 392   database_->UpdateFinished(true);
 393
 394   GetListsInfo(&lists);
 395   ASSERT_LE(2U, lists.size());
 396   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
 397   EXPECT_EQ("1-3", lists[0].adds);
 398   EXPECT_EQ("7", lists[0].subs);
 399   EXPECT_EQ(safe_browsing_util::kPhishingList, lists[1].name);
 400   EXPECT_EQ("47", lists[1].adds);
 401   EXPECT_EQ("200-201", lists[1].subs);
 402 }
 403
 404 TEST_F(SafeBrowsingDatabaseTest, ListNames) {
 405   ScopedVector<SBChunkData> chunks;
 406
 407   std::vector<SBListChunkRanges> lists;
 408   ASSERT_TRUE(database_->UpdateStarted(&lists));
 409
 410   // Insert malware, phish, binurl and bindownload add chunks.
 411   chunks.push_back(AddChunkPrefixValue(1, "www.evil.com/malware.html"));
 412   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
 413
 414   chunks.clear();
 415   chunks.push_back(AddChunkPrefixValue(2, "www.foo.com/malware.html"));
 416   database_->InsertChunks(safe_browsing_util::kPhishingList, chunks.get());
 417
 418   chunks.clear();
 419   chunks.push_back(AddChunkPrefixValue(3, "www.whatever.com/download.html"));
 420   database_->InsertChunks(safe_browsing_util::kBinUrlList, chunks.get());
 421
 422   chunks.clear();
 423   chunks.push_back(AddChunkFullHashValue(5, "www.forwhitelist.com/a.html"));
 424   database_->InsertChunks(safe_browsing_util::kCsdWhiteList, chunks.get());
 425
 426   chunks.clear();
 427   chunks.push_back(AddChunkFullHashValue(6, "www.download.com/"));
 428   database_->InsertChunks(safe_browsing_util::kDownloadWhiteList, chunks.get());
 429
 430   chunks.clear();
 431   chunks.push_back(AddChunkFullHashValue(7, "www.inclusion.com/"));
 432   database_->InsertChunks(safe_browsing_util::kInclusionWhitelist,
 433                           chunks.get());
 434
 435   chunks.clear();
 436   chunks.push_back(AddChunkFullHashValue(8,
 437                                          "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
 438                                          "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"));
 439   database_->InsertChunks(safe_browsing_util::kExtensionBlacklist,
 440                           chunks.get());
 441
 442   chunks.clear();
 443   chunks.push_back(AddChunkFullHashValue(9, "www.sideeffectfree.com"));
 444   database_->InsertChunks(safe_browsing_util::kSideEffectFreeWhitelist,
 445                           chunks.get());
 446
 447   chunks.clear();
 448   chunks.push_back(AddChunkHashedIpValue(10, "::ffff:192.168.1.0", 120));
 449   database_->InsertChunks(safe_browsing_util::kIPBlacklist, chunks.get());
 450
 451   chunks.clear();
 452   chunks.push_back(AddChunkPrefixValue(11, "www.unwanted.com/software.html"));
 453   database_->InsertChunks(safe_browsing_util::kUnwantedUrlList, chunks.get());
 454
 455   database_->UpdateFinished(true);
 456
 457   GetListsInfo(&lists);
 458   ASSERT_EQ(10U, lists.size());
 459   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
 460   EXPECT_EQ("1", lists[0].adds);
 461   EXPECT_TRUE(lists[0].subs.empty());
 462   EXPECT_EQ(safe_browsing_util::kPhishingList, lists[1].name);
 463   EXPECT_EQ("2", lists[1].adds);
 464   EXPECT_TRUE(lists[1].subs.empty());
 465   EXPECT_EQ(safe_browsing_util::kBinUrlList, lists[2].name);
 466   EXPECT_EQ("3", lists[2].adds);
 467   EXPECT_TRUE(lists[2].subs.empty());
 468   EXPECT_EQ(safe_browsing_util::kCsdWhiteList, lists[3].name);
 469   EXPECT_EQ("5", lists[3].adds);
 470   EXPECT_TRUE(lists[3].subs.empty());
 471   EXPECT_EQ(safe_browsing_util::kDownloadWhiteList, lists[4].name);
 472   EXPECT_EQ("6", lists[4].adds);
 473   EXPECT_TRUE(lists[4].subs.empty());
 474   EXPECT_EQ(safe_browsing_util::kInclusionWhitelist, lists[5].name);
 475   EXPECT_EQ("7", lists[5].adds);
 476   EXPECT_TRUE(lists[5].subs.empty());
 477   EXPECT_EQ(safe_browsing_util::kExtensionBlacklist, lists[6].name);
 478   EXPECT_EQ("8", lists[6].adds);
 479   EXPECT_TRUE(lists[6].subs.empty());
 480   EXPECT_EQ(safe_browsing_util::kSideEffectFreeWhitelist, lists[7].name);
 481   EXPECT_EQ("9", lists[7].adds);
 482   EXPECT_TRUE(lists[7].subs.empty());
 483   EXPECT_EQ(safe_browsing_util::kIPBlacklist, lists[8].name);
 484   EXPECT_EQ("10", lists[8].adds);
 485   EXPECT_TRUE(lists[8].subs.empty());
 486   EXPECT_EQ(safe_browsing_util::kUnwantedUrlList, lists[9].name);
 487   EXPECT_EQ("11", lists[9].adds);
 488   EXPECT_TRUE(lists[9].subs.empty());
 489
 490   database_.reset();
 491 }
 492
 493 // Checks database reading and writing for browse and unwanted PrefixSets.
 494 TEST_F(SafeBrowsingDatabaseTest, BrowseAndUnwantedDatabasesAndPrefixSets) {
 495   struct TestCase {
 496     using TestListContainsBadUrl = bool (SafeBrowsingDatabase::*)(
 497         const GURL& url,
 498         std::vector<SBPrefix>* prefix_hits,
 499         std::vector<SBFullHashResult>* cache_hits);
 500
 501     const char* test_list_name;
 502     size_t expected_list_index;
 503     TestListContainsBadUrl test_list_contains_bad_url;
 504   } const kTestCases[] {
 505     { safe_browsing_util::kMalwareList, 0U,
 506       &SafeBrowsingDatabase::ContainsBrowseUrl },
 507     { safe_browsing_util::kPhishingList, 1U,
 508       &SafeBrowsingDatabase::ContainsBrowseUrl },
 509     { safe_browsing_util::kUnwantedUrlList, 9U,
 510       &SafeBrowsingDatabase::ContainsUnwantedSoftwareUrl },
 511   };
 512
 513   for (const auto& test_case : kTestCases) {
 514     SCOPED_TRACE(std::string("Tested list at fault => ") +
 515                  test_case.test_list_name);
 516
 517     std::vector<SBListChunkRanges> lists;
 518     ScopedVector<SBChunkData> chunks;
 519
 520     chunks.push_back(AddChunkPrefix2Value(1,
 521                                           "www.evil.com/phishing.html",
 522                                           "www.evil.com/malware.html"));
 523     chunks.push_back(AddChunkPrefix4Value(2,
 524                                           "www.evil.com/notevil1.html",
 525                                           "www.evil.com/notevil2.html",
 526                                           "www.good.com/good1.html",
 527                                           "www.good.com/good2.html"));
 528     chunks.push_back(AddChunkPrefixValue(3, "192.168.0.1/malware.html"));
 529     chunks.push_back(AddChunkFullHashValue(7, "www.evil.com/evil.html"));
 530
 531     ASSERT_TRUE(database_->UpdateStarted(&lists));
 532     database_->InsertChunks(test_case.test_list_name, chunks.get());
 533     database_->UpdateFinished(true);
 534
 535     // Make sure they were added correctly.
 536     GetListsInfo(&lists);
 537
 538     ASSERT_LE(1U, lists.size());
 539     EXPECT_EQ(test_case.test_list_name,
 540               lists[test_case.expected_list_index].name);
 541     EXPECT_EQ("1-3,7", lists[test_case.expected_list_index].adds);
 542     EXPECT_TRUE(lists[test_case.expected_list_index].subs.empty());
 543
 544     std::vector<SBPrefix> prefix_hits;
 545     std::vector<SBFullHashResult> cache_hits;
 546     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 547         GURL("http://www.evil.com/phishing.html"), &prefix_hits, &cache_hits));
 548     ASSERT_EQ(1U, prefix_hits.size());
 549     EXPECT_EQ(SBPrefixForString("www.evil.com/phishing.html"), prefix_hits[0]);
 550     EXPECT_TRUE(cache_hits.empty());
 551
 552     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 553         GURL("http://www.evil.com/malware.html"), &prefix_hits, &cache_hits));
 554
 555     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 556         GURL("http://www.evil.com/notevil1.html"), &prefix_hits, &cache_hits));
 557
 558     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 559         GURL("http://www.evil.com/notevil2.html"), &prefix_hits, &cache_hits));
 560
 561     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 562         GURL("http://www.good.com/good1.html"), &prefix_hits, &cache_hits));
 563
 564     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 565         GURL("http://www.good.com/good2.html"), &prefix_hits, &cache_hits));
 566
 567     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 568         GURL("http://192.168.0.1/malware.html"), &prefix_hits, &cache_hits));
 569
 570     EXPECT_FALSE((database_.get()->*test_case.test_list_contains_bad_url)(
 571         GURL("http://www.evil.com/"), &prefix_hits, &cache_hits));
 572     EXPECT_TRUE(prefix_hits.empty());
 573     EXPECT_TRUE(cache_hits.empty());
 574
 575     EXPECT_FALSE((database_.get()->*test_case.test_list_contains_bad_url)(
 576         GURL("http://www.evil.com/robots.txt"), &prefix_hits, &cache_hits));
 577
 578     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 579         GURL("http://www.evil.com/evil.html"), &prefix_hits, &cache_hits));
 580     ASSERT_EQ(1U, prefix_hits.size());
 581     EXPECT_EQ(SBPrefixForString("www.evil.com/evil.html"), prefix_hits[0]);
 582
 583     // Attempt to re-add the first chunk (should be a no-op).
 584     // see bug: http://code.google.com/p/chromium/issues/detail?id=4522
 585     chunks.clear();
 586     chunks.push_back(AddChunkPrefix2Value(1,
 587                                           "www.evil.com/phishing.html",
 588                                           "www.evil.com/malware.html"));
 589     ASSERT_TRUE(database_->UpdateStarted(&lists));
 590     database_->InsertChunks(test_case.test_list_name, chunks.get());
 591     database_->UpdateFinished(true);
 592
 593     GetListsInfo(&lists);
 594     ASSERT_LE(1U, lists.size());
 595     EXPECT_EQ(test_case.test_list_name,
 596               lists[test_case.expected_list_index].name);
 597     EXPECT_EQ("1-3,7", lists[test_case.expected_list_index].adds);
 598     EXPECT_TRUE(lists[test_case.expected_list_index].subs.empty());
 599
 600     // Test removing a single prefix from the add chunk.
 601     chunks.clear();
 602     chunks.push_back(SubChunkPrefixValue(4, "www.evil.com/notevil1.html", 2));
 603     ASSERT_TRUE(database_->UpdateStarted(&lists));
 604     database_->InsertChunks(test_case.test_list_name, chunks.get());
 605     database_->UpdateFinished(true);
 606
 607     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 608         GURL("http://www.evil.com/phishing.html"), &prefix_hits, &cache_hits));
 609     ASSERT_EQ(1U, prefix_hits.size());
 610     EXPECT_EQ(SBPrefixForString("www.evil.com/phishing.html"), prefix_hits[0]);
 611     EXPECT_TRUE(cache_hits.empty());
 612
 613     EXPECT_FALSE((database_.get()->*test_case.test_list_contains_bad_url)(
 614         GURL("http://www.evil.com/notevil1.html"), &prefix_hits, &cache_hits));
 615     EXPECT_TRUE(prefix_hits.empty());
 616     EXPECT_TRUE(cache_hits.empty());
 617
 618     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 619         GURL("http://www.evil.com/notevil2.html"), &prefix_hits, &cache_hits));
 620
 621     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 622         GURL("http://www.good.com/good1.html"), &prefix_hits, &cache_hits));
 623
 624     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 625         GURL("http://www.good.com/good2.html"), &prefix_hits, &cache_hits));
 626
 627     GetListsInfo(&lists);
 628     ASSERT_LE(1U, lists.size());
 629     EXPECT_EQ(test_case.test_list_name,
 630               lists[test_case.expected_list_index].name);
 631     EXPECT_EQ("1-3,7", lists[test_case.expected_list_index].adds);
 632     EXPECT_EQ("4", lists[test_case.expected_list_index].subs);
 633
 634     // Test the same sub chunk again.  This should be a no-op.
 635     // see bug: http://code.google.com/p/chromium/issues/detail?id=4522
 636     chunks.clear();
 637     chunks.push_back(SubChunkPrefixValue(4, "www.evil.com/notevil1.html", 2));
 638
 639     ASSERT_TRUE(database_->UpdateStarted(&lists));
 640     database_->InsertChunks(test_case.test_list_name, chunks.get());
 641     database_->UpdateFinished(true);
 642
 643     GetListsInfo(&lists);
 644     ASSERT_LE(1U, lists.size());
 645     EXPECT_EQ(test_case.test_list_name,
 646               lists[test_case.expected_list_index].name);
 647     EXPECT_EQ("1-3,7", lists[test_case.expected_list_index].adds);
 648     EXPECT_EQ("4", lists[test_case.expected_list_index].subs);
 649
 650     // Test removing all the prefixes from an add chunk.
 651     ASSERT_TRUE(database_->UpdateStarted(&lists));
 652     AddDelChunk(test_case.test_list_name, 2);
 653     database_->UpdateFinished(true);
 654
 655     EXPECT_FALSE((database_.get()->*test_case.test_list_contains_bad_url)(
 656         GURL("http://www.evil.com/notevil2.html"), &prefix_hits, &cache_hits));
 657
 658     EXPECT_FALSE((database_.get()->*test_case.test_list_contains_bad_url)(
 659         GURL("http://www.good.com/good1.html"), &prefix_hits, &cache_hits));
 660
 661     EXPECT_FALSE((database_.get()->*test_case.test_list_contains_bad_url)(
 662         GURL("http://www.good.com/good2.html"), &prefix_hits, &cache_hits));
 663
 664     GetListsInfo(&lists);
 665     ASSERT_LE(1U, lists.size());
 666     EXPECT_EQ(test_case.test_list_name,
 667               lists[test_case.expected_list_index].name);
 668     EXPECT_EQ("1,3,7", lists[test_case.expected_list_index].adds);
 669     EXPECT_EQ("4", lists[test_case.expected_list_index].subs);
 670
 671     // The adddel command exposed a bug in the transaction code where any
 672     // transaction after it would fail.  Add a dummy entry and remove it to
 673     // make sure the transcation works fine.
 674     chunks.clear();
 675     chunks.push_back(AddChunkPrefixValue(44, "www.redherring.com/index.html"));
 676     ASSERT_TRUE(database_->UpdateStarted(&lists));
 677     database_->InsertChunks(test_case.test_list_name, chunks.get());
 678
 679     // Now remove the dummy entry.  If there are any problems with the
 680     // transactions, asserts will fire.
 681     AddDelChunk(test_case.test_list_name, 44);
 682
 683     // Test the subdel command.
 684     SubDelChunk(test_case.test_list_name, 4);
 685     database_->UpdateFinished(true);
 686
 687     GetListsInfo(&lists);
 688     ASSERT_LE(1U, lists.size());
 689     EXPECT_EQ(test_case.test_list_name,
 690               lists[test_case.expected_list_index].name);
 691     EXPECT_EQ("1,3,7", lists[test_case.expected_list_index].adds);
 692     EXPECT_TRUE(lists[test_case.expected_list_index].subs.empty());
 693
 694     // Test a sub command coming in before the add.
 695     chunks.clear();
 696     chunks.push_back(SubChunkPrefix2Value(5,
 697                                           "www.notevilanymore.com/index.html",
 698                                           10,
 699                                           "www.notevilanymore.com/good.html",
 700                                           10));
 701     ASSERT_TRUE(database_->UpdateStarted(&lists));
 702     database_->InsertChunks(test_case.test_list_name, chunks.get());
 703     database_->UpdateFinished(true);
 704
 705     EXPECT_FALSE((database_.get()->*test_case.test_list_contains_bad_url)(
 706         GURL("http://www.notevilanymore.com/index.html"),
 707         &prefix_hits,
 708         &cache_hits));
 709
 710     // Now insert the tardy add chunk and we don't expect them to appear
 711     // in database because of the previous sub chunk.
 712     chunks.clear();
 713     chunks.push_back(AddChunkPrefix2Value(10,
 714                                           "www.notevilanymore.com/index.html",
 715                                           "www.notevilanymore.com/good.html"));
 716     ASSERT_TRUE(database_->UpdateStarted(&lists));
 717     database_->InsertChunks(test_case.test_list_name, chunks.get());
 718     database_->UpdateFinished(true);
 719
 720     EXPECT_FALSE((database_.get()->*test_case.test_list_contains_bad_url)(
 721         GURL("http://www.notevilanymore.com/index.html"),
 722         &prefix_hits,
 723         &cache_hits));
 724
 725     EXPECT_FALSE((database_.get()->*test_case.test_list_contains_bad_url)(
 726         GURL("http://www.notevilanymore.com/good.html"),
 727         &prefix_hits,
 728         &cache_hits));
 729
 730     // Reset and reload the database.  The database will rely on the prefix set.
 731     ResetAndReloadFullDatabase();
 732
 733     // Check that a prefix still hits.
 734     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 735         GURL("http://www.evil.com/phishing.html"), &prefix_hits, &cache_hits));
 736     ASSERT_EQ(1U, prefix_hits.size());
 737     EXPECT_EQ(SBPrefixForString("www.evil.com/phishing.html"), prefix_hits[0]);
 738
 739     // Also check that it's not just always returning true in this case.
 740     EXPECT_FALSE((database_.get()->*test_case.test_list_contains_bad_url)(
 741         GURL("http://www.evil.com/"), &prefix_hits, &cache_hits));
 742
 743     // Check that the full hash is still present.
 744     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 745         GURL("http://www.evil.com/evil.html"), &prefix_hits, &cache_hits));
 746     ASSERT_EQ(1U, prefix_hits.size());
 747     EXPECT_EQ(SBPrefixForString("www.evil.com/evil.html"), prefix_hits[0]);
 748   }
 749 }
 750
 751 // Test adding zero length chunks to the database.
 752 TEST_F(SafeBrowsingDatabaseTest, ZeroSizeChunk) {
 753   std::vector<SBListChunkRanges> lists;
 754   ScopedVector<SBChunkData> chunks;
 755
 756   // Populate with a couple of normal chunks.
 757   chunks.push_back(AddChunkPrefix2Value(1,
 758                                         "www.test.com/test1.html",
 759                                         "www.test.com/test2.html"));
 760   chunks.push_back(AddChunkPrefix2Value(10,
 761                                         "www.random.com/random1.html",
 762                                         "www.random.com/random2.html"));
 763
 764   ASSERT_TRUE(database_->UpdateStarted(&lists));
 765   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
 766   database_->UpdateFinished(true);
 767
 768   // Add an empty ADD and SUB chunk.
 769   GetListsInfo(&lists);
 770   ASSERT_LE(1U, lists.size());
 771   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
 772   EXPECT_EQ("1,10", lists[0].adds);
 773   EXPECT_TRUE(lists[0].subs.empty());
 774
 775   chunks.clear();
 776   chunks.push_back(BuildChunk(19, safe_browsing::ChunkData::ADD,
 777                               safe_browsing::ChunkData::PREFIX_4B,
 778                               NULL, 0, std::vector<int>()));
 779   chunks.push_back(BuildChunk(7, safe_browsing::ChunkData::SUB,
 780                               safe_browsing::ChunkData::PREFIX_4B,
 781                               NULL, 0, std::vector<int>()));
 782
 783   ASSERT_TRUE(database_->UpdateStarted(&lists));
 784   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
 785   database_->UpdateFinished(true);
 786
 787   GetListsInfo(&lists);
 788   ASSERT_LE(1U, lists.size());
 789   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
 790   EXPECT_EQ("1,10,19", lists[0].adds);
 791   EXPECT_EQ("7", lists[0].subs);
 792
 793   // Add an empty chunk along with a couple that contain data. This should
 794   // result in the chunk range being reduced in size.
 795   chunks.clear();
 796   chunks.push_back(AddChunkPrefixValue(20, "www.notempty.com/full1.html"));
 797   chunks.push_back(BuildChunk(21, safe_browsing::ChunkData::ADD,
 798                               safe_browsing::ChunkData::PREFIX_4B,
 799                               NULL, 0, std::vector<int>()));
 800   chunks.push_back(AddChunkPrefixValue(22, "www.notempty.com/full2.html"));
 801
 802   ASSERT_TRUE(database_->UpdateStarted(&lists));
 803   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
 804   database_->UpdateFinished(true);
 805
 806   std::vector<SBPrefix> prefix_hits;
 807   std::vector<SBFullHashResult> cache_hits;
 808   EXPECT_TRUE(database_->ContainsBrowseUrl(
 809       GURL("http://www.notempty.com/full1.html"), &prefix_hits, &cache_hits));
 810   EXPECT_TRUE(database_->ContainsBrowseUrl(
 811       GURL("http://www.notempty.com/full2.html"), &prefix_hits, &cache_hits));
 812
 813   GetListsInfo(&lists);
 814   ASSERT_LE(1U, lists.size());
 815   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
 816   EXPECT_EQ("1,10,19-22", lists[0].adds);
 817   EXPECT_EQ("7", lists[0].subs);
 818
 819   // Handle AddDel and SubDel commands for empty chunks.
 820   ASSERT_TRUE(database_->UpdateStarted(&lists));
 821   AddDelChunk(safe_browsing_util::kMalwareList, 21);
 822   database_->UpdateFinished(true);
 823
 824   GetListsInfo(&lists);
 825   ASSERT_LE(1U, lists.size());
 826   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
 827   EXPECT_EQ("1,10,19-20,22", lists[0].adds);
 828   EXPECT_EQ("7", lists[0].subs);
 829
 830   ASSERT_TRUE(database_->UpdateStarted(&lists));
 831   SubDelChunk(safe_browsing_util::kMalwareList, 7);
 832   database_->UpdateFinished(true);
 833
 834   GetListsInfo(&lists);
 835   ASSERT_LE(1U, lists.size());
 836   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
 837   EXPECT_EQ("1,10,19-20,22", lists[0].adds);
 838   EXPECT_TRUE(lists[0].subs.empty());
 839 }
 840
 841 // Utility function for setting up the database for the caching test.
 842 void SafeBrowsingDatabaseTest::PopulateDatabaseForCacheTest() {
 843   // Add a couple prefixes.
 844   ScopedVector<SBChunkData> chunks;
 845   chunks.push_back(AddChunkPrefix2Value(1,
 846                                         "www.evil.com/phishing.html",
 847                                         "www.evil.com/malware.html"));
 848
 849   std::vector<SBListChunkRanges> lists;
 850   ASSERT_TRUE(database_->UpdateStarted(&lists));
 851   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
 852   database_->UpdateFinished(true);
 853
 854   // Cache should be cleared after updating.
 855   EXPECT_TRUE(
 856       database_->GetUnsynchronizedPrefixGetHashCacheForTesting()->empty());
 857
 858   SBFullHashResult full_hash;
 859   full_hash.list_id = safe_browsing_util::MALWARE;
 860
 861   std::vector<SBFullHashResult> results;
 862   std::vector<SBPrefix> prefixes;
 863
 864   // Add a fullhash result for each prefix.
 865   full_hash.hash = SBFullHashForString("www.evil.com/phishing.html");
 866   results.push_back(full_hash);
 867   prefixes.push_back(full_hash.hash.prefix);
 868
 869   full_hash.hash = SBFullHashForString("www.evil.com/malware.html");
 870   results.push_back(full_hash);
 871   prefixes.push_back(full_hash.hash.prefix);
 872
 873   database_->CacheHashResults(prefixes, results, kCacheLifetime);
 874 }
 875
 876 TEST_F(SafeBrowsingDatabaseTest, HashCaching) {
 877   PopulateDatabaseForCacheTest();
 878
 879   // We should have both full hashes in the cache.
 880   EXPECT_EQ(2U,
 881             database_->GetUnsynchronizedPrefixGetHashCacheForTesting()->size());
 882
 883   // Test the cache lookup for the first prefix.
 884   std::vector<SBPrefix> prefix_hits;
 885   std::vector<SBFullHashResult> cache_hits;
 886   EXPECT_TRUE(database_->ContainsBrowseUrl(
 887       GURL("http://www.evil.com/phishing.html"), &prefix_hits, &cache_hits));
 888   EXPECT_TRUE(prefix_hits.empty());
 889   ASSERT_EQ(1U, cache_hits.size());
 890   EXPECT_TRUE(SBFullHashEqual(
 891       cache_hits[0].hash, SBFullHashForString("www.evil.com/phishing.html")));
 892
 893   prefix_hits.clear();
 894   cache_hits.clear();
 895
 896   // Test the cache lookup for the second prefix.
 897   EXPECT_TRUE(database_->ContainsBrowseUrl(
 898       GURL("http://www.evil.com/malware.html"), &prefix_hits, &cache_hits));
 899   EXPECT_TRUE(prefix_hits.empty());
 900   ASSERT_EQ(1U, cache_hits.size());
 901   EXPECT_TRUE(SBFullHashEqual(
 902       cache_hits[0].hash, SBFullHashForString("www.evil.com/malware.html")));
 903
 904   prefix_hits.clear();
 905   cache_hits.clear();
 906
 907   // Test removing a prefix via a sub chunk.
 908   ScopedVector<SBChunkData> chunks;
 909   chunks.push_back(SubChunkPrefixValue(2, "www.evil.com/phishing.html", 1));
 910
 911   std::vector<SBListChunkRanges> lists;
 912   ASSERT_TRUE(database_->UpdateStarted(&lists));
 913   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
 914   database_->UpdateFinished(true);
 915
 916   // This prefix should still be there, but cached fullhash should be gone.
 917   EXPECT_TRUE(database_->ContainsBrowseUrl(
 918       GURL("http://www.evil.com/malware.html"), &prefix_hits, &cache_hits));
 919   ASSERT_EQ(1U, prefix_hits.size());
 920   EXPECT_EQ(SBPrefixForString("www.evil.com/malware.html"), prefix_hits[0]);
 921   EXPECT_TRUE(cache_hits.empty());
 922   prefix_hits.clear();
 923   cache_hits.clear();
 924
 925   // This prefix should be gone.
 926   EXPECT_FALSE(database_->ContainsBrowseUrl(
 927       GURL("http://www.evil.com/phishing.html"), &prefix_hits, &cache_hits));
 928   prefix_hits.clear();
 929   cache_hits.clear();
 930
 931   // Test that an AddDel for the original chunk removes the last cached entry.
 932   ASSERT_TRUE(database_->UpdateStarted(&lists));
 933   AddDelChunk(safe_browsing_util::kMalwareList, 1);
 934   database_->UpdateFinished(true);
 935   EXPECT_FALSE(database_->ContainsBrowseUrl(
 936       GURL("http://www.evil.com/malware.html"), &prefix_hits, &cache_hits));
 937   EXPECT_TRUE(
 938       database_->GetUnsynchronizedPrefixGetHashCacheForTesting()->empty());
 939   prefix_hits.clear();
 940   cache_hits.clear();
 941
 942   // Test that the cache won't return expired values. First we have to adjust
 943   // the cached entries' received time to make them older, since the database
 944   // cache insert uses Time::Now(). First, store some entries.
 945   PopulateDatabaseForCacheTest();
 946
 947   SafeBrowsingDatabaseNew::PrefixGetHashCache* hash_cache =
 948       database_->GetUnsynchronizedPrefixGetHashCacheForTesting();
 949   EXPECT_EQ(2U, hash_cache->size());
 950
 951   // Now adjust one of the entries times to be in the past.
 952   const SBPrefix key = SBPrefixForString("www.evil.com/malware.html");
 953   SafeBrowsingDatabaseNew::PrefixGetHashCache::iterator iter =
 954       hash_cache->find(key);
 955   ASSERT_TRUE(iter != hash_cache->end());
 956   iter->second.expire_after = Time::Now() - TimeDelta::FromMinutes(1);
 957
 958   EXPECT_TRUE(database_->ContainsBrowseUrl(
 959       GURL("http://www.evil.com/malware.html"), &prefix_hits, &cache_hits));
 960   EXPECT_EQ(1U, prefix_hits.size());
 961   EXPECT_TRUE(cache_hits.empty());
 962   // Expired entry should have been removed from cache.
 963   EXPECT_EQ(1U, hash_cache->size());
 964
 965   // This entry should still exist.
 966   EXPECT_TRUE(database_->ContainsBrowseUrl(
 967       GURL("http://www.evil.com/phishing.html"), &prefix_hits, &cache_hits));
 968   EXPECT_TRUE(prefix_hits.empty());
 969   EXPECT_EQ(1U, cache_hits.size());
 970
 971   // Testing prefix miss caching. First, we clear out the existing database,
 972   // Since PopulateDatabaseForCacheTest() doesn't handle adding duplicate
 973   // chunks.
 974   ASSERT_TRUE(database_->UpdateStarted(&lists));
 975   AddDelChunk(safe_browsing_util::kMalwareList, 1);
 976   database_->UpdateFinished(true);
 977
 978   // Cache should be cleared after updating.
 979   EXPECT_TRUE(hash_cache->empty());
 980
 981   std::vector<SBPrefix> prefix_misses;
 982   std::vector<SBFullHashResult> empty_full_hash;
 983   prefix_misses.push_back(SBPrefixForString("http://www.bad.com/malware.html"));
 984   prefix_misses.push_back(
 985       SBPrefixForString("http://www.bad.com/phishing.html"));
 986   database_->CacheHashResults(prefix_misses, empty_full_hash, kCacheLifetime);
 987
 988   // Prefixes with no full results are misses.
 989   EXPECT_EQ(hash_cache->size(), prefix_misses.size());
 990   ASSERT_TRUE(
 991       hash_cache->count(SBPrefixForString("http://www.bad.com/malware.html")));
 992   EXPECT_TRUE(
 993       hash_cache->find(SBPrefixForString("http://www.bad.com/malware.html"))
 994           ->second.full_hashes.empty());
 995   ASSERT_TRUE(
 996       hash_cache->count(SBPrefixForString("http://www.bad.com/phishing.html")));
 997   EXPECT_TRUE(
 998       hash_cache->find(SBPrefixForString("http://www.bad.com/phishing.html"))
 999           ->second.full_hashes.empty());
1000
1001   // Update the database.
1002   PopulateDatabaseForCacheTest();
1003
1004   // Cache a GetHash miss for a particular prefix, and even though the prefix is
1005   // in the database, it is flagged as a miss so looking up the associated URL
1006   // will not succeed.
1007   prefix_hits.clear();
1008   cache_hits.clear();
1009   prefix_misses.clear();
1010   empty_full_hash.clear();
1011   prefix_misses.push_back(SBPrefixForString("www.evil.com/phishing.html"));
1012   database_->CacheHashResults(prefix_misses, empty_full_hash, kCacheLifetime);
1013   EXPECT_FALSE(database_->ContainsBrowseUrl(
1014       GURL("http://www.evil.com/phishing.html"), &prefix_hits, &cache_hits));
1015   prefix_hits.clear();
1016   cache_hits.clear();
1017
1018   // Test receiving a full add chunk.
1019   chunks.clear();
1020   chunks.push_back(AddChunkFullHash2Value(20,
1021                                           "www.fullevil.com/bad1.html",
1022                                           "www.fullevil.com/bad2.html"));
1023   ASSERT_TRUE(database_->UpdateStarted(&lists));
1024   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1025   database_->UpdateFinished(true);
1026
1027   EXPECT_TRUE(database_->ContainsBrowseUrl(
1028       GURL("http://www.fullevil.com/bad1.html"), &prefix_hits, &cache_hits));
1029   ASSERT_EQ(1U, prefix_hits.size());
1030   EXPECT_EQ(SBPrefixForString("www.fullevil.com/bad1.html"), prefix_hits[0]);
1031   EXPECT_TRUE(cache_hits.empty());
1032   prefix_hits.clear();
1033   cache_hits.clear();
1034
1035   EXPECT_TRUE(database_->ContainsBrowseUrl(
1036       GURL("http://www.fullevil.com/bad2.html"), &prefix_hits, &cache_hits));
1037   ASSERT_EQ(1U, prefix_hits.size());
1038   EXPECT_EQ(SBPrefixForString("www.fullevil.com/bad2.html"), prefix_hits[0]);
1039   EXPECT_TRUE(cache_hits.empty());
1040   prefix_hits.clear();
1041   cache_hits.clear();
1042
1043   // Test receiving a full sub chunk, which will remove one of the full adds.
1044   chunks.clear();
1045   chunks.push_back(SubChunkFullHashValue(200,
1046                                          "www.fullevil.com/bad1.html",
1047                                          20));
1048   ASSERT_TRUE(database_->UpdateStarted(&lists));
1049   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1050   database_->UpdateFinished(true);
1051
1052   EXPECT_FALSE(database_->ContainsBrowseUrl(
1053       GURL("http://www.fullevil.com/bad1.html"), &prefix_hits, &cache_hits));
1054
1055   // There should be one remaining full add.
1056   EXPECT_TRUE(database_->ContainsBrowseUrl(
1057       GURL("http://www.fullevil.com/bad2.html"), &prefix_hits, &cache_hits));
1058   ASSERT_EQ(1U, prefix_hits.size());
1059   EXPECT_EQ(SBPrefixForString("www.fullevil.com/bad2.html"), prefix_hits[0]);
1060   EXPECT_TRUE(cache_hits.empty());
1061   prefix_hits.clear();
1062   cache_hits.clear();
1063
1064   // Now test an AddDel for the remaining full add.
1065   ASSERT_TRUE(database_->UpdateStarted(&lists));
1066   AddDelChunk(safe_browsing_util::kMalwareList, 20);
1067   database_->UpdateFinished(true);
1068
1069   EXPECT_FALSE(database_->ContainsBrowseUrl(
1070       GURL("http://www.fullevil.com/bad1.html"), &prefix_hits, &cache_hits));
1071   EXPECT_FALSE(database_->ContainsBrowseUrl(
1072       GURL("http://www.fullevil.com/bad2.html"), &prefix_hits, &cache_hits));
1073
1074   // Add a fullhash which has a prefix collision for a known url.
1075   static const char kExampleFine[] = "www.example.com/fine.html";
1076   static const char kExampleCollision[] =
1077       "www.example.com/3123364814/malware.htm";
1078   ASSERT_EQ(SBPrefixForString(kExampleFine),
1079             SBPrefixForString(kExampleCollision));
1080   ASSERT_TRUE(database_->UpdateStarted(&lists));
1081   {
1082     ScopedVector<SBChunkData> chunks;
1083     chunks.push_back(AddChunkPrefixValue(21, kExampleCollision));
1084     database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1085   }
1086   database_->UpdateFinished(true);
1087
1088   // Expect a prefix hit due to the collision between |kExampleFine| and
1089   // |kExampleCollision|.
1090   EXPECT_TRUE(database_->ContainsBrowseUrl(
1091       GURL(std::string("http://") + kExampleFine), &prefix_hits, &cache_hits));
1092   ASSERT_EQ(1U, prefix_hits.size());
1093   EXPECT_EQ(SBPrefixForString(kExampleFine), prefix_hits[0]);
1094   EXPECT_TRUE(cache_hits.empty());
1095
1096   // Cache gethash response for |kExampleCollision|.
1097   {
1098     SBFullHashResult result;
1099     result.hash = SBFullHashForString(kExampleCollision);
1100     result.list_id = safe_browsing_util::MALWARE;
1101     database_->CacheHashResults(std::vector<SBPrefix>(1, result.hash.prefix),
1102                                 std::vector<SBFullHashResult>(1, result),
1103                                 kCacheLifetime);
1104   }
1105
1106   // The cached response means the collision no longer causes a hit.
1107   EXPECT_FALSE(database_->ContainsBrowseUrl(
1108       GURL(std::string("http://") + kExampleFine), &prefix_hits, &cache_hits));
1109 }
1110
1111 // Test that corrupt databases are appropriately handled, even if the
1112 // corruption is detected in the midst of the update.
1113 // TODO(shess): Disabled until ScopedLogMessageIgnorer resolved.
1114 // http://crbug.com/56448
1115 TEST_F(SafeBrowsingDatabaseTest, DISABLED_FileCorruptionHandling) {
1116   // Re-create the database in a captive message loop so that we can
1117   // influence task-posting.  Database specifically needs to the
1118   // file-backed.
1119   database_.reset();
1120   base::MessageLoop loop;
1121   SafeBrowsingStoreFile* store = new SafeBrowsingStoreFile(task_runner_);
1122   database_.reset(new SafeBrowsingDatabaseNew(
1123       task_runner_, store, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL));
1124   database_->Init(database_filename_);
1125
1126   // This will cause an empty database to be created.
1127   std::vector<SBListChunkRanges> lists;
1128   ASSERT_TRUE(database_->UpdateStarted(&lists));
1129   database_->UpdateFinished(true);
1130
1131   // Create a sub chunk to insert.
1132   ScopedVector<SBChunkData> chunks;
1133   chunks.push_back(SubChunkPrefixValue(7,
1134                                        "www.subbed.com/notevil1.html",
1135                                        19));
1136
1137   // Corrupt the file by corrupting the checksum, which is not checked
1138   // until the entire table is read in |UpdateFinished()|.
1139   FILE* fp = base::OpenFile(database_filename_, "r+");
1140   ASSERT_TRUE(fp);
1141   ASSERT_NE(-1, fseek(fp, -8, SEEK_END));
1142   for (size_t i = 0; i < 8; ++i) {
1143     fputc('!', fp);
1144   }
1145   fclose(fp);
1146
1147   {
1148     // The following code will cause DCHECKs, so suppress the crashes.
1149     ScopedLogMessageIgnorer ignorer;
1150
1151     // Start an update.  The insert will fail due to corruption.
1152     ASSERT_TRUE(database_->UpdateStarted(&lists));
1153     database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1154     database_->UpdateFinished(true);
1155
1156     // Database file still exists until the corruption handler has run.
1157     EXPECT_TRUE(base::PathExists(database_filename_));
1158
1159     // Flush through the corruption-handler task.
1160     DVLOG(1) << "Expect failed check on: SafeBrowsing database reset";
1161     base::MessageLoop::current()->RunUntilIdle();
1162   }
1163
1164   // Database file should not exist.
1165   EXPECT_FALSE(base::PathExists(database_filename_));
1166
1167   // Run the update again successfully.
1168   ASSERT_TRUE(database_->UpdateStarted(&lists));
1169   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1170   database_->UpdateFinished(true);
1171   EXPECT_TRUE(base::PathExists(database_filename_));
1172
1173   database_.reset();
1174 }
1175
1176 // Checks database reading and writing.
1177 TEST_F(SafeBrowsingDatabaseTest, ContainsDownloadUrl) {
1178   const char kEvil1Url1[] = "www.evil1.com/download1/";
1179   const char kEvil1Url2[] = "www.evil1.com/download2.html";
1180
1181   // Add a simple chunk with one hostkey for download url list.
1182   ScopedVector<SBChunkData> chunks;
1183   chunks.push_back(AddChunkPrefix2Value(1, kEvil1Url1, kEvil1Url2));
1184
1185   std::vector<SBListChunkRanges> lists;
1186   ASSERT_TRUE(database_->UpdateStarted(&lists));
1187   database_->InsertChunks(safe_browsing_util::kBinUrlList, chunks.get());
1188   database_->UpdateFinished(true);
1189
1190   std::vector<SBPrefix> prefix_hits;
1191   std::vector<GURL> urls(1);
1192
1193   urls[0] = GURL(std::string("http://") + kEvil1Url1);
1194   EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1195   ASSERT_EQ(1U, prefix_hits.size());
1196   EXPECT_EQ(SBPrefixForString(kEvil1Url1), prefix_hits[0]);
1197
1198   urls[0] = GURL(std::string("http://") + kEvil1Url2);
1199   EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1200   ASSERT_EQ(1U, prefix_hits.size());
1201   EXPECT_EQ(SBPrefixForString(kEvil1Url2), prefix_hits[0]);
1202
1203   urls[0] = GURL(std::string("https://") + kEvil1Url2);
1204   EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1205   ASSERT_EQ(1U, prefix_hits.size());
1206   EXPECT_EQ(SBPrefixForString(kEvil1Url2), prefix_hits[0]);
1207
1208   urls[0] = GURL(std::string("ftp://") + kEvil1Url2);
1209   EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1210   ASSERT_EQ(1U, prefix_hits.size());
1211   EXPECT_EQ(SBPrefixForString(kEvil1Url2), prefix_hits[0]);
1212
1213   urls[0] = GURL("http://www.randomevil.com");
1214   EXPECT_FALSE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1215
1216   // Should match with query args stripped.
1217   urls[0] = GURL(std::string("http://") + kEvil1Url2 + "?blah");
1218   EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1219   ASSERT_EQ(1U, prefix_hits.size());
1220   EXPECT_EQ(SBPrefixForString(kEvil1Url2), prefix_hits[0]);
1221
1222   // Should match with extra path stuff and query args stripped.
1223   urls[0] = GURL(std::string("http://") + kEvil1Url1 + "foo/bar?blah");
1224   EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1225   ASSERT_EQ(1U, prefix_hits.size());
1226   EXPECT_EQ(SBPrefixForString(kEvil1Url1), prefix_hits[0]);
1227
1228   // First hit in redirect chain is malware.
1229   urls.clear();
1230   urls.push_back(GURL(std::string("http://") + kEvil1Url1));
1231   urls.push_back(GURL("http://www.randomevil.com"));
1232   EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1233   ASSERT_EQ(1U, prefix_hits.size());
1234   EXPECT_EQ(SBPrefixForString(kEvil1Url1), prefix_hits[0]);
1235
1236   // Middle hit in redirect chain is malware.
1237   urls.clear();
1238   urls.push_back(GURL("http://www.randomevil.com"));
1239   urls.push_back(GURL(std::string("http://") + kEvil1Url1));
1240   urls.push_back(GURL("http://www.randomevil2.com"));
1241   EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1242   ASSERT_EQ(1U, prefix_hits.size());
1243   EXPECT_EQ(SBPrefixForString(kEvil1Url1), prefix_hits[0]);
1244
1245   // Final hit in redirect chain is malware.
1246   urls.clear();
1247   urls.push_back(GURL("http://www.randomevil.com"));
1248   urls.push_back(GURL(std::string("http://") + kEvil1Url1));
1249   EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1250   ASSERT_EQ(1U, prefix_hits.size());
1251   EXPECT_EQ(SBPrefixForString(kEvil1Url1), prefix_hits[0]);
1252
1253   // Multiple hits in redirect chain are in malware list.
1254   urls.clear();
1255   urls.push_back(GURL(std::string("http://") + kEvil1Url1));
1256   urls.push_back(GURL(std::string("https://") + kEvil1Url2));
1257   EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1258   ASSERT_EQ(2U, prefix_hits.size());
1259   EXPECT_EQ(SBPrefixForString(kEvil1Url1), prefix_hits[0]);
1260   EXPECT_EQ(SBPrefixForString(kEvil1Url2), prefix_hits[1]);
1261   database_.reset();
1262 }
1263
1264 // Checks that the whitelists are handled properly.
1265 TEST_F(SafeBrowsingDatabaseTest, Whitelists) {
1266   struct TestCase {
1267     using TestListContainsWhitelistedUrl =
1268         bool (SafeBrowsingDatabase::*)(const GURL& url);
1269     using TestListContainsWhitelistedString =
1270         bool (SafeBrowsingDatabase::*)(const std::string& str);
1271
1272     // Returns true if strings should be tested in this test case (i.e.
1273     // |test_list_contains_whitelisted_string| is not null).
1274     bool TestStrings() const {
1275       return test_list_contains_whitelisted_string != nullptr;
1276     }
1277
1278     const char* test_list_name;
1279     TestListContainsWhitelistedUrl test_list_contains_whitelisted_url;
1280     // Optional test case field, if set the tested whitelist will also be tested
1281     // for strings.
1282     TestListContainsWhitelistedString test_list_contains_whitelisted_string;
1283   } const kTestCases[]{
1284       {safe_browsing_util::kCsdWhiteList,
1285        &SafeBrowsingDatabase::ContainsCsdWhitelistedUrl,
1286        nullptr},
1287       {safe_browsing_util::kDownloadWhiteList,
1288        &SafeBrowsingDatabase::ContainsDownloadWhitelistedUrl,
1289        &SafeBrowsingDatabase::ContainsDownloadWhitelistedString},
1290       {safe_browsing_util::kInclusionWhitelist,
1291        &SafeBrowsingDatabase::ContainsInclusionWhitelistedUrl,
1292        nullptr},
1293   };
1294
1295   // If the whitelist is disabled everything should match the whitelist.
1296   database_.reset(new SafeBrowsingDatabaseNew(
1297       task_runner_, new SafeBrowsingStoreFile(task_runner_), NULL, NULL, NULL,
1298       NULL, NULL, NULL, NULL, NULL));
1299   database_->Init(database_filename_);
1300   for (const auto& test_case : kTestCases) {
1301     SCOPED_TRACE(std::string("Tested list at fault => ") +
1302                  test_case.test_list_name);
1303
1304     EXPECT_TRUE(
1305         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1306             GURL(std::string("http://www.phishing.com/"))));
1307     if (test_case.TestStrings()) {
1308       EXPECT_TRUE(
1309           (database_.get()->*test_case.test_list_contains_whitelisted_string)(
1310               "asdf"));
1311     }
1312   }
1313   ResetAndReloadFullDatabase();
1314
1315   // Now test every whitelist one-by-one; intentionally not resetting the
1316   // database in-between to further stress potential inter-dependencies.
1317   for (const auto& test_case : kTestCases) {
1318     SCOPED_TRACE(std::string("Tested list at fault => ") +
1319                  test_case.test_list_name);
1320
1321     const char kGood1Host[] = "www.good1.com/";
1322     const char kGood1Url1[] = "www.good1.com/a/b.html";
1323     const char kGood1Url2[] = "www.good1.com/b/";
1324
1325     const char kGood2Url1[] = "www.good2.com/c";  // Should match '/c/bla'.
1326
1327     // good3.com/a/b/c/d/e/f/g/ should match because it's a whitelist.
1328     const char kGood3Url1[] = "good3.com/";
1329
1330     const char kGoodString[] = "good_string";
1331
1332     // Nothing should be whitelisted before the database receives the chunks.
1333     EXPECT_FALSE(
1334         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1335             GURL(std::string("http://") + kGood1Host)));
1336     EXPECT_FALSE(
1337         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1338             GURL(std::string("http://") + kGood1Url1)));
1339     EXPECT_FALSE(
1340         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1341             GURL(std::string("http://") + kGood1Url2)));
1342     EXPECT_FALSE(
1343         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1344             GURL(std::string("http://") + kGood2Url1)));
1345     EXPECT_FALSE(
1346         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1347             GURL(std::string("http://") + kGood3Url1)));
1348     if (test_case.TestStrings()) {
1349       EXPECT_FALSE(
1350           (database_.get()->*test_case.test_list_contains_whitelisted_string)(
1351               kGoodString));
1352     }
1353
1354     ScopedVector<SBChunkData> chunks;
1355
1356     // Add a few test chunks to the whitelist under test.
1357     chunks.push_back(AddChunkFullHash2Value(1, kGood1Url1, kGood1Url2));
1358     chunks.push_back(AddChunkFullHashValue(2, kGood2Url1));
1359     if (test_case.TestStrings())
1360       chunks.push_back(AddChunkFullHashValue(3, kGoodString));
1361     chunks.push_back(AddChunkFullHashValue(4, kGood3Url1));
1362
1363     std::vector<SBListChunkRanges> lists;
1364     ASSERT_TRUE(database_->UpdateStarted(&lists));
1365     database_->InsertChunks(test_case.test_list_name, chunks.get());
1366     database_->UpdateFinished(true);
1367
1368     EXPECT_FALSE(
1369         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1370             GURL(std::string("http://") + kGood1Host)));
1371
1372     EXPECT_TRUE(
1373         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1374             GURL(std::string("http://") + kGood1Url1)));
1375     EXPECT_TRUE(
1376         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1377             GURL(std::string("http://") + kGood1Url1 + "?a=b")));
1378
1379     EXPECT_TRUE(
1380         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1381             GURL(std::string("http://") + kGood1Url2)));
1382     EXPECT_TRUE(
1383         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1384             GURL(std::string("http://") + kGood1Url2 + "/c.html")));
1385
1386     EXPECT_TRUE(
1387         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1388             GURL(std::string("https://") + kGood1Url2 + "/c.html")));
1389
1390     EXPECT_TRUE(
1391         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1392             GURL(std::string("http://") + kGood2Url1 + "/c")));
1393     EXPECT_TRUE(
1394         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1395             GURL(std::string("http://") + kGood2Url1 + "/c?bla")));
1396     EXPECT_TRUE(
1397         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1398             GURL(std::string("http://") + kGood2Url1 + "/c/bla")));
1399
1400     EXPECT_FALSE(
1401         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1402             GURL(std::string("http://www.google.com/"))));
1403
1404     EXPECT_TRUE(
1405         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1406             GURL(std::string("http://") + kGood3Url1 + "a/b/c/d/e/f/g/")));
1407     EXPECT_TRUE(
1408         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1409             GURL(std::string("http://a.b.") + kGood3Url1)));
1410
1411     if (test_case.TestStrings()) {
1412       EXPECT_FALSE(
1413           (database_.get()->*test_case.test_list_contains_whitelisted_string)(
1414               "asdf"));
1415       EXPECT_TRUE(
1416           (database_.get()->*test_case.test_list_contains_whitelisted_string)(
1417               kGoodString));
1418     }
1419
1420     EXPECT_FALSE(
1421         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1422             GURL(std::string("http://www.google.com/"))));
1423
1424     // The malware kill switch is for the CSD whitelist only.
1425     if (test_case.test_list_name == safe_browsing_util::kCsdWhiteList) {
1426       // The CSD whitelist killswitch is not present.
1427       EXPECT_FALSE(database_->IsCsdWhitelistKillSwitchOn());
1428
1429       // Test only add the malware IP killswitch
1430       chunks.clear();
1431       chunks.push_back(AddChunkFullHashValue(
1432           15, "sb-ssl.google.com/safebrowsing/csd/killswitch_malware"));
1433
1434       ASSERT_TRUE(database_->UpdateStarted(&lists));
1435       database_->InsertChunks(safe_browsing_util::kCsdWhiteList, chunks.get());
1436       database_->UpdateFinished(true);
1437
1438       EXPECT_TRUE(database_->IsMalwareIPMatchKillSwitchOn());
1439       // The CSD whitelist killswitch is not present.
1440       EXPECT_FALSE(database_->IsCsdWhitelistKillSwitchOn());
1441     }
1442
1443     // Test that the generic whitelist kill-switch works as intended.
1444     chunks.clear();
1445     lists.clear();
1446     chunks.push_back(AddChunkFullHashValue(
1447         5, "sb-ssl.google.com/safebrowsing/csd/killswitch"));
1448
1449     ASSERT_TRUE(database_->UpdateStarted(&lists));
1450     database_->InsertChunks(test_case.test_list_name, chunks.get());
1451     database_->UpdateFinished(true);
1452
1453     // Test CSD whitelist specific methods.
1454     if (test_case.test_list_name == safe_browsing_util::kCsdWhiteList) {
1455       // The CSD whitelist killswitch is present.
1456       EXPECT_TRUE(database_->IsCsdWhitelistKillSwitchOn());
1457       EXPECT_TRUE(database_->IsMalwareIPMatchKillSwitchOn());
1458     }
1459
1460     EXPECT_TRUE(
1461         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1462             GURL(std::string("https://") + kGood1Url2 + "/c.html")));
1463     EXPECT_TRUE(
1464         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1465             GURL(std::string("http://www.google.com/"))));
1466     EXPECT_TRUE(
1467         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1468             GURL(std::string("http://www.phishing_url.com/"))));
1469
1470     if (test_case.TestStrings()) {
1471       EXPECT_TRUE(
1472           (database_.get()->*test_case.test_list_contains_whitelisted_string)(
1473               "asdf"));
1474       EXPECT_TRUE(
1475           (database_.get()->*test_case.test_list_contains_whitelisted_string)(
1476               kGoodString));
1477     }
1478
1479     // Remove the kill-switch and verify that we can recover.
1480     chunks.clear();
1481     lists.clear();
1482     chunks.push_back(SubChunkFullHashValue(
1483         1, "sb-ssl.google.com/safebrowsing/csd/killswitch", 5));
1484     if (test_case.test_list_name == safe_browsing_util::kCsdWhiteList) {
1485       chunks.push_back(SubChunkFullHashValue(
1486           10, "sb-ssl.google.com/safebrowsing/csd/killswitch_malware", 15));
1487     }
1488
1489     ASSERT_TRUE(database_->UpdateStarted(&lists));
1490     database_->InsertChunks(test_case.test_list_name, chunks.get());
1491     database_->UpdateFinished(true);
1492
1493     if (test_case.test_list_name == safe_browsing_util::kCsdWhiteList) {
1494       EXPECT_FALSE(database_->IsMalwareIPMatchKillSwitchOn());
1495       EXPECT_FALSE(database_->IsCsdWhitelistKillSwitchOn());
1496     }
1497     EXPECT_TRUE(
1498         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1499             GURL(std::string("https://") + kGood1Url2 + "/c.html")));
1500     EXPECT_TRUE(
1501         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1502             GURL(std::string("https://") + kGood2Url1 + "/c/bla")));
1503     EXPECT_TRUE(
1504         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1505             GURL(std::string("https://") + kGood3Url1)));
1506     EXPECT_FALSE(
1507         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1508             GURL(std::string("http://www.google.com/"))));
1509     EXPECT_FALSE(
1510         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1511             GURL(std::string("http://www.phishing_url.com/"))));
1512     if (test_case.TestStrings()) {
1513       EXPECT_TRUE(
1514           (database_.get()->*test_case.test_list_contains_whitelisted_string)(
1515               kGoodString));
1516       EXPECT_FALSE(
1517           (database_.get()->*test_case.test_list_contains_whitelisted_string)(
1518               "asdf"));
1519     }
1520   }
1521 }
1522
1523 // Test to make sure we could insert chunk list that
1524 // contains entries for the same host.
1525 TEST_F(SafeBrowsingDatabaseTest, SameHostEntriesOkay) {
1526   ScopedVector<SBChunkData> chunks;
1527
1528   // Add a malware add chunk with two entries of the same host.
1529   chunks.push_back(AddChunkPrefix2Value(1,
1530                                         "www.evil.com/malware1.html",
1531                                         "www.evil.com/malware2.html"));
1532
1533   // Insert the testing chunks into database.
1534   std::vector<SBListChunkRanges> lists;
1535   ASSERT_TRUE(database_->UpdateStarted(&lists));
1536   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1537   database_->UpdateFinished(true);
1538
1539   GetListsInfo(&lists);
1540   ASSERT_LE(1U, lists.size());
1541   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
1542   EXPECT_EQ("1", lists[0].adds);
1543   EXPECT_TRUE(lists[0].subs.empty());
1544
1545   // Add a phishing add chunk with two entries of the same host.
1546   chunks.clear();
1547   chunks.push_back(AddChunkPrefix2Value(47,
1548                                         "www.evil.com/phishing1.html",
1549                                         "www.evil.com/phishing2.html"));
1550
1551   ASSERT_TRUE(database_->UpdateStarted(&lists));
1552   database_->InsertChunks(safe_browsing_util::kPhishingList, chunks.get());
1553   database_->UpdateFinished(true);
1554
1555   GetListsInfo(&lists);
1556   ASSERT_LE(2U, lists.size());
1557   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
1558   EXPECT_EQ("1", lists[0].adds);
1559   EXPECT_TRUE(lists[0].subs.empty());
1560   EXPECT_EQ(safe_browsing_util::kPhishingList, lists[1].name);
1561   EXPECT_EQ("47", lists[1].adds);
1562   EXPECT_TRUE(lists[1].subs.empty());
1563
1564   std::vector<SBPrefix> prefix_hits;
1565   std::vector<SBFullHashResult> cache_hits;
1566
1567   EXPECT_TRUE(database_->ContainsBrowseUrl(
1568       GURL("http://www.evil.com/malware1.html"), &prefix_hits, &cache_hits));
1569   EXPECT_TRUE(database_->ContainsBrowseUrl(
1570       GURL("http://www.evil.com/malware2.html"), &prefix_hits, &cache_hits));
1571   EXPECT_TRUE(database_->ContainsBrowseUrl(
1572       GURL("http://www.evil.com/phishing1.html"), &prefix_hits, &cache_hits));
1573   EXPECT_TRUE(database_->ContainsBrowseUrl(
1574       GURL("http://www.evil.com/phishing2.html"), &prefix_hits, &cache_hits));
1575
1576   // Test removing a single prefix from the add chunk.
1577   // Remove the prefix that added first.
1578   chunks.clear();
1579   chunks.push_back(SubChunkPrefixValue(4, "www.evil.com/malware1.html", 1));
1580   ASSERT_TRUE(database_->UpdateStarted(&lists));
1581   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1582   database_->UpdateFinished(true);
1583
1584   // Remove the prefix that added last.
1585   chunks.clear();
1586   chunks.push_back(SubChunkPrefixValue(5, "www.evil.com/phishing2.html", 47));
1587   ASSERT_TRUE(database_->UpdateStarted(&lists));
1588   database_->InsertChunks(safe_browsing_util::kPhishingList, chunks.get());
1589   database_->UpdateFinished(true);
1590
1591   // Verify that the database contains urls expected.
1592   EXPECT_FALSE(database_->ContainsBrowseUrl(
1593       GURL("http://www.evil.com/malware1.html"), &prefix_hits, &cache_hits));
1594   EXPECT_TRUE(database_->ContainsBrowseUrl(
1595       GURL("http://www.evil.com/malware2.html"), &prefix_hits, &cache_hits));
1596   EXPECT_TRUE(database_->ContainsBrowseUrl(
1597       GURL("http://www.evil.com/phishing1.html"), &prefix_hits, &cache_hits));
1598   EXPECT_FALSE(database_->ContainsBrowseUrl(
1599       GURL("http://www.evil.com/phishing2.html"), &prefix_hits, &cache_hits));
1600 }
1601
1602 // Test that an empty update doesn't actually update the database.
1603 // This isn't a functionality requirement, but it is a useful
1604 // optimization.
1605 TEST_F(SafeBrowsingDatabaseTest, EmptyUpdate) {
1606   ScopedVector<SBChunkData> chunks;
1607
1608   base::FilePath filename = database_->BrowseDBFilename(database_filename_);
1609
1610   // Prime the database.
1611   std::vector<SBListChunkRanges> lists;
1612   ASSERT_TRUE(database_->UpdateStarted(&lists));
1613   chunks.push_back(AddChunkPrefixValue(1, "www.evil.com/malware.html"));
1614   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1615   database_->UpdateFinished(true);
1616
1617   // Get an older time to reset the lastmod time for detecting whether
1618   // the file has been updated.
1619   base::File::Info before_info, after_info;
1620   ASSERT_TRUE(base::GetFileInfo(filename, &before_info));
1621   const Time old_last_modified =
1622       before_info.last_modified - TimeDelta::FromSeconds(10);
1623
1624   // Inserting another chunk updates the database file.  The sleep is
1625   // needed because otherwise the entire test can finish w/in the
1626   // resolution of the lastmod time.
1627   ASSERT_TRUE(base::TouchFile(filename, old_last_modified, old_last_modified));
1628   ASSERT_TRUE(base::GetFileInfo(filename, &before_info));
1629   ASSERT_TRUE(database_->UpdateStarted(&lists));
1630   chunks.push_back(AddChunkPrefixValue(2, "www.foo.com/malware.html"));
1631   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1632   database_->UpdateFinished(true);
1633   ASSERT_TRUE(base::GetFileInfo(filename, &after_info));
1634   EXPECT_LT(before_info.last_modified, after_info.last_modified);
1635
1636   // Deleting a chunk updates the database file.
1637   ASSERT_TRUE(base::TouchFile(filename, old_last_modified, old_last_modified));
1638   ASSERT_TRUE(base::GetFileInfo(filename, &before_info));
1639   ASSERT_TRUE(database_->UpdateStarted(&lists));
1640   AddDelChunk(safe_browsing_util::kMalwareList, 2);
1641   database_->UpdateFinished(true);
1642   ASSERT_TRUE(base::GetFileInfo(filename, &after_info));
1643   EXPECT_LT(before_info.last_modified, after_info.last_modified);
1644
1645   // Simply calling |UpdateStarted()| then |UpdateFinished()| does not
1646   // update the database file.
1647   ASSERT_TRUE(base::TouchFile(filename, old_last_modified, old_last_modified));
1648   ASSERT_TRUE(base::GetFileInfo(filename, &before_info));
1649   ASSERT_TRUE(database_->UpdateStarted(&lists));
1650   database_->UpdateFinished(true);
1651   ASSERT_TRUE(base::GetFileInfo(filename, &after_info));
1652   EXPECT_EQ(before_info.last_modified, after_info.last_modified);
1653 }
1654
1655 // Test that a filter file is written out during update and read back
1656 // in during setup.
1657 TEST_F(SafeBrowsingDatabaseTest, FilterFile) {
1658   // Create a database with trivial example data and write it out.
1659   {
1660     // Prime the database.
1661     std::vector<SBListChunkRanges> lists;
1662     ASSERT_TRUE(database_->UpdateStarted(&lists));
1663
1664     ScopedVector<SBChunkData> chunks;
1665     chunks.push_back(AddChunkPrefixValue(1, "www.evil.com/malware.html"));
1666     database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1667     database_->UpdateFinished(true);
1668   }
1669
1670   // Find the malware url in the database, don't find a good url.
1671   std::vector<SBPrefix> prefix_hits;
1672   std::vector<SBFullHashResult> cache_hits;
1673   EXPECT_TRUE(database_->ContainsBrowseUrl(
1674       GURL("http://www.evil.com/malware.html"), &prefix_hits, &cache_hits));
1675   EXPECT_FALSE(database_->ContainsBrowseUrl(
1676       GURL("http://www.good.com/goodware.html"), &prefix_hits, &cache_hits));
1677
1678   base::FilePath filter_file = database_->PrefixSetForFilename(
1679       database_->BrowseDBFilename(database_filename_));
1680
1681   // After re-creating the database, it should have a filter read from
1682   // a file, so it should find the same results.
1683   ASSERT_TRUE(base::PathExists(filter_file));
1684   ResetAndReloadFullDatabase();
1685   EXPECT_TRUE(database_->ContainsBrowseUrl(
1686       GURL("http://www.evil.com/malware.html"), &prefix_hits, &cache_hits));
1687   EXPECT_FALSE(database_->ContainsBrowseUrl(
1688       GURL("http://www.good.com/goodware.html"), &prefix_hits, &cache_hits));
1689
1690   // If there is no filter file, the database cannot find malware urls.
1691   base::DeleteFile(filter_file, false);
1692   ASSERT_FALSE(base::PathExists(filter_file));
1693   ResetAndReloadFullDatabase();
1694   EXPECT_FALSE(database_->ContainsBrowseUrl(
1695       GURL("http://www.evil.com/malware.html"), &prefix_hits, &cache_hits));
1696   EXPECT_FALSE(database_->ContainsBrowseUrl(
1697       GURL("http://www.good.com/goodware.html"), &prefix_hits, &cache_hits));
1698 }
1699
1700 TEST_F(SafeBrowsingDatabaseTest, CachedFullMiss) {
1701   const SBPrefix kPrefix1 = 1001U;
1702   const SBFullHash kFullHash1_1 =
1703       SBFullHashForPrefixAndSuffix(kPrefix1, "\x01");
1704
1705   const SBPrefix kPrefix2 = 1002U;
1706   const SBFullHash kFullHash2_1 =
1707       SBFullHashForPrefixAndSuffix(kPrefix2, "\x01");
1708
1709   // Insert prefix kPrefix1 and kPrefix2 into database.
1710   ScopedVector<SBChunkData> chunks;
1711   chunks.push_back(AddChunkPrefix(1, kPrefix1));
1712   chunks.push_back(AddChunkPrefix(2, kPrefix2));
1713
1714   std::vector<SBListChunkRanges> lists;
1715   ASSERT_TRUE(database_->UpdateStarted(&lists));
1716   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1717   database_->UpdateFinished(true);
1718
1719   {
1720     // Cache a full miss result for kPrefix1.
1721     std::vector<SBPrefix> prefixes(1, kPrefix1);
1722     std::vector<SBFullHashResult> cache_results;
1723     database_->CacheHashResults(prefixes, cache_results, kCacheLifetime);
1724   }
1725
1726   {
1727     // kFullHash1_1 gets no prefix hit because of the cached item, and also does
1728     // not have a cache hit.
1729     std::vector<SBFullHash> full_hashes(1, kFullHash1_1);
1730     std::vector<SBPrefix> prefix_hits;
1731     std::vector<SBFullHashResult> cache_hits;
1732     EXPECT_FALSE(database_->ContainsBrowseUrlHashesForTesting(
1733         full_hashes, &prefix_hits, &cache_hits));
1734
1735     // kFullHash2_1 gets a hit from the prefix in the database.
1736     full_hashes.push_back(kFullHash2_1);
1737     prefix_hits.clear();
1738     cache_hits.clear();
1739     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
1740         full_hashes, &prefix_hits, &cache_hits));
1741     ASSERT_EQ(1U, prefix_hits.size());
1742     EXPECT_EQ(kPrefix2, prefix_hits[0]);
1743     EXPECT_TRUE(cache_hits.empty());
1744   }
1745 }
1746
1747 TEST_F(SafeBrowsingDatabaseTest, CachedPrefixHitFullMiss) {
1748   const SBPrefix kPrefix1 = 1001U;
1749   const SBFullHash kFullHash1_1 =
1750       SBFullHashForPrefixAndSuffix(kPrefix1, "\x01");
1751   const SBFullHash kFullHash1_2 =
1752       SBFullHashForPrefixAndSuffix(kPrefix1, "\x02");
1753   const SBFullHash kFullHash1_3 =
1754       SBFullHashForPrefixAndSuffix(kPrefix1, "\x03");
1755
1756   const SBPrefix kPrefix2 = 1002U;
1757   const SBFullHash kFullHash2_1 =
1758       SBFullHashForPrefixAndSuffix(kPrefix2, "\x01");
1759
1760   const SBPrefix kPrefix3 = 1003U;
1761   const SBFullHash kFullHash3_1 =
1762       SBFullHashForPrefixAndSuffix(kPrefix3, "\x01");
1763
1764   // Insert prefix kPrefix1 and kPrefix2 into database.
1765   ScopedVector<SBChunkData> chunks;
1766   chunks.push_back(AddChunkPrefix(1, kPrefix1));
1767   chunks.push_back(AddChunkPrefix(2, kPrefix2));
1768
1769   std::vector<SBListChunkRanges> lists;
1770   ASSERT_TRUE(database_->UpdateStarted(&lists));
1771   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1772   database_->UpdateFinished(true);
1773
1774   {
1775     // kFullHash1_1 has a prefix hit of kPrefix1.
1776     std::vector<SBFullHash> full_hashes;
1777     full_hashes.push_back(kFullHash1_1);
1778     std::vector<SBPrefix> prefix_hits;
1779     std::vector<SBFullHashResult> cache_hits;
1780     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
1781         full_hashes, &prefix_hits, &cache_hits));
1782     ASSERT_EQ(1U, prefix_hits.size());
1783     EXPECT_EQ(kPrefix1, prefix_hits[0]);
1784     EXPECT_TRUE(cache_hits.empty());
1785
1786     // kFullHash2_1 has a prefix hit of kPrefix2.
1787     full_hashes.push_back(kFullHash2_1);
1788     prefix_hits.clear();
1789     cache_hits.clear();
1790     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
1791         full_hashes, &prefix_hits, &cache_hits));
1792     ASSERT_EQ(2U, prefix_hits.size());
1793     EXPECT_EQ(kPrefix1, prefix_hits[0]);
1794     EXPECT_EQ(kPrefix2, prefix_hits[1]);
1795     EXPECT_TRUE(cache_hits.empty());
1796
1797     // kFullHash3_1 has no hits.
1798     full_hashes.push_back(kFullHash3_1);
1799     prefix_hits.clear();
1800     cache_hits.clear();
1801     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
1802         full_hashes, &prefix_hits, &cache_hits));
1803     ASSERT_EQ(2U, prefix_hits.size());
1804     EXPECT_EQ(kPrefix1, prefix_hits[0]);
1805     EXPECT_EQ(kPrefix2, prefix_hits[1]);
1806     EXPECT_TRUE(cache_hits.empty());
1807   }
1808
1809   {
1810     // Cache a fullhash result for two kPrefix1 full hashes.
1811     std::vector<SBPrefix> prefixes(1, kPrefix1);
1812     std::vector<SBFullHashResult> cache_results;
1813
1814     SBFullHashResult full_hash_result;
1815     full_hash_result.list_id = safe_browsing_util::MALWARE;
1816
1817     full_hash_result.hash = kFullHash1_1;
1818     cache_results.push_back(full_hash_result);
1819
1820     full_hash_result.hash = kFullHash1_3;
1821     cache_results.push_back(full_hash_result);
1822
1823     database_->CacheHashResults(prefixes, cache_results, kCacheLifetime);
1824   }
1825
1826   {
1827     // kFullHash1_1 should now see a cache hit.
1828     std::vector<SBFullHash> full_hashes(1, kFullHash1_1);
1829     std::vector<SBPrefix> prefix_hits;
1830     std::vector<SBFullHashResult> cache_hits;
1831     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
1832         full_hashes, &prefix_hits, &cache_hits));
1833     EXPECT_TRUE(prefix_hits.empty());
1834     ASSERT_EQ(1U, cache_hits.size());
1835     EXPECT_TRUE(SBFullHashEqual(kFullHash1_1, cache_hits[0].hash));
1836
1837     // Adding kFullHash2_1 will see the existing cache hit plus the prefix hit
1838     // for kPrefix2.
1839     full_hashes.push_back(kFullHash2_1);
1840     prefix_hits.clear();
1841     cache_hits.clear();
1842     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
1843         full_hashes, &prefix_hits, &cache_hits));
1844     ASSERT_EQ(1U, prefix_hits.size());
1845     EXPECT_EQ(kPrefix2, prefix_hits[0]);
1846     ASSERT_EQ(1U, cache_hits.size());
1847     EXPECT_TRUE(SBFullHashEqual(kFullHash1_1, cache_hits[0].hash));
1848
1849     // kFullHash1_3 also gets a cache hit.
1850     full_hashes.push_back(kFullHash1_3);
1851     prefix_hits.clear();
1852     cache_hits.clear();
1853     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
1854         full_hashes, &prefix_hits, &cache_hits));
1855     ASSERT_EQ(1U, prefix_hits.size());
1856     EXPECT_EQ(kPrefix2, prefix_hits[0]);
1857     ASSERT_EQ(2U, cache_hits.size());
1858     EXPECT_TRUE(SBFullHashEqual(kFullHash1_1, cache_hits[0].hash));
1859     EXPECT_TRUE(SBFullHashEqual(kFullHash1_3, cache_hits[1].hash));
1860   }
1861
1862   {
1863     // Check if DB contains only kFullHash1_3. Should return a cache hit.
1864     std::vector<SBFullHash> full_hashes(1, kFullHash1_3);
1865     std::vector<SBPrefix> prefix_hits;
1866     std::vector<SBFullHashResult> cache_hits;
1867     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
1868         full_hashes, &prefix_hits, &cache_hits));
1869     EXPECT_TRUE(prefix_hits.empty());
1870     ASSERT_EQ(1U, cache_hits.size());
1871     EXPECT_TRUE(SBFullHashEqual(kFullHash1_3, cache_hits[0].hash));
1872   }
1873
1874   {
1875     // kFullHash1_2 has no cache hit, and no prefix hit because of the cache for
1876     // kPrefix1.
1877     std::vector<SBFullHash> full_hashes(1, kFullHash1_2);
1878     std::vector<SBPrefix> prefix_hits;
1879     std::vector<SBFullHashResult> cache_hits;
1880     EXPECT_FALSE(database_->ContainsBrowseUrlHashesForTesting(
1881         full_hashes, &prefix_hits, &cache_hits));
1882
1883     // Other prefix hits possible when kFullHash1_2 hits nothing.
1884     full_hashes.push_back(kFullHash2_1);
1885     prefix_hits.clear();
1886     cache_hits.clear();
1887     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
1888         full_hashes, &prefix_hits, &cache_hits));
1889     ASSERT_EQ(1U, prefix_hits.size());
1890     EXPECT_EQ(kPrefix2, prefix_hits[0]);
1891     EXPECT_TRUE(cache_hits.empty());
1892   }
1893 }
1894
1895 TEST_F(SafeBrowsingDatabaseTest, BrowseFullHashMatching) {
1896   const SBPrefix kPrefix1 = 1001U;
1897   const SBFullHash kFullHash1_1 =
1898       SBFullHashForPrefixAndSuffix(kPrefix1, "\x01");
1899   const SBFullHash kFullHash1_2 =
1900       SBFullHashForPrefixAndSuffix(kPrefix1, "\x02");
1901   const SBFullHash kFullHash1_3 =
1902       SBFullHashForPrefixAndSuffix(kPrefix1, "\x03");
1903
1904   // Insert two full hashes with a shared prefix.
1905   ScopedVector<SBChunkData> chunks;
1906   chunks.push_back(AddChunkFullHash(1, kFullHash1_1));
1907   chunks.push_back(AddChunkFullHash(2, kFullHash1_2));
1908
1909   std::vector<SBListChunkRanges> lists;
1910   ASSERT_TRUE(database_->UpdateStarted(&lists));
1911   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1912   database_->UpdateFinished(true);
1913
1914   {
1915     // Check a full hash which isn't present.
1916     std::vector<SBFullHash> full_hashes(1, kFullHash1_3);
1917     std::vector<SBPrefix> prefix_hits;
1918     std::vector<SBFullHashResult> cache_hits;
1919     EXPECT_FALSE(database_->ContainsBrowseUrlHashesForTesting(
1920         full_hashes, &prefix_hits, &cache_hits));
1921
1922     // Also one which is present, should have a prefix hit.
1923     full_hashes.push_back(kFullHash1_1);
1924     prefix_hits.clear();
1925     cache_hits.clear();
1926     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
1927         full_hashes, &prefix_hits, &cache_hits));
1928     ASSERT_EQ(1U, prefix_hits.size());
1929     EXPECT_EQ(kPrefix1, prefix_hits[0]);
1930     EXPECT_TRUE(cache_hits.empty());
1931
1932     // Two full hash matches with the same prefix should return one prefix hit.
1933     full_hashes.push_back(kFullHash1_2);
1934     prefix_hits.clear();
1935     cache_hits.clear();
1936     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
1937         full_hashes, &prefix_hits, &cache_hits));
1938     ASSERT_EQ(1U, prefix_hits.size());
1939     EXPECT_EQ(kPrefix1, prefix_hits[0]);
1940     EXPECT_TRUE(cache_hits.empty());
1941   }
1942
1943   {
1944     // Cache a gethash result for kFullHash1_2.
1945     SBFullHashResult full_hash_result;
1946     full_hash_result.list_id = safe_browsing_util::MALWARE;
1947     full_hash_result.hash = kFullHash1_2;
1948
1949     std::vector<SBPrefix> prefixes(1, kPrefix1);
1950     std::vector<SBFullHashResult> cache_results(1, full_hash_result);
1951
1952     database_->CacheHashResults(prefixes, cache_results, kCacheLifetime);
1953   }
1954
1955   {
1956     // kFullHash1_3 should still return false, because the cached
1957     // result for kPrefix1 doesn't contain kFullHash1_3.
1958     std::vector<SBFullHash> full_hashes(1, kFullHash1_3);
1959     std::vector<SBPrefix> prefix_hits;
1960     std::vector<SBFullHashResult> cache_hits;
1961     EXPECT_FALSE(database_->ContainsBrowseUrlHashesForTesting(
1962         full_hashes, &prefix_hits, &cache_hits));
1963
1964     // kFullHash1_1 is also not in the cached result, which takes
1965     // priority over the database.
1966     prefix_hits.clear();
1967     full_hashes.push_back(kFullHash1_1);
1968     cache_hits.clear();
1969     EXPECT_FALSE(database_->ContainsBrowseUrlHashesForTesting(
1970         full_hashes, &prefix_hits, &cache_hits));
1971
1972     // kFullHash1_2 is in the cached result.
1973     full_hashes.push_back(kFullHash1_2);
1974     prefix_hits.clear();
1975     cache_hits.clear();
1976     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
1977         full_hashes, &prefix_hits, &cache_hits));
1978     EXPECT_TRUE(prefix_hits.empty());
1979     ASSERT_EQ(1U, cache_hits.size());
1980     EXPECT_TRUE(SBFullHashEqual(kFullHash1_2, cache_hits[0].hash));
1981   }
1982
1983   // Remove kFullHash1_1 from the database.
1984   chunks.clear();
1985   chunks.push_back(SubChunkFullHash(11, kFullHash1_1, 1));
1986
1987   ASSERT_TRUE(database_->UpdateStarted(&lists));
1988   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1989   database_->UpdateFinished(true);
1990
1991   // Cache should be cleared after updating.
1992   EXPECT_TRUE(
1993       database_->GetUnsynchronizedPrefixGetHashCacheForTesting()->empty());
1994
1995   {
1996     // Now the database doesn't contain kFullHash1_1.
1997     std::vector<SBFullHash> full_hashes(1, kFullHash1_1);
1998     std::vector<SBPrefix> prefix_hits;
1999     std::vector<SBFullHashResult> cache_hits;
2000     EXPECT_FALSE(database_->ContainsBrowseUrlHashesForTesting(
2001         full_hashes, &prefix_hits, &cache_hits));
2002
2003     // Nor kFullHash1_3.
2004     full_hashes.push_back(kFullHash1_3);
2005     prefix_hits.clear();
2006     cache_hits.clear();
2007     EXPECT_FALSE(database_->ContainsBrowseUrlHashesForTesting(
2008         full_hashes, &prefix_hits, &cache_hits));
2009
2010     // Still has kFullHash1_2.
2011     full_hashes.push_back(kFullHash1_2);
2012     prefix_hits.clear();
2013     cache_hits.clear();
2014     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
2015         full_hashes, &prefix_hits, &cache_hits));
2016     ASSERT_EQ(1U, prefix_hits.size());
2017     EXPECT_EQ(kPrefix1, prefix_hits[0]);
2018     EXPECT_TRUE(cache_hits.empty());
2019   }
2020
2021   // Remove kFullHash1_2 from the database.
2022   chunks.clear();
2023   chunks.push_back(SubChunkFullHash(12, kFullHash1_2, 2));
2024
2025   ASSERT_TRUE(database_->UpdateStarted(&lists));
2026   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
2027   database_->UpdateFinished(true);
2028
2029   // Cache should be cleared after updating.
2030   EXPECT_TRUE(
2031       database_->GetUnsynchronizedPrefixGetHashCacheForTesting()->empty());
2032
2033   {
2034     // None are present.
2035     std::vector<SBFullHash> full_hashes;
2036     std::vector<SBPrefix> prefix_hits;
2037     std::vector<SBFullHashResult> cache_hits;
2038     full_hashes.push_back(kFullHash1_1);
2039     full_hashes.push_back(kFullHash1_2);
2040     full_hashes.push_back(kFullHash1_3);
2041     EXPECT_FALSE(database_->ContainsBrowseUrlHashesForTesting(
2042         full_hashes, &prefix_hits, &cache_hits));
2043   }
2044 }
2045
2046 TEST_F(SafeBrowsingDatabaseTest, BrowseFullHashAndPrefixMatching) {
2047   const SBPrefix kPrefix1 = 1001U;
2048   const SBFullHash kFullHash1_1 =
2049       SBFullHashForPrefixAndSuffix(kPrefix1, "\x01");
2050   const SBFullHash kFullHash1_2 =
2051       SBFullHashForPrefixAndSuffix(kPrefix1, "\x02");
2052
2053   ScopedVector<SBChunkData> chunks;
2054   chunks.push_back(AddChunkFullHash(1, kFullHash1_1));
2055
2056   std::vector<SBListChunkRanges> lists;
2057   ASSERT_TRUE(database_->UpdateStarted(&lists));
2058   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
2059   database_->UpdateFinished(true);
2060
2061   {
2062     // kFullHash1_2 does not match kFullHash1_1.
2063     std::vector<SBFullHash> full_hashes(1, kFullHash1_2);
2064     std::vector<SBPrefix> prefix_hits;
2065     std::vector<SBFullHashResult> cache_hits;
2066     EXPECT_FALSE(database_->ContainsBrowseUrlHashesForTesting(
2067         full_hashes, &prefix_hits, &cache_hits));
2068   }
2069
2070   // Add a prefix match.
2071   chunks.clear();
2072   chunks.push_back(AddChunkPrefix(2, kPrefix1));
2073
2074   ASSERT_TRUE(database_->UpdateStarted(&lists));
2075   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
2076   database_->UpdateFinished(true);
2077
2078   {
2079     // kFullHash1_2 does match kPrefix1.
2080     std::vector<SBFullHash> full_hashes(1, kFullHash1_2);
2081     std::vector<SBPrefix> prefix_hits;
2082     std::vector<SBFullHashResult> cache_hits;
2083     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
2084         full_hashes, &prefix_hits, &cache_hits));
2085     ASSERT_EQ(1U, prefix_hits.size());
2086     EXPECT_EQ(kPrefix1, prefix_hits[0]);
2087     EXPECT_TRUE(cache_hits.empty());
2088   }
2089
2090   // Remove the full hash.
2091   chunks.clear();
2092   chunks.push_back(SubChunkFullHash(11, kFullHash1_1, 1));
2093
2094   ASSERT_TRUE(database_->UpdateStarted(&lists));
2095   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
2096   database_->UpdateFinished(true);
2097
2098   {
2099     // kFullHash1_2 still returns true due to the prefix hit.
2100     std::vector<SBFullHash> full_hashes(1, kFullHash1_2);
2101     std::vector<SBPrefix> prefix_hits;
2102     std::vector<SBFullHashResult> cache_hits;
2103     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
2104         full_hashes, &prefix_hits, &cache_hits));
2105     ASSERT_EQ(1U, prefix_hits.size());
2106     EXPECT_EQ(kPrefix1, prefix_hits[0]);
2107     EXPECT_TRUE(cache_hits.empty());
2108   }
2109 }
2110
2111 TEST_F(SafeBrowsingDatabaseTest, MalwareIpBlacklist) {
2112   std::vector<SBListChunkRanges> lists;
2113   ASSERT_TRUE(database_->UpdateStarted(&lists));
2114
2115   ScopedVector<SBChunkData> chunks;
2116
2117   // IPv4 prefix match for ::ffff:192.168.1.0/120.
2118   chunks.push_back(AddChunkHashedIpValue(1, "::ffff:192.168.1.0", 120));
2119
2120   // IPv4 exact match for ::ffff:192.1.1.1.
2121   chunks.push_back(AddChunkHashedIpValue(2, "::ffff:192.1.1.1", 128));
2122
2123   // IPv6 exact match for: fe80::31a:a0ff:fe10:786e/128.
2124   chunks.push_back(AddChunkHashedIpValue(3, "fe80::31a:a0ff:fe10:786e", 128));
2125
2126   // IPv6 prefix match for: 2620:0:1000:3103::/64.
2127   chunks.push_back(AddChunkHashedIpValue(4, "2620:0:1000:3103::", 64));
2128
2129   // IPv4 prefix match for ::ffff:192.1.122.0/119.
2130   chunks.push_back(AddChunkHashedIpValue(5, "::ffff:192.1.122.0", 119));
2131
2132   // IPv4 prefix match for ::ffff:192.1.128.0/113.
2133   chunks.push_back(AddChunkHashedIpValue(6, "::ffff:192.1.128.0", 113));
2134
2135   database_->InsertChunks(safe_browsing_util::kIPBlacklist, chunks.get());
2136   database_->UpdateFinished(true);
2137
2138   EXPECT_FALSE(database_->ContainsMalwareIP("192.168.0.255"));
2139   EXPECT_TRUE(database_->ContainsMalwareIP("192.168.1.0"));
2140   EXPECT_TRUE(database_->ContainsMalwareIP("192.168.1.255"));
2141   EXPECT_TRUE(database_->ContainsMalwareIP("192.168.1.10"));
2142   EXPECT_TRUE(database_->ContainsMalwareIP("::ffff:192.168.1.2"));
2143   EXPECT_FALSE(database_->ContainsMalwareIP("192.168.2.0"));
2144
2145   EXPECT_FALSE(database_->ContainsMalwareIP("192.1.1.0"));
2146   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.1.1"));
2147   EXPECT_FALSE(database_->ContainsMalwareIP("192.1.1.2"));
2148
2149   EXPECT_FALSE(database_->ContainsMalwareIP(
2150       "2620:0:1000:3102:ffff:ffff:ffff:ffff"));
2151   EXPECT_TRUE(database_->ContainsMalwareIP("2620:0:1000:3103::"));
2152   EXPECT_TRUE(database_->ContainsMalwareIP(
2153       "2620:0:1000:3103:ffff:ffff:ffff:ffff"));
2154   EXPECT_FALSE(database_->ContainsMalwareIP("2620:0:1000:3104::"));
2155
2156   EXPECT_FALSE(database_->ContainsMalwareIP("fe80::21a:a0ff:fe10:786d"));
2157   EXPECT_TRUE(database_->ContainsMalwareIP("fe80::31a:a0ff:fe10:786e"));
2158   EXPECT_FALSE(database_->ContainsMalwareIP("fe80::21a:a0ff:fe10:786f"));
2159
2160   EXPECT_FALSE(database_->ContainsMalwareIP("192.1.121.255"));
2161   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.122.0"));
2162   EXPECT_TRUE(database_->ContainsMalwareIP("::ffff:192.1.122.1"));
2163   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.122.255"));
2164   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.123.0"));
2165   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.123.255"));
2166   EXPECT_FALSE(database_->ContainsMalwareIP("192.1.124.0"));
2167
2168   EXPECT_FALSE(database_->ContainsMalwareIP("192.1.127.255"));
2169   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.128.0"));
2170   EXPECT_TRUE(database_->ContainsMalwareIP("::ffff:192.1.128.1"));
2171   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.128.255"));
2172   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.255.0"));
2173   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.255.255"));
2174   EXPECT_FALSE(database_->ContainsMalwareIP("192.2.0.0"));
2175 }
2176
2177 TEST_F(SafeBrowsingDatabaseTest, ContainsBrowseURL) {
2178   std::vector<SBListChunkRanges> lists;
2179   ASSERT_TRUE(database_->UpdateStarted(&lists));
2180
2181   // Add a host-level hit.
2182   {
2183     ScopedVector<SBChunkData> chunks;
2184     chunks.push_back(AddChunkPrefixValue(1, "www.evil.com/"));
2185     database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
2186   }
2187
2188   // Add a specific fullhash.
2189   static const char kWhateverMalware[] = "www.whatever.com/malware.html";
2190   {
2191     ScopedVector<SBChunkData> chunks;
2192     chunks.push_back(AddChunkFullHashValue(2, kWhateverMalware));
2193     database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
2194   }
2195
2196   // Add a fullhash which has a prefix collision for a known url.
2197   static const char kExampleFine[] = "www.example.com/fine.html";
2198   static const char kExampleCollision[] =
2199       "www.example.com/3123364814/malware.htm";
2200   ASSERT_EQ(SBPrefixForString(kExampleFine),
2201             SBPrefixForString(kExampleCollision));
2202   {
2203     ScopedVector<SBChunkData> chunks;
2204     chunks.push_back(AddChunkFullHashValue(3, kExampleCollision));
2205     database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
2206   }
2207
2208   database_->UpdateFinished(true);
2209
2210   std::vector<SBPrefix> prefix_hits;
2211   std::vector<SBFullHashResult> cache_hits;
2212
2213   // Anything will hit the host prefix.
2214   EXPECT_TRUE(database_->ContainsBrowseUrl(
2215       GURL("http://www.evil.com/malware.html"), &prefix_hits, &cache_hits));
2216   ASSERT_EQ(1U, prefix_hits.size());
2217   EXPECT_EQ(SBPrefixForString("www.evil.com/"), prefix_hits[0]);
2218   EXPECT_TRUE(cache_hits.empty());
2219
2220   // Hit the specific URL prefix.
2221   EXPECT_TRUE(database_->ContainsBrowseUrl(
2222       GURL(std::string("http://") + kWhateverMalware),
2223       &prefix_hits, &cache_hits));
2224   ASSERT_EQ(1U, prefix_hits.size());
2225   EXPECT_EQ(SBPrefixForString(kWhateverMalware), prefix_hits[0]);
2226   EXPECT_TRUE(cache_hits.empty());
2227
2228   // Other URLs at that host are fine.
2229   EXPECT_FALSE(database_->ContainsBrowseUrl(
2230       GURL("http://www.whatever.com/fine.html"), &prefix_hits, &cache_hits));
2231   EXPECT_TRUE(prefix_hits.empty());
2232   EXPECT_TRUE(cache_hits.empty());
2233
2234   // Hit the specific URL full hash.
2235   EXPECT_TRUE(database_->ContainsBrowseUrl(
2236       GURL(std::string("http://") + kExampleCollision),
2237       &prefix_hits, &cache_hits));
2238   ASSERT_EQ(1U, prefix_hits.size());
2239   EXPECT_EQ(SBPrefixForString(kExampleCollision), prefix_hits[0]);
2240   EXPECT_TRUE(cache_hits.empty());
2241
2242   // This prefix collides, but no full hash match.
2243   EXPECT_FALSE(database_->ContainsBrowseUrl(
2244       GURL(std::string("http://") + kExampleFine), &prefix_hits, &cache_hits));
2245 }