chrome/browser/safe_browsing/safe_browsing_database_unittest.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4 //
   5 // Unit tests for the SafeBrowsing storage system.
   6
   7 #include "chrome/browser/safe_browsing/safe_browsing_database.h"
   8
   9 #include "base/files/file_util.h"
  10 #include "base/files/scoped_temp_dir.h"
  11 #include "base/logging.h"
  12 #include "base/memory/scoped_vector.h"
  13 #include "base/message_loop/message_loop.h"
  14 #include "base/sha1.h"
  15 #include "base/strings/string_number_conversions.h"
  16 #include "base/strings/string_split.h"
  17 #include "base/test/test_simple_task_runner.h"
  18 #include "base/time/time.h"
  19 #include "chrome/browser/safe_browsing/chunk.pb.h"
  20 #include "chrome/browser/safe_browsing/safe_browsing_store_file.h"
  21 #include "crypto/sha2.h"
  22 #include "net/base/net_util.h"
  23 #include "sql/connection.h"
  24 #include "sql/statement.h"
  25 #include "testing/gtest/include/gtest/gtest.h"
  26 #include "testing/platform_test.h"
  27 #include "url/gurl.h"
  28
  29 using base::Time;
  30 using base::TimeDelta;
  31
  32 namespace {
  33
  34 const TimeDelta kCacheLifetime = TimeDelta::FromMinutes(45);
  35
  36 SBPrefix SBPrefixForString(const std::string& str) {
  37   return SBFullHashForString(str).prefix;
  38 }
  39
  40 // Construct a full hash which has the given prefix, with the given
  41 // suffix data coming after the prefix.
  42 SBFullHash SBFullHashForPrefixAndSuffix(SBPrefix prefix,
  43                                         const base::StringPiece& suffix) {
  44   SBFullHash full_hash;
  45   memset(&full_hash, 0, sizeof(SBFullHash));
  46   full_hash.prefix = prefix;
  47   CHECK_LE(suffix.size() + sizeof(SBPrefix), sizeof(SBFullHash));
  48   memcpy(full_hash.full_hash + sizeof(SBPrefix), suffix.data(), suffix.size());
  49   return full_hash;
  50 }
  51
  52 std::string HashedIpPrefix(const std::string& ip_prefix, size_t prefix_size) {
  53   net::IPAddressNumber ip_number;
  54   EXPECT_TRUE(net::ParseIPLiteralToNumber(ip_prefix, &ip_number));
  55   EXPECT_EQ(net::kIPv6AddressSize, ip_number.size());
  56   const std::string hashed_ip_prefix = base::SHA1HashString(
  57       net::IPAddressToPackedString(ip_number));
  58   std::string hash(crypto::kSHA256Length, '\0');
  59   hash.replace(0, hashed_ip_prefix.size(), hashed_ip_prefix);
  60   hash[base::kSHA1Length] = static_cast<char>(prefix_size);
  61   return hash;
  62 }
  63
  64 // Helper to build a chunk.  Caller takes ownership.
  65 SBChunkData* BuildChunk(int chunk_number,
  66                         safe_browsing::ChunkData::ChunkType chunk_type,
  67                         safe_browsing::ChunkData::PrefixType prefix_type,
  68                         const void* data, size_t data_size,
  69                         const std::vector<int>& add_chunk_numbers) {
  70   scoped_ptr<safe_browsing::ChunkData> raw_data(new safe_browsing::ChunkData);
  71   raw_data->set_chunk_number(chunk_number);
  72   raw_data->set_chunk_type(chunk_type);
  73   raw_data->set_prefix_type(prefix_type);
  74   raw_data->set_hashes(data, data_size);
  75   raw_data->clear_add_numbers();
  76   for (size_t i = 0; i < add_chunk_numbers.size(); ++i) {
  77     raw_data->add_add_numbers(add_chunk_numbers[i]);
  78   }
  79
  80   return new SBChunkData(raw_data.release());
  81 }
  82
  83 // Create add chunk with a single prefix.
  84 SBChunkData* AddChunkPrefix(int chunk_number, SBPrefix prefix) {
  85   return BuildChunk(chunk_number, safe_browsing::ChunkData::ADD,
  86                     safe_browsing::ChunkData::PREFIX_4B,
  87                     &prefix, sizeof(prefix),
  88                     std::vector<int>());
  89 }
  90
  91 // Create add chunk with a single prefix generated from |value|.
  92 SBChunkData* AddChunkPrefixValue(int chunk_number,
  93                                  const std::string& value) {
  94   return AddChunkPrefix(chunk_number, SBPrefixForString(value));
  95 }
  96
  97 // Generate an add chunk with two prefixes.
  98 SBChunkData* AddChunkPrefix2Value(int chunk_number,
  99                                   const std::string& value1,
 100                                   const std::string& value2) {
 101   const SBPrefix prefixes[2] = {
 102     SBPrefixForString(value1),
 103     SBPrefixForString(value2),
 104   };
 105   return BuildChunk(chunk_number, safe_browsing::ChunkData::ADD,
 106                     safe_browsing::ChunkData::PREFIX_4B,
 107                     &prefixes[0], sizeof(prefixes),
 108                     std::vector<int>());
 109 }
 110
 111 // Generate an add chunk with four prefixes.
 112 SBChunkData* AddChunkPrefix4Value(int chunk_number,
 113                                   const std::string& value1,
 114                                   const std::string& value2,
 115                                   const std::string& value3,
 116                                   const std::string& value4) {
 117   const SBPrefix prefixes[4] = {
 118     SBPrefixForString(value1),
 119     SBPrefixForString(value2),
 120     SBPrefixForString(value3),
 121     SBPrefixForString(value4),
 122   };
 123   return BuildChunk(chunk_number, safe_browsing::ChunkData::ADD,
 124                     safe_browsing::ChunkData::PREFIX_4B,
 125                     &prefixes[0], sizeof(prefixes),
 126                     std::vector<int>());
 127 }
 128
 129 // Generate an add chunk with a full hash.
 130 SBChunkData* AddChunkFullHash(int chunk_number, SBFullHash full_hash) {
 131   return BuildChunk(chunk_number, safe_browsing::ChunkData::ADD,
 132                     safe_browsing::ChunkData::FULL_32B,
 133                     &full_hash, sizeof(full_hash),
 134                     std::vector<int>());
 135 }
 136
 137 // Generate an add chunk with a full hash generated from |value|.
 138 SBChunkData* AddChunkFullHashValue(int chunk_number,
 139                                    const std::string& value) {
 140   return AddChunkFullHash(chunk_number, SBFullHashForString(value));
 141 }
 142
 143 // Generate an add chunk with two full hashes.
 144 SBChunkData* AddChunkFullHash2Value(int chunk_number,
 145                                    const std::string& value1,
 146                                    const std::string& value2) {
 147   const SBFullHash full_hashes[2] = {
 148     SBFullHashForString(value1),
 149     SBFullHashForString(value2),
 150   };
 151   return BuildChunk(chunk_number, safe_browsing::ChunkData::ADD,
 152                     safe_browsing::ChunkData::FULL_32B,
 153                     &full_hashes[0], sizeof(full_hashes),
 154                     std::vector<int>());
 155 }
 156
 157 // Generate a sub chunk with a prefix generated from |value|.
 158 SBChunkData* SubChunkPrefixValue(int chunk_number,
 159                                  const std::string& value,
 160                                  int add_chunk_number) {
 161   const SBPrefix prefix = SBPrefixForString(value);
 162   return BuildChunk(chunk_number, safe_browsing::ChunkData::SUB,
 163                     safe_browsing::ChunkData::PREFIX_4B,
 164                     &prefix, sizeof(prefix),
 165                     std::vector<int>(1, add_chunk_number));
 166 }
 167
 168 // Generate a sub chunk with two prefixes.
 169 SBChunkData* SubChunkPrefix2Value(int chunk_number,
 170                                   const std::string& value1,
 171                                   int add_chunk_number1,
 172                                   const std::string& value2,
 173                                   int add_chunk_number2) {
 174   const SBPrefix prefixes[2] = {
 175     SBPrefixForString(value1),
 176     SBPrefixForString(value2),
 177   };
 178   std::vector<int> add_chunk_numbers;
 179   add_chunk_numbers.push_back(add_chunk_number1);
 180   add_chunk_numbers.push_back(add_chunk_number2);
 181   return BuildChunk(chunk_number, safe_browsing::ChunkData::SUB,
 182                     safe_browsing::ChunkData::PREFIX_4B,
 183                     &prefixes[0], sizeof(prefixes),
 184                     add_chunk_numbers);
 185 }
 186
 187 // Generate a sub chunk with a full hash.
 188 SBChunkData* SubChunkFullHash(int chunk_number,
 189                               SBFullHash full_hash,
 190                               int add_chunk_number) {
 191   return BuildChunk(chunk_number, safe_browsing::ChunkData::SUB,
 192                     safe_browsing::ChunkData::FULL_32B,
 193                     &full_hash, sizeof(full_hash),
 194                     std::vector<int>(1, add_chunk_number));
 195 }
 196
 197 // Generate a sub chunk with a full hash generated from |value|.
 198 SBChunkData* SubChunkFullHashValue(int chunk_number,
 199                                    const std::string& value,
 200                                    int add_chunk_number) {
 201   return SubChunkFullHash(chunk_number,
 202                           SBFullHashForString(value),
 203                           add_chunk_number);
 204 }
 205
 206 // Generate an add chunk with a single full hash for the ip blacklist.
 207 SBChunkData* AddChunkHashedIpValue(int chunk_number,
 208                                    const std::string& ip_str,
 209                                    size_t prefix_size) {
 210   const std::string full_hash_str = HashedIpPrefix(ip_str, prefix_size);
 211   EXPECT_EQ(sizeof(SBFullHash), full_hash_str.size());
 212   SBFullHash full_hash;
 213   std::memcpy(&(full_hash.full_hash), full_hash_str.data(), sizeof(SBFullHash));
 214   return BuildChunk(chunk_number, safe_browsing::ChunkData::ADD,
 215                     safe_browsing::ChunkData::FULL_32B,
 216                     &full_hash, sizeof(full_hash),
 217                     std::vector<int>());
 218 }
 219
 220 // Prevent DCHECK from killing tests.
 221 // TODO(shess): Pawel disputes the use of this, so the test which uses
 222 // it is DISABLED.  http://crbug.com/56448
 223 class ScopedLogMessageIgnorer {
 224  public:
 225   ScopedLogMessageIgnorer() {
 226     logging::SetLogMessageHandler(&LogMessageIgnorer);
 227   }
 228   ~ScopedLogMessageIgnorer() {
 229     // TODO(shess): Would be better to verify whether anyone else
 230     // changed it, and then restore it to the previous value.
 231     logging::SetLogMessageHandler(NULL);
 232   }
 233
 234  private:
 235   static bool LogMessageIgnorer(int severity, const char* file, int line,
 236       size_t message_start, const std::string& str) {
 237     // Intercept FATAL, strip the stack backtrace, and log it without
 238     // the crash part.
 239     if (severity == logging::LOG_FATAL) {
 240       size_t newline = str.find('\n');
 241       if (newline != std::string::npos) {
 242         const std::string msg = str.substr(0, newline + 1);
 243         fprintf(stderr, "%s", msg.c_str());
 244         fflush(stderr);
 245       }
 246       return true;
 247     }
 248
 249     return false;
 250   }
 251 };
 252
 253 }  // namespace
 254
 255 class SafeBrowsingDatabaseTest : public PlatformTest {
 256  public:
 257   SafeBrowsingDatabaseTest() : task_runner_(new base::TestSimpleTaskRunner) {}
 258
 259   void SetUp() override {
 260     PlatformTest::SetUp();
 261
 262     // Setup a database in a temporary directory.
 263     ASSERT_TRUE(temp_dir_.CreateUniqueTempDir());
 264     database_filename_ =
 265         temp_dir_.path().AppendASCII("SafeBrowsingTestDatabase");
 266
 267     ResetAndReloadFullDatabase();
 268   }
 269
 270   void TearDown() override {
 271     database_.reset();
 272
 273     PlatformTest::TearDown();
 274   }
 275
 276   // Reloads the |database_| in a new SafeBrowsingDatabaseNew object with all
 277   // stores enabled.
 278   void ResetAndReloadFullDatabase() {
 279     SafeBrowsingStoreFile* browse_store =
 280         new SafeBrowsingStoreFile(task_runner_);
 281     SafeBrowsingStoreFile* download_store =
 282         new SafeBrowsingStoreFile(task_runner_);
 283     SafeBrowsingStoreFile* csd_whitelist_store =
 284         new SafeBrowsingStoreFile(task_runner_);
 285     SafeBrowsingStoreFile* download_whitelist_store =
 286         new SafeBrowsingStoreFile(task_runner_);
 287     SafeBrowsingStoreFile* inclusion_whitelist_store =
 288         new SafeBrowsingStoreFile(task_runner_);
 289     SafeBrowsingStoreFile* extension_blacklist_store =
 290         new SafeBrowsingStoreFile(task_runner_);
 291     SafeBrowsingStoreFile* side_effect_free_whitelist_store =
 292         new SafeBrowsingStoreFile(task_runner_);
 293     SafeBrowsingStoreFile* ip_blacklist_store =
 294         new SafeBrowsingStoreFile(task_runner_);
 295     SafeBrowsingStoreFile* unwanted_software_store =
 296         new SafeBrowsingStoreFile(task_runner_);
 297     database_.reset(new SafeBrowsingDatabaseNew(
 298         task_runner_, browse_store, download_store, csd_whitelist_store,
 299         download_whitelist_store, inclusion_whitelist_store,
 300         extension_blacklist_store, side_effect_free_whitelist_store,
 301         ip_blacklist_store, unwanted_software_store));
 302     database_->Init(database_filename_);
 303   }
 304
 305   bool ContainsDownloadUrl(const std::vector<GURL>& urls,
 306                            std::vector<SBPrefix>* prefix_hits) {
 307     std::vector<SBPrefix> prefixes;
 308     SafeBrowsingDatabase::GetDownloadUrlPrefixes(urls, &prefixes);
 309     return database_->ContainsDownloadUrlPrefixes(prefixes, prefix_hits);
 310   }
 311
 312   void GetListsInfo(std::vector<SBListChunkRanges>* lists) {
 313     lists->clear();
 314     ASSERT_TRUE(database_->UpdateStarted(lists));
 315     database_->UpdateFinished(true);
 316   }
 317
 318   // Helper function to do an AddDel or SubDel command.
 319   void DelChunk(const std::string& list,
 320                 int chunk_id,
 321                 bool is_sub_del) {
 322     std::vector<SBChunkDelete> deletes;
 323     SBChunkDelete chunk_delete;
 324     chunk_delete.list_name = list;
 325     chunk_delete.is_sub_del = is_sub_del;
 326     chunk_delete.chunk_del.push_back(ChunkRange(chunk_id));
 327     deletes.push_back(chunk_delete);
 328     database_->DeleteChunks(deletes);
 329   }
 330
 331   void AddDelChunk(const std::string& list, int chunk_id) {
 332     DelChunk(list, chunk_id, false);
 333   }
 334
 335   void SubDelChunk(const std::string& list, int chunk_id) {
 336     DelChunk(list, chunk_id, true);
 337   }
 338
 339   // Utility function for setting up the database for the caching test.
 340   void PopulateDatabaseForCacheTest();
 341
 342   scoped_refptr<base::TestSimpleTaskRunner> task_runner_;
 343   scoped_ptr<SafeBrowsingDatabaseNew> database_;
 344   base::FilePath database_filename_;
 345   base::ScopedTempDir temp_dir_;
 346 };
 347
 348 // Tests retrieving list name information.
 349 TEST_F(SafeBrowsingDatabaseTest, BrowseListsInfo) {
 350   std::vector<SBListChunkRanges> lists;
 351   ScopedVector<SBChunkData> chunks;
 352
 353   chunks.push_back(AddChunkPrefixValue(1, "www.evil.com/malware.html"));
 354   chunks.push_back(AddChunkPrefixValue(2, "www.foo.com/malware.html"));
 355   chunks.push_back(AddChunkPrefixValue(3, "www.whatever.com/malware.html"));
 356
 357   ASSERT_TRUE(database_->UpdateStarted(&lists));
 358   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
 359   database_->UpdateFinished(true);
 360
 361   GetListsInfo(&lists);
 362   ASSERT_LE(1U, lists.size());
 363   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
 364   EXPECT_EQ("1-3", lists[0].adds);
 365   EXPECT_TRUE(lists[0].subs.empty());
 366
 367   // Insert a malware sub chunk.
 368   chunks.clear();
 369   chunks.push_back(SubChunkPrefixValue(7, "www.subbed.com/noteveil1.html", 19));
 370
 371   ASSERT_TRUE(database_->UpdateStarted(&lists));
 372   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
 373   database_->UpdateFinished(true);
 374
 375   GetListsInfo(&lists);
 376   ASSERT_LE(1U, lists.size());
 377   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
 378   EXPECT_EQ("1-3", lists[0].adds);
 379   EXPECT_EQ("7", lists[0].subs);
 380   if (lists.size() == 2) {
 381     // Old style database won't have the second entry since it creates the lists
 382     // when it receives an update containing that list. The filter-based
 383     // database has these values hard coded.
 384     EXPECT_EQ(safe_browsing_util::kPhishingList, lists[1].name);
 385     EXPECT_TRUE(lists[1].adds.empty());
 386     EXPECT_TRUE(lists[1].subs.empty());
 387   }
 388
 389   // Add phishing chunks.
 390   chunks.clear();
 391   chunks.push_back(AddChunkPrefixValue(47, "www.evil.com/phishing.html"));
 392   chunks.push_back(
 393       SubChunkPrefixValue(200, "www.phishy.com/notevil1.html", 1999));
 394   chunks.push_back(
 395       SubChunkPrefixValue(201, "www.phishy2.com/notevil1.html", 1999));
 396
 397   ASSERT_TRUE(database_->UpdateStarted(&lists));
 398   database_->InsertChunks(safe_browsing_util::kPhishingList, chunks.get());
 399   database_->UpdateFinished(true);
 400
 401   GetListsInfo(&lists);
 402   ASSERT_LE(2U, lists.size());
 403   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
 404   EXPECT_EQ("1-3", lists[0].adds);
 405   EXPECT_EQ("7", lists[0].subs);
 406   EXPECT_EQ(safe_browsing_util::kPhishingList, lists[1].name);
 407   EXPECT_EQ("47", lists[1].adds);
 408   EXPECT_EQ("200-201", lists[1].subs);
 409 }
 410
 411 TEST_F(SafeBrowsingDatabaseTest, ListNames) {
 412   ScopedVector<SBChunkData> chunks;
 413
 414   std::vector<SBListChunkRanges> lists;
 415   ASSERT_TRUE(database_->UpdateStarted(&lists));
 416
 417   // Insert malware, phish, binurl and bindownload add chunks.
 418   chunks.push_back(AddChunkPrefixValue(1, "www.evil.com/malware.html"));
 419   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
 420
 421   chunks.clear();
 422   chunks.push_back(AddChunkPrefixValue(2, "www.foo.com/malware.html"));
 423   database_->InsertChunks(safe_browsing_util::kPhishingList, chunks.get());
 424
 425   chunks.clear();
 426   chunks.push_back(AddChunkPrefixValue(3, "www.whatever.com/download.html"));
 427   database_->InsertChunks(safe_browsing_util::kBinUrlList, chunks.get());
 428
 429   chunks.clear();
 430   chunks.push_back(AddChunkFullHashValue(5, "www.forwhitelist.com/a.html"));
 431   database_->InsertChunks(safe_browsing_util::kCsdWhiteList, chunks.get());
 432
 433   chunks.clear();
 434   chunks.push_back(AddChunkFullHashValue(6, "www.download.com/"));
 435   database_->InsertChunks(safe_browsing_util::kDownloadWhiteList, chunks.get());
 436
 437   chunks.clear();
 438   chunks.push_back(AddChunkFullHashValue(7, "www.inclusion.com/"));
 439   database_->InsertChunks(safe_browsing_util::kInclusionWhitelist,
 440                           chunks.get());
 441
 442   chunks.clear();
 443   chunks.push_back(AddChunkFullHashValue(8,
 444                                          "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
 445                                          "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"));
 446   database_->InsertChunks(safe_browsing_util::kExtensionBlacklist,
 447                           chunks.get());
 448
 449   chunks.clear();
 450   chunks.push_back(AddChunkFullHashValue(9, "www.sideeffectfree.com"));
 451   database_->InsertChunks(safe_browsing_util::kSideEffectFreeWhitelist,
 452                           chunks.get());
 453
 454   chunks.clear();
 455   chunks.push_back(AddChunkHashedIpValue(10, "::ffff:192.168.1.0", 120));
 456   database_->InsertChunks(safe_browsing_util::kIPBlacklist, chunks.get());
 457
 458   chunks.clear();
 459   chunks.push_back(AddChunkPrefixValue(11, "www.unwanted.com/software.html"));
 460   database_->InsertChunks(safe_browsing_util::kUnwantedUrlList, chunks.get());
 461
 462   database_->UpdateFinished(true);
 463
 464   GetListsInfo(&lists);
 465   ASSERT_EQ(10U, lists.size());
 466   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
 467   EXPECT_EQ("1", lists[0].adds);
 468   EXPECT_TRUE(lists[0].subs.empty());
 469   EXPECT_EQ(safe_browsing_util::kPhishingList, lists[1].name);
 470   EXPECT_EQ("2", lists[1].adds);
 471   EXPECT_TRUE(lists[1].subs.empty());
 472   EXPECT_EQ(safe_browsing_util::kBinUrlList, lists[2].name);
 473   EXPECT_EQ("3", lists[2].adds);
 474   EXPECT_TRUE(lists[2].subs.empty());
 475   EXPECT_EQ(safe_browsing_util::kCsdWhiteList, lists[3].name);
 476   EXPECT_EQ("5", lists[3].adds);
 477   EXPECT_TRUE(lists[3].subs.empty());
 478   EXPECT_EQ(safe_browsing_util::kDownloadWhiteList, lists[4].name);
 479   EXPECT_EQ("6", lists[4].adds);
 480   EXPECT_TRUE(lists[4].subs.empty());
 481   EXPECT_EQ(safe_browsing_util::kInclusionWhitelist, lists[5].name);
 482   EXPECT_EQ("7", lists[5].adds);
 483   EXPECT_TRUE(lists[5].subs.empty());
 484   EXPECT_EQ(safe_browsing_util::kExtensionBlacklist, lists[6].name);
 485   EXPECT_EQ("8", lists[6].adds);
 486   EXPECT_TRUE(lists[6].subs.empty());
 487   EXPECT_EQ(safe_browsing_util::kSideEffectFreeWhitelist, lists[7].name);
 488   EXPECT_EQ("9", lists[7].adds);
 489   EXPECT_TRUE(lists[7].subs.empty());
 490   EXPECT_EQ(safe_browsing_util::kIPBlacklist, lists[8].name);
 491   EXPECT_EQ("10", lists[8].adds);
 492   EXPECT_TRUE(lists[8].subs.empty());
 493   EXPECT_EQ(safe_browsing_util::kUnwantedUrlList, lists[9].name);
 494   EXPECT_EQ("11", lists[9].adds);
 495   EXPECT_TRUE(lists[9].subs.empty());
 496
 497   database_.reset();
 498 }
 499
 500 // Checks database reading and writing for browse and unwanted PrefixSets.
 501 TEST_F(SafeBrowsingDatabaseTest, BrowseAndUnwantedDatabasesAndPrefixSets) {
 502   struct TestCase {
 503     using TestListContainsBadUrl = bool (SafeBrowsingDatabase::*)(
 504         const GURL& url,
 505         std::vector<SBPrefix>* prefix_hits,
 506         std::vector<SBFullHashResult>* cache_hits);
 507
 508     const char* test_list_name;
 509     size_t expected_list_index;
 510     TestListContainsBadUrl test_list_contains_bad_url;
 511   } const kTestCases[] {
 512     { safe_browsing_util::kMalwareList, 0U,
 513       &SafeBrowsingDatabase::ContainsBrowseUrl },
 514     { safe_browsing_util::kPhishingList, 1U,
 515       &SafeBrowsingDatabase::ContainsBrowseUrl },
 516     { safe_browsing_util::kUnwantedUrlList, 9U,
 517       &SafeBrowsingDatabase::ContainsUnwantedSoftwareUrl },
 518   };
 519
 520   for (const auto& test_case : kTestCases) {
 521     SCOPED_TRACE(std::string("Tested list at fault => ") +
 522                  test_case.test_list_name);
 523
 524     std::vector<SBListChunkRanges> lists;
 525     ScopedVector<SBChunkData> chunks;
 526
 527     chunks.push_back(AddChunkPrefix2Value(1,
 528                                           "www.evil.com/phishing.html",
 529                                           "www.evil.com/malware.html"));
 530     chunks.push_back(AddChunkPrefix4Value(2,
 531                                           "www.evil.com/notevil1.html",
 532                                           "www.evil.com/notevil2.html",
 533                                           "www.good.com/good1.html",
 534                                           "www.good.com/good2.html"));
 535     chunks.push_back(AddChunkPrefixValue(3, "192.168.0.1/malware.html"));
 536     chunks.push_back(AddChunkFullHashValue(7, "www.evil.com/evil.html"));
 537
 538     ASSERT_TRUE(database_->UpdateStarted(&lists));
 539     database_->InsertChunks(test_case.test_list_name, chunks.get());
 540     database_->UpdateFinished(true);
 541
 542     // Make sure they were added correctly.
 543     GetListsInfo(&lists);
 544
 545     ASSERT_LE(1U, lists.size());
 546     EXPECT_EQ(test_case.test_list_name,
 547               lists[test_case.expected_list_index].name);
 548     EXPECT_EQ("1-3,7", lists[test_case.expected_list_index].adds);
 549     EXPECT_TRUE(lists[test_case.expected_list_index].subs.empty());
 550
 551     std::vector<SBPrefix> prefix_hits;
 552     std::vector<SBFullHashResult> cache_hits;
 553     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 554         GURL("http://www.evil.com/phishing.html"), &prefix_hits, &cache_hits));
 555     ASSERT_EQ(1U, prefix_hits.size());
 556     EXPECT_EQ(SBPrefixForString("www.evil.com/phishing.html"), prefix_hits[0]);
 557     EXPECT_TRUE(cache_hits.empty());
 558
 559     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 560         GURL("http://www.evil.com/malware.html"), &prefix_hits, &cache_hits));
 561
 562     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 563         GURL("http://www.evil.com/notevil1.html"), &prefix_hits, &cache_hits));
 564
 565     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 566         GURL("http://www.evil.com/notevil2.html"), &prefix_hits, &cache_hits));
 567
 568     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 569         GURL("http://www.good.com/good1.html"), &prefix_hits, &cache_hits));
 570
 571     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 572         GURL("http://www.good.com/good2.html"), &prefix_hits, &cache_hits));
 573
 574     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 575         GURL("http://192.168.0.1/malware.html"), &prefix_hits, &cache_hits));
 576
 577     EXPECT_FALSE((database_.get()->*test_case.test_list_contains_bad_url)(
 578         GURL("http://www.evil.com/"), &prefix_hits, &cache_hits));
 579     EXPECT_TRUE(prefix_hits.empty());
 580     EXPECT_TRUE(cache_hits.empty());
 581
 582     EXPECT_FALSE((database_.get()->*test_case.test_list_contains_bad_url)(
 583         GURL("http://www.evil.com/robots.txt"), &prefix_hits, &cache_hits));
 584
 585     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 586         GURL("http://www.evil.com/evil.html"), &prefix_hits, &cache_hits));
 587     ASSERT_EQ(1U, prefix_hits.size());
 588     EXPECT_EQ(SBPrefixForString("www.evil.com/evil.html"), prefix_hits[0]);
 589
 590     // Attempt to re-add the first chunk (should be a no-op).
 591     // see bug: http://code.google.com/p/chromium/issues/detail?id=4522
 592     chunks.clear();
 593     chunks.push_back(AddChunkPrefix2Value(1,
 594                                           "www.evil.com/phishing.html",
 595                                           "www.evil.com/malware.html"));
 596     ASSERT_TRUE(database_->UpdateStarted(&lists));
 597     database_->InsertChunks(test_case.test_list_name, chunks.get());
 598     database_->UpdateFinished(true);
 599
 600     GetListsInfo(&lists);
 601     ASSERT_LE(1U, lists.size());
 602     EXPECT_EQ(test_case.test_list_name,
 603               lists[test_case.expected_list_index].name);
 604     EXPECT_EQ("1-3,7", lists[test_case.expected_list_index].adds);
 605     EXPECT_TRUE(lists[test_case.expected_list_index].subs.empty());
 606
 607     // Test removing a single prefix from the add chunk.
 608     chunks.clear();
 609     chunks.push_back(SubChunkPrefixValue(4, "www.evil.com/notevil1.html", 2));
 610     ASSERT_TRUE(database_->UpdateStarted(&lists));
 611     database_->InsertChunks(test_case.test_list_name, chunks.get());
 612     database_->UpdateFinished(true);
 613
 614     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 615         GURL("http://www.evil.com/phishing.html"), &prefix_hits, &cache_hits));
 616     ASSERT_EQ(1U, prefix_hits.size());
 617     EXPECT_EQ(SBPrefixForString("www.evil.com/phishing.html"), prefix_hits[0]);
 618     EXPECT_TRUE(cache_hits.empty());
 619
 620     EXPECT_FALSE((database_.get()->*test_case.test_list_contains_bad_url)(
 621         GURL("http://www.evil.com/notevil1.html"), &prefix_hits, &cache_hits));
 622     EXPECT_TRUE(prefix_hits.empty());
 623     EXPECT_TRUE(cache_hits.empty());
 624
 625     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 626         GURL("http://www.evil.com/notevil2.html"), &prefix_hits, &cache_hits));
 627
 628     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 629         GURL("http://www.good.com/good1.html"), &prefix_hits, &cache_hits));
 630
 631     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 632         GURL("http://www.good.com/good2.html"), &prefix_hits, &cache_hits));
 633
 634     GetListsInfo(&lists);
 635     ASSERT_LE(1U, lists.size());
 636     EXPECT_EQ(test_case.test_list_name,
 637               lists[test_case.expected_list_index].name);
 638     EXPECT_EQ("1-3,7", lists[test_case.expected_list_index].adds);
 639     EXPECT_EQ("4", lists[test_case.expected_list_index].subs);
 640
 641     // Test the same sub chunk again.  This should be a no-op.
 642     // see bug: http://code.google.com/p/chromium/issues/detail?id=4522
 643     chunks.clear();
 644     chunks.push_back(SubChunkPrefixValue(4, "www.evil.com/notevil1.html", 2));
 645
 646     ASSERT_TRUE(database_->UpdateStarted(&lists));
 647     database_->InsertChunks(test_case.test_list_name, chunks.get());
 648     database_->UpdateFinished(true);
 649
 650     GetListsInfo(&lists);
 651     ASSERT_LE(1U, lists.size());
 652     EXPECT_EQ(test_case.test_list_name,
 653               lists[test_case.expected_list_index].name);
 654     EXPECT_EQ("1-3,7", lists[test_case.expected_list_index].adds);
 655     EXPECT_EQ("4", lists[test_case.expected_list_index].subs);
 656
 657     // Test removing all the prefixes from an add chunk.
 658     ASSERT_TRUE(database_->UpdateStarted(&lists));
 659     AddDelChunk(test_case.test_list_name, 2);
 660     database_->UpdateFinished(true);
 661
 662     EXPECT_FALSE((database_.get()->*test_case.test_list_contains_bad_url)(
 663         GURL("http://www.evil.com/notevil2.html"), &prefix_hits, &cache_hits));
 664
 665     EXPECT_FALSE((database_.get()->*test_case.test_list_contains_bad_url)(
 666         GURL("http://www.good.com/good1.html"), &prefix_hits, &cache_hits));
 667
 668     EXPECT_FALSE((database_.get()->*test_case.test_list_contains_bad_url)(
 669         GURL("http://www.good.com/good2.html"), &prefix_hits, &cache_hits));
 670
 671     GetListsInfo(&lists);
 672     ASSERT_LE(1U, lists.size());
 673     EXPECT_EQ(test_case.test_list_name,
 674               lists[test_case.expected_list_index].name);
 675     EXPECT_EQ("1,3,7", lists[test_case.expected_list_index].adds);
 676     EXPECT_EQ("4", lists[test_case.expected_list_index].subs);
 677
 678     // The adddel command exposed a bug in the transaction code where any
 679     // transaction after it would fail.  Add a dummy entry and remove it to
 680     // make sure the transcation works fine.
 681     chunks.clear();
 682     chunks.push_back(AddChunkPrefixValue(44, "www.redherring.com/index.html"));
 683     ASSERT_TRUE(database_->UpdateStarted(&lists));
 684     database_->InsertChunks(test_case.test_list_name, chunks.get());
 685
 686     // Now remove the dummy entry.  If there are any problems with the
 687     // transactions, asserts will fire.
 688     AddDelChunk(test_case.test_list_name, 44);
 689
 690     // Test the subdel command.
 691     SubDelChunk(test_case.test_list_name, 4);
 692     database_->UpdateFinished(true);
 693
 694     GetListsInfo(&lists);
 695     ASSERT_LE(1U, lists.size());
 696     EXPECT_EQ(test_case.test_list_name,
 697               lists[test_case.expected_list_index].name);
 698     EXPECT_EQ("1,3,7", lists[test_case.expected_list_index].adds);
 699     EXPECT_TRUE(lists[test_case.expected_list_index].subs.empty());
 700
 701     // Test a sub command coming in before the add.
 702     chunks.clear();
 703     chunks.push_back(SubChunkPrefix2Value(5,
 704                                           "www.notevilanymore.com/index.html",
 705                                           10,
 706                                           "www.notevilanymore.com/good.html",
 707                                           10));
 708     ASSERT_TRUE(database_->UpdateStarted(&lists));
 709     database_->InsertChunks(test_case.test_list_name, chunks.get());
 710     database_->UpdateFinished(true);
 711
 712     EXPECT_FALSE((database_.get()->*test_case.test_list_contains_bad_url)(
 713         GURL("http://www.notevilanymore.com/index.html"),
 714         &prefix_hits,
 715         &cache_hits));
 716
 717     // Now insert the tardy add chunk and we don't expect them to appear
 718     // in database because of the previous sub chunk.
 719     chunks.clear();
 720     chunks.push_back(AddChunkPrefix2Value(10,
 721                                           "www.notevilanymore.com/index.html",
 722                                           "www.notevilanymore.com/good.html"));
 723     ASSERT_TRUE(database_->UpdateStarted(&lists));
 724     database_->InsertChunks(test_case.test_list_name, chunks.get());
 725     database_->UpdateFinished(true);
 726
 727     EXPECT_FALSE((database_.get()->*test_case.test_list_contains_bad_url)(
 728         GURL("http://www.notevilanymore.com/index.html"),
 729         &prefix_hits,
 730         &cache_hits));
 731
 732     EXPECT_FALSE((database_.get()->*test_case.test_list_contains_bad_url)(
 733         GURL("http://www.notevilanymore.com/good.html"),
 734         &prefix_hits,
 735         &cache_hits));
 736
 737     // Reset and reload the database.  The database will rely on the prefix set.
 738     ResetAndReloadFullDatabase();
 739
 740     // Check that a prefix still hits.
 741     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 742         GURL("http://www.evil.com/phishing.html"), &prefix_hits, &cache_hits));
 743     ASSERT_EQ(1U, prefix_hits.size());
 744     EXPECT_EQ(SBPrefixForString("www.evil.com/phishing.html"), prefix_hits[0]);
 745
 746     // Also check that it's not just always returning true in this case.
 747     EXPECT_FALSE((database_.get()->*test_case.test_list_contains_bad_url)(
 748         GURL("http://www.evil.com/"), &prefix_hits, &cache_hits));
 749
 750     // Check that the full hash is still present.
 751     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 752         GURL("http://www.evil.com/evil.html"), &prefix_hits, &cache_hits));
 753     ASSERT_EQ(1U, prefix_hits.size());
 754     EXPECT_EQ(SBPrefixForString("www.evil.com/evil.html"), prefix_hits[0]);
 755   }
 756 }
 757
 758 // Test adding zero length chunks to the database.
 759 TEST_F(SafeBrowsingDatabaseTest, ZeroSizeChunk) {
 760   std::vector<SBListChunkRanges> lists;
 761   ScopedVector<SBChunkData> chunks;
 762
 763   // Populate with a couple of normal chunks.
 764   chunks.push_back(AddChunkPrefix2Value(1,
 765                                         "www.test.com/test1.html",
 766                                         "www.test.com/test2.html"));
 767   chunks.push_back(AddChunkPrefix2Value(10,
 768                                         "www.random.com/random1.html",
 769                                         "www.random.com/random2.html"));
 770
 771   ASSERT_TRUE(database_->UpdateStarted(&lists));
 772   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
 773   database_->UpdateFinished(true);
 774
 775   // Add an empty ADD and SUB chunk.
 776   GetListsInfo(&lists);
 777   ASSERT_LE(1U, lists.size());
 778   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
 779   EXPECT_EQ("1,10", lists[0].adds);
 780   EXPECT_TRUE(lists[0].subs.empty());
 781
 782   chunks.clear();
 783   chunks.push_back(BuildChunk(19, safe_browsing::ChunkData::ADD,
 784                               safe_browsing::ChunkData::PREFIX_4B,
 785                               NULL, 0, std::vector<int>()));
 786   chunks.push_back(BuildChunk(7, safe_browsing::ChunkData::SUB,
 787                               safe_browsing::ChunkData::PREFIX_4B,
 788                               NULL, 0, std::vector<int>()));
 789
 790   ASSERT_TRUE(database_->UpdateStarted(&lists));
 791   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
 792   database_->UpdateFinished(true);
 793
 794   GetListsInfo(&lists);
 795   ASSERT_LE(1U, lists.size());
 796   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
 797   EXPECT_EQ("1,10,19", lists[0].adds);
 798   EXPECT_EQ("7", lists[0].subs);
 799
 800   // Add an empty chunk along with a couple that contain data. This should
 801   // result in the chunk range being reduced in size.
 802   chunks.clear();
 803   chunks.push_back(AddChunkPrefixValue(20, "www.notempty.com/full1.html"));
 804   chunks.push_back(BuildChunk(21, safe_browsing::ChunkData::ADD,
 805                               safe_browsing::ChunkData::PREFIX_4B,
 806                               NULL, 0, std::vector<int>()));
 807   chunks.push_back(AddChunkPrefixValue(22, "www.notempty.com/full2.html"));
 808
 809   ASSERT_TRUE(database_->UpdateStarted(&lists));
 810   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
 811   database_->UpdateFinished(true);
 812
 813   std::vector<SBPrefix> prefix_hits;
 814   std::vector<SBFullHashResult> cache_hits;
 815   EXPECT_TRUE(database_->ContainsBrowseUrl(
 816       GURL("http://www.notempty.com/full1.html"), &prefix_hits, &cache_hits));
 817   EXPECT_TRUE(database_->ContainsBrowseUrl(
 818       GURL("http://www.notempty.com/full2.html"), &prefix_hits, &cache_hits));
 819
 820   GetListsInfo(&lists);
 821   ASSERT_LE(1U, lists.size());
 822   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
 823   EXPECT_EQ("1,10,19-22", lists[0].adds);
 824   EXPECT_EQ("7", lists[0].subs);
 825
 826   // Handle AddDel and SubDel commands for empty chunks.
 827   ASSERT_TRUE(database_->UpdateStarted(&lists));
 828   AddDelChunk(safe_browsing_util::kMalwareList, 21);
 829   database_->UpdateFinished(true);
 830
 831   GetListsInfo(&lists);
 832   ASSERT_LE(1U, lists.size());
 833   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
 834   EXPECT_EQ("1,10,19-20,22", lists[0].adds);
 835   EXPECT_EQ("7", lists[0].subs);
 836
 837   ASSERT_TRUE(database_->UpdateStarted(&lists));
 838   SubDelChunk(safe_browsing_util::kMalwareList, 7);
 839   database_->UpdateFinished(true);
 840
 841   GetListsInfo(&lists);
 842   ASSERT_LE(1U, lists.size());
 843   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
 844   EXPECT_EQ("1,10,19-20,22", lists[0].adds);
 845   EXPECT_TRUE(lists[0].subs.empty());
 846 }
 847
 848 // Utility function for setting up the database for the caching test.
 849 void SafeBrowsingDatabaseTest::PopulateDatabaseForCacheTest() {
 850   // Add a couple prefixes.
 851   ScopedVector<SBChunkData> chunks;
 852   chunks.push_back(AddChunkPrefix2Value(1,
 853                                         "www.evil.com/phishing.html",
 854                                         "www.evil.com/malware.html"));
 855
 856   std::vector<SBListChunkRanges> lists;
 857   ASSERT_TRUE(database_->UpdateStarted(&lists));
 858   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
 859   database_->UpdateFinished(true);
 860
 861   // Cache should be cleared after updating.
 862   EXPECT_TRUE(
 863       database_->GetUnsynchronizedPrefixGetHashCacheForTesting()->empty());
 864
 865   SBFullHashResult full_hash;
 866   full_hash.list_id = safe_browsing_util::MALWARE;
 867
 868   std::vector<SBFullHashResult> results;
 869   std::vector<SBPrefix> prefixes;
 870
 871   // Add a fullhash result for each prefix.
 872   full_hash.hash = SBFullHashForString("www.evil.com/phishing.html");
 873   results.push_back(full_hash);
 874   prefixes.push_back(full_hash.hash.prefix);
 875
 876   full_hash.hash = SBFullHashForString("www.evil.com/malware.html");
 877   results.push_back(full_hash);
 878   prefixes.push_back(full_hash.hash.prefix);
 879
 880   database_->CacheHashResults(prefixes, results, kCacheLifetime);
 881 }
 882
 883 TEST_F(SafeBrowsingDatabaseTest, HashCaching) {
 884   PopulateDatabaseForCacheTest();
 885
 886   // We should have both full hashes in the cache.
 887   EXPECT_EQ(2U,
 888             database_->GetUnsynchronizedPrefixGetHashCacheForTesting()->size());
 889
 890   // Test the cache lookup for the first prefix.
 891   std::vector<SBPrefix> prefix_hits;
 892   std::vector<SBFullHashResult> cache_hits;
 893   EXPECT_TRUE(database_->ContainsBrowseUrl(
 894       GURL("http://www.evil.com/phishing.html"), &prefix_hits, &cache_hits));
 895   EXPECT_TRUE(prefix_hits.empty());
 896   ASSERT_EQ(1U, cache_hits.size());
 897   EXPECT_TRUE(SBFullHashEqual(
 898       cache_hits[0].hash, SBFullHashForString("www.evil.com/phishing.html")));
 899
 900   prefix_hits.clear();
 901   cache_hits.clear();
 902
 903   // Test the cache lookup for the second prefix.
 904   EXPECT_TRUE(database_->ContainsBrowseUrl(
 905       GURL("http://www.evil.com/malware.html"), &prefix_hits, &cache_hits));
 906   EXPECT_TRUE(prefix_hits.empty());
 907   ASSERT_EQ(1U, cache_hits.size());
 908   EXPECT_TRUE(SBFullHashEqual(
 909       cache_hits[0].hash, SBFullHashForString("www.evil.com/malware.html")));
 910
 911   prefix_hits.clear();
 912   cache_hits.clear();
 913
 914   // Test removing a prefix via a sub chunk.
 915   ScopedVector<SBChunkData> chunks;
 916   chunks.push_back(SubChunkPrefixValue(2, "www.evil.com/phishing.html", 1));
 917
 918   std::vector<SBListChunkRanges> lists;
 919   ASSERT_TRUE(database_->UpdateStarted(&lists));
 920   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
 921   database_->UpdateFinished(true);
 922
 923   // This prefix should still be there, but cached fullhash should be gone.
 924   EXPECT_TRUE(database_->ContainsBrowseUrl(
 925       GURL("http://www.evil.com/malware.html"), &prefix_hits, &cache_hits));
 926   ASSERT_EQ(1U, prefix_hits.size());
 927   EXPECT_EQ(SBPrefixForString("www.evil.com/malware.html"), prefix_hits[0]);
 928   EXPECT_TRUE(cache_hits.empty());
 929   prefix_hits.clear();
 930   cache_hits.clear();
 931
 932   // This prefix should be gone.
 933   EXPECT_FALSE(database_->ContainsBrowseUrl(
 934       GURL("http://www.evil.com/phishing.html"), &prefix_hits, &cache_hits));
 935   prefix_hits.clear();
 936   cache_hits.clear();
 937
 938   // Test that an AddDel for the original chunk removes the last cached entry.
 939   ASSERT_TRUE(database_->UpdateStarted(&lists));
 940   AddDelChunk(safe_browsing_util::kMalwareList, 1);
 941   database_->UpdateFinished(true);
 942   EXPECT_FALSE(database_->ContainsBrowseUrl(
 943       GURL("http://www.evil.com/malware.html"), &prefix_hits, &cache_hits));
 944   EXPECT_TRUE(
 945       database_->GetUnsynchronizedPrefixGetHashCacheForTesting()->empty());
 946   prefix_hits.clear();
 947   cache_hits.clear();
 948
 949   // Test that the cache won't return expired values. First we have to adjust
 950   // the cached entries' received time to make them older, since the database
 951   // cache insert uses Time::Now(). First, store some entries.
 952   PopulateDatabaseForCacheTest();
 953
 954   SafeBrowsingDatabaseNew::PrefixGetHashCache* hash_cache =
 955       database_->GetUnsynchronizedPrefixGetHashCacheForTesting();
 956   EXPECT_EQ(2U, hash_cache->size());
 957
 958   // Now adjust one of the entries times to be in the past.
 959   const SBPrefix key = SBPrefixForString("www.evil.com/malware.html");
 960   SafeBrowsingDatabaseNew::PrefixGetHashCache::iterator iter =
 961       hash_cache->find(key);
 962   ASSERT_TRUE(iter != hash_cache->end());
 963   iter->second.expire_after = Time::Now() - TimeDelta::FromMinutes(1);
 964
 965   EXPECT_TRUE(database_->ContainsBrowseUrl(
 966       GURL("http://www.evil.com/malware.html"), &prefix_hits, &cache_hits));
 967   EXPECT_EQ(1U, prefix_hits.size());
 968   EXPECT_TRUE(cache_hits.empty());
 969   // Expired entry should have been removed from cache.
 970   EXPECT_EQ(1U, hash_cache->size());
 971
 972   // This entry should still exist.
 973   EXPECT_TRUE(database_->ContainsBrowseUrl(
 974       GURL("http://www.evil.com/phishing.html"), &prefix_hits, &cache_hits));
 975   EXPECT_TRUE(prefix_hits.empty());
 976   EXPECT_EQ(1U, cache_hits.size());
 977
 978   // Testing prefix miss caching. First, we clear out the existing database,
 979   // Since PopulateDatabaseForCacheTest() doesn't handle adding duplicate
 980   // chunks.
 981   ASSERT_TRUE(database_->UpdateStarted(&lists));
 982   AddDelChunk(safe_browsing_util::kMalwareList, 1);
 983   database_->UpdateFinished(true);
 984
 985   // Cache should be cleared after updating.
 986   EXPECT_TRUE(hash_cache->empty());
 987
 988   std::vector<SBPrefix> prefix_misses;
 989   std::vector<SBFullHashResult> empty_full_hash;
 990   prefix_misses.push_back(SBPrefixForString("http://www.bad.com/malware.html"));
 991   prefix_misses.push_back(
 992       SBPrefixForString("http://www.bad.com/phishing.html"));
 993   database_->CacheHashResults(prefix_misses, empty_full_hash, kCacheLifetime);
 994
 995   // Prefixes with no full results are misses.
 996   EXPECT_EQ(hash_cache->size(), prefix_misses.size());
 997   ASSERT_TRUE(
 998       hash_cache->count(SBPrefixForString("http://www.bad.com/malware.html")));
 999   EXPECT_TRUE(
1000       hash_cache->find(SBPrefixForString("http://www.bad.com/malware.html"))
1001           ->second.full_hashes.empty());
1002   ASSERT_TRUE(
1003       hash_cache->count(SBPrefixForString("http://www.bad.com/phishing.html")));
1004   EXPECT_TRUE(
1005       hash_cache->find(SBPrefixForString("http://www.bad.com/phishing.html"))
1006           ->second.full_hashes.empty());
1007
1008   // Update the database.
1009   PopulateDatabaseForCacheTest();
1010
1011   // Cache a GetHash miss for a particular prefix, and even though the prefix is
1012   // in the database, it is flagged as a miss so looking up the associated URL
1013   // will not succeed.
1014   prefix_hits.clear();
1015   cache_hits.clear();
1016   prefix_misses.clear();
1017   empty_full_hash.clear();
1018   prefix_misses.push_back(SBPrefixForString("www.evil.com/phishing.html"));
1019   database_->CacheHashResults(prefix_misses, empty_full_hash, kCacheLifetime);
1020   EXPECT_FALSE(database_->ContainsBrowseUrl(
1021       GURL("http://www.evil.com/phishing.html"), &prefix_hits, &cache_hits));
1022   prefix_hits.clear();
1023   cache_hits.clear();
1024
1025   // Test receiving a full add chunk.
1026   chunks.clear();
1027   chunks.push_back(AddChunkFullHash2Value(20,
1028                                           "www.fullevil.com/bad1.html",
1029                                           "www.fullevil.com/bad2.html"));
1030   ASSERT_TRUE(database_->UpdateStarted(&lists));
1031   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1032   database_->UpdateFinished(true);
1033
1034   EXPECT_TRUE(database_->ContainsBrowseUrl(
1035       GURL("http://www.fullevil.com/bad1.html"), &prefix_hits, &cache_hits));
1036   ASSERT_EQ(1U, prefix_hits.size());
1037   EXPECT_EQ(SBPrefixForString("www.fullevil.com/bad1.html"), prefix_hits[0]);
1038   EXPECT_TRUE(cache_hits.empty());
1039   prefix_hits.clear();
1040   cache_hits.clear();
1041
1042   EXPECT_TRUE(database_->ContainsBrowseUrl(
1043       GURL("http://www.fullevil.com/bad2.html"), &prefix_hits, &cache_hits));
1044   ASSERT_EQ(1U, prefix_hits.size());
1045   EXPECT_EQ(SBPrefixForString("www.fullevil.com/bad2.html"), prefix_hits[0]);
1046   EXPECT_TRUE(cache_hits.empty());
1047   prefix_hits.clear();
1048   cache_hits.clear();
1049
1050   // Test receiving a full sub chunk, which will remove one of the full adds.
1051   chunks.clear();
1052   chunks.push_back(SubChunkFullHashValue(200,
1053                                          "www.fullevil.com/bad1.html",
1054                                          20));
1055   ASSERT_TRUE(database_->UpdateStarted(&lists));
1056   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1057   database_->UpdateFinished(true);
1058
1059   EXPECT_FALSE(database_->ContainsBrowseUrl(
1060       GURL("http://www.fullevil.com/bad1.html"), &prefix_hits, &cache_hits));
1061
1062   // There should be one remaining full add.
1063   EXPECT_TRUE(database_->ContainsBrowseUrl(
1064       GURL("http://www.fullevil.com/bad2.html"), &prefix_hits, &cache_hits));
1065   ASSERT_EQ(1U, prefix_hits.size());
1066   EXPECT_EQ(SBPrefixForString("www.fullevil.com/bad2.html"), prefix_hits[0]);
1067   EXPECT_TRUE(cache_hits.empty());
1068   prefix_hits.clear();
1069   cache_hits.clear();
1070
1071   // Now test an AddDel for the remaining full add.
1072   ASSERT_TRUE(database_->UpdateStarted(&lists));
1073   AddDelChunk(safe_browsing_util::kMalwareList, 20);
1074   database_->UpdateFinished(true);
1075
1076   EXPECT_FALSE(database_->ContainsBrowseUrl(
1077       GURL("http://www.fullevil.com/bad1.html"), &prefix_hits, &cache_hits));
1078   EXPECT_FALSE(database_->ContainsBrowseUrl(
1079       GURL("http://www.fullevil.com/bad2.html"), &prefix_hits, &cache_hits));
1080
1081   // Add a fullhash which has a prefix collision for a known url.
1082   static const char kExampleFine[] = "www.example.com/fine.html";
1083   static const char kExampleCollision[] =
1084       "www.example.com/3123364814/malware.htm";
1085   ASSERT_EQ(SBPrefixForString(kExampleFine),
1086             SBPrefixForString(kExampleCollision));
1087   ASSERT_TRUE(database_->UpdateStarted(&lists));
1088   {
1089     ScopedVector<SBChunkData> chunks;
1090     chunks.push_back(AddChunkPrefixValue(21, kExampleCollision));
1091     database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1092   }
1093   database_->UpdateFinished(true);
1094
1095   // Expect a prefix hit due to the collision between |kExampleFine| and
1096   // |kExampleCollision|.
1097   EXPECT_TRUE(database_->ContainsBrowseUrl(
1098       GURL(std::string("http://") + kExampleFine), &prefix_hits, &cache_hits));
1099   ASSERT_EQ(1U, prefix_hits.size());
1100   EXPECT_EQ(SBPrefixForString(kExampleFine), prefix_hits[0]);
1101   EXPECT_TRUE(cache_hits.empty());
1102
1103   // Cache gethash response for |kExampleCollision|.
1104   {
1105     SBFullHashResult result;
1106     result.hash = SBFullHashForString(kExampleCollision);
1107     result.list_id = safe_browsing_util::MALWARE;
1108     database_->CacheHashResults(std::vector<SBPrefix>(1, result.hash.prefix),
1109                                 std::vector<SBFullHashResult>(1, result),
1110                                 kCacheLifetime);
1111   }
1112
1113   // The cached response means the collision no longer causes a hit.
1114   EXPECT_FALSE(database_->ContainsBrowseUrl(
1115       GURL(std::string("http://") + kExampleFine), &prefix_hits, &cache_hits));
1116 }
1117
1118 // Test that corrupt databases are appropriately handled, even if the
1119 // corruption is detected in the midst of the update.
1120 // TODO(shess): Disabled until ScopedLogMessageIgnorer resolved.
1121 // http://crbug.com/56448
1122 TEST_F(SafeBrowsingDatabaseTest, DISABLED_FileCorruptionHandling) {
1123   // Re-create the database in a captive message loop so that we can
1124   // influence task-posting.  Database specifically needs to the
1125   // file-backed.
1126   database_.reset();
1127   base::MessageLoop loop;
1128   SafeBrowsingStoreFile* store = new SafeBrowsingStoreFile(task_runner_);
1129   database_.reset(new SafeBrowsingDatabaseNew(
1130       task_runner_, store, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL));
1131   database_->Init(database_filename_);
1132
1133   // This will cause an empty database to be created.
1134   std::vector<SBListChunkRanges> lists;
1135   ASSERT_TRUE(database_->UpdateStarted(&lists));
1136   database_->UpdateFinished(true);
1137
1138   // Create a sub chunk to insert.
1139   ScopedVector<SBChunkData> chunks;
1140   chunks.push_back(SubChunkPrefixValue(7,
1141                                        "www.subbed.com/notevil1.html",
1142                                        19));
1143
1144   // Corrupt the file by corrupting the checksum, which is not checked
1145   // until the entire table is read in |UpdateFinished()|.
1146   FILE* fp = base::OpenFile(database_filename_, "r+");
1147   ASSERT_TRUE(fp);
1148   ASSERT_NE(-1, fseek(fp, -8, SEEK_END));
1149   for (size_t i = 0; i < 8; ++i) {
1150     fputc('!', fp);
1151   }
1152   fclose(fp);
1153
1154   {
1155     // The following code will cause DCHECKs, so suppress the crashes.
1156     ScopedLogMessageIgnorer ignorer;
1157
1158     // Start an update.  The insert will fail due to corruption.
1159     ASSERT_TRUE(database_->UpdateStarted(&lists));
1160     database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1161     database_->UpdateFinished(true);
1162
1163     // Database file still exists until the corruption handler has run.
1164     EXPECT_TRUE(base::PathExists(database_filename_));
1165
1166     // Flush through the corruption-handler task.
1167     DVLOG(1) << "Expect failed check on: SafeBrowsing database reset";
1168     base::MessageLoop::current()->RunUntilIdle();
1169   }
1170
1171   // Database file should not exist.
1172   EXPECT_FALSE(base::PathExists(database_filename_));
1173
1174   // Run the update again successfully.
1175   ASSERT_TRUE(database_->UpdateStarted(&lists));
1176   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1177   database_->UpdateFinished(true);
1178   EXPECT_TRUE(base::PathExists(database_filename_));
1179
1180   database_.reset();
1181 }
1182
1183 // Checks database reading and writing.
1184 TEST_F(SafeBrowsingDatabaseTest, ContainsDownloadUrlPrefixes) {
1185   const char kEvil1Url1[] = "www.evil1.com/download1/";
1186   const char kEvil1Url2[] = "www.evil1.com/download2.html";
1187
1188   // Add a simple chunk with one hostkey for download url list.
1189   ScopedVector<SBChunkData> chunks;
1190   chunks.push_back(AddChunkPrefix2Value(1, kEvil1Url1, kEvil1Url2));
1191
1192   std::vector<SBListChunkRanges> lists;
1193   ASSERT_TRUE(database_->UpdateStarted(&lists));
1194   database_->InsertChunks(safe_browsing_util::kBinUrlList, chunks.get());
1195   database_->UpdateFinished(true);
1196
1197   std::vector<SBPrefix> prefix_hits;
1198   std::vector<GURL> urls(1);
1199
1200   urls[0] = GURL(std::string("http://") + kEvil1Url1);
1201   EXPECT_TRUE(ContainsDownloadUrl(urls, &prefix_hits));
1202   ASSERT_EQ(1U, prefix_hits.size());
1203   EXPECT_EQ(SBPrefixForString(kEvil1Url1), prefix_hits[0]);
1204
1205   urls[0] = GURL(std::string("http://") + kEvil1Url2);
1206   EXPECT_TRUE(ContainsDownloadUrl(urls, &prefix_hits));
1207   ASSERT_EQ(1U, prefix_hits.size());
1208   EXPECT_EQ(SBPrefixForString(kEvil1Url2), prefix_hits[0]);
1209
1210   urls[0] = GURL(std::string("https://") + kEvil1Url2);
1211   EXPECT_TRUE(ContainsDownloadUrl(urls, &prefix_hits));
1212   ASSERT_EQ(1U, prefix_hits.size());
1213   EXPECT_EQ(SBPrefixForString(kEvil1Url2), prefix_hits[0]);
1214
1215   urls[0] = GURL(std::string("ftp://") + kEvil1Url2);
1216   EXPECT_TRUE(ContainsDownloadUrl(urls, &prefix_hits));
1217   ASSERT_EQ(1U, prefix_hits.size());
1218   EXPECT_EQ(SBPrefixForString(kEvil1Url2), prefix_hits[0]);
1219
1220   urls[0] = GURL("http://www.randomevil.com");
1221   EXPECT_FALSE(ContainsDownloadUrl(urls, &prefix_hits));
1222
1223   // Should match with query args stripped.
1224   urls[0] = GURL(std::string("http://") + kEvil1Url2 + "?blah");
1225   EXPECT_TRUE(ContainsDownloadUrl(urls, &prefix_hits));
1226   ASSERT_EQ(1U, prefix_hits.size());
1227   EXPECT_EQ(SBPrefixForString(kEvil1Url2), prefix_hits[0]);
1228
1229   // Should match with extra path stuff and query args stripped.
1230   urls[0] = GURL(std::string("http://") + kEvil1Url1 + "foo/bar?blah");
1231   EXPECT_TRUE(ContainsDownloadUrl(urls, &prefix_hits));
1232   ASSERT_EQ(1U, prefix_hits.size());
1233   EXPECT_EQ(SBPrefixForString(kEvil1Url1), prefix_hits[0]);
1234
1235   // First hit in redirect chain is malware.
1236   urls.clear();
1237   urls.push_back(GURL(std::string("http://") + kEvil1Url1));
1238   urls.push_back(GURL("http://www.randomevil.com"));
1239   EXPECT_TRUE(ContainsDownloadUrl(urls, &prefix_hits));
1240   ASSERT_EQ(1U, prefix_hits.size());
1241   EXPECT_EQ(SBPrefixForString(kEvil1Url1), prefix_hits[0]);
1242
1243   // Middle hit in redirect chain is malware.
1244   urls.clear();
1245   urls.push_back(GURL("http://www.randomevil.com"));
1246   urls.push_back(GURL(std::string("http://") + kEvil1Url1));
1247   urls.push_back(GURL("http://www.randomevil2.com"));
1248   EXPECT_TRUE(ContainsDownloadUrl(urls, &prefix_hits));
1249   ASSERT_EQ(1U, prefix_hits.size());
1250   EXPECT_EQ(SBPrefixForString(kEvil1Url1), prefix_hits[0]);
1251
1252   // Final hit in redirect chain is malware.
1253   urls.clear();
1254   urls.push_back(GURL("http://www.randomevil.com"));
1255   urls.push_back(GURL(std::string("http://") + kEvil1Url1));
1256   EXPECT_TRUE(ContainsDownloadUrl(urls, &prefix_hits));
1257   ASSERT_EQ(1U, prefix_hits.size());
1258   EXPECT_EQ(SBPrefixForString(kEvil1Url1), prefix_hits[0]);
1259
1260   // Multiple hits in redirect chain are in malware list.
1261   urls.clear();
1262   urls.push_back(GURL(std::string("http://") + kEvil1Url1));
1263   urls.push_back(GURL(std::string("https://") + kEvil1Url2));
1264   EXPECT_TRUE(ContainsDownloadUrl(urls, &prefix_hits));
1265   ASSERT_EQ(2U, prefix_hits.size());
1266   EXPECT_EQ(SBPrefixForString(kEvil1Url1), prefix_hits[0]);
1267   EXPECT_EQ(SBPrefixForString(kEvil1Url2), prefix_hits[1]);
1268   database_.reset();
1269 }
1270
1271 // Checks that the whitelists are handled properly.
1272 TEST_F(SafeBrowsingDatabaseTest, Whitelists) {
1273   struct TestCase {
1274     using TestListContainsWhitelistedUrl =
1275         bool (SafeBrowsingDatabase::*)(const GURL& url);
1276     using TestListContainsWhitelistedString =
1277         bool (SafeBrowsingDatabase::*)(const std::string& str);
1278
1279     // Returns true if strings should be tested in this test case (i.e.
1280     // |test_list_contains_whitelisted_string| is not null).
1281     bool TestStrings() const {
1282       return test_list_contains_whitelisted_string != nullptr;
1283     }
1284
1285     const char* test_list_name;
1286     TestListContainsWhitelistedUrl test_list_contains_whitelisted_url;
1287     // Optional test case field, if set the tested whitelist will also be tested
1288     // for strings.
1289     TestListContainsWhitelistedString test_list_contains_whitelisted_string;
1290   } const kTestCases[]{
1291       {safe_browsing_util::kCsdWhiteList,
1292        &SafeBrowsingDatabase::ContainsCsdWhitelistedUrl,
1293        nullptr},
1294       {safe_browsing_util::kDownloadWhiteList,
1295        &SafeBrowsingDatabase::ContainsDownloadWhitelistedUrl,
1296        &SafeBrowsingDatabase::ContainsDownloadWhitelistedString},
1297       {safe_browsing_util::kInclusionWhitelist,
1298        &SafeBrowsingDatabase::ContainsInclusionWhitelistedUrl,
1299        nullptr},
1300   };
1301
1302   // If the whitelist is disabled everything should match the whitelist.
1303   database_.reset(new SafeBrowsingDatabaseNew(
1304       task_runner_, new SafeBrowsingStoreFile(task_runner_), NULL, NULL, NULL,
1305       NULL, NULL, NULL, NULL, NULL));
1306   database_->Init(database_filename_);
1307   for (const auto& test_case : kTestCases) {
1308     SCOPED_TRACE(std::string("Tested list at fault => ") +
1309                  test_case.test_list_name);
1310
1311     EXPECT_TRUE(
1312         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1313             GURL(std::string("http://www.phishing.com/"))));
1314     if (test_case.TestStrings()) {
1315       EXPECT_TRUE(
1316           (database_.get()->*test_case.test_list_contains_whitelisted_string)(
1317               "asdf"));
1318     }
1319   }
1320   ResetAndReloadFullDatabase();
1321
1322   // Now test every whitelist one-by-one; intentionally not resetting the
1323   // database in-between to further stress potential inter-dependencies.
1324   for (const auto& test_case : kTestCases) {
1325     SCOPED_TRACE(std::string("Tested list at fault => ") +
1326                  test_case.test_list_name);
1327
1328     const char kGood1Host[] = "www.good1.com/";
1329     const char kGood1Url1[] = "www.good1.com/a/b.html";
1330     const char kGood1Url2[] = "www.good1.com/b/";
1331
1332     const char kGood2Url1[] = "www.good2.com/c";  // Should match '/c/bla'.
1333
1334     // good3.com/a/b/c/d/e/f/g/ should match because it's a whitelist.
1335     const char kGood3Url1[] = "good3.com/";
1336
1337     const char kGoodString[] = "good_string";
1338
1339     // Nothing should be whitelisted before the database receives the chunks.
1340     EXPECT_FALSE(
1341         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1342             GURL(std::string("http://") + kGood1Host)));
1343     EXPECT_FALSE(
1344         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1345             GURL(std::string("http://") + kGood1Url1)));
1346     EXPECT_FALSE(
1347         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1348             GURL(std::string("http://") + kGood1Url2)));
1349     EXPECT_FALSE(
1350         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1351             GURL(std::string("http://") + kGood2Url1)));
1352     EXPECT_FALSE(
1353         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1354             GURL(std::string("http://") + kGood3Url1)));
1355     if (test_case.TestStrings()) {
1356       EXPECT_FALSE(
1357           (database_.get()->*test_case.test_list_contains_whitelisted_string)(
1358               kGoodString));
1359     }
1360
1361     ScopedVector<SBChunkData> chunks;
1362
1363     // Add a few test chunks to the whitelist under test.
1364     chunks.push_back(AddChunkFullHash2Value(1, kGood1Url1, kGood1Url2));
1365     chunks.push_back(AddChunkFullHashValue(2, kGood2Url1));
1366     if (test_case.TestStrings())
1367       chunks.push_back(AddChunkFullHashValue(3, kGoodString));
1368     chunks.push_back(AddChunkFullHashValue(4, kGood3Url1));
1369
1370     std::vector<SBListChunkRanges> lists;
1371     ASSERT_TRUE(database_->UpdateStarted(&lists));
1372     database_->InsertChunks(test_case.test_list_name, chunks.get());
1373     database_->UpdateFinished(true);
1374
1375     EXPECT_FALSE(
1376         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1377             GURL(std::string("http://") + kGood1Host)));
1378
1379     EXPECT_TRUE(
1380         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1381             GURL(std::string("http://") + kGood1Url1)));
1382     EXPECT_TRUE(
1383         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1384             GURL(std::string("http://") + kGood1Url1 + "?a=b")));
1385
1386     EXPECT_TRUE(
1387         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1388             GURL(std::string("http://") + kGood1Url2)));
1389     EXPECT_TRUE(
1390         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1391             GURL(std::string("http://") + kGood1Url2 + "/c.html")));
1392
1393     EXPECT_TRUE(
1394         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1395             GURL(std::string("https://") + kGood1Url2 + "/c.html")));
1396
1397     EXPECT_TRUE(
1398         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1399             GURL(std::string("http://") + kGood2Url1 + "/c")));
1400     EXPECT_TRUE(
1401         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1402             GURL(std::string("http://") + kGood2Url1 + "/c?bla")));
1403     EXPECT_TRUE(
1404         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1405             GURL(std::string("http://") + kGood2Url1 + "/c/bla")));
1406
1407     EXPECT_FALSE(
1408         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1409             GURL(std::string("http://www.google.com/"))));
1410
1411     EXPECT_TRUE(
1412         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1413             GURL(std::string("http://") + kGood3Url1 + "a/b/c/d/e/f/g/")));
1414     EXPECT_TRUE(
1415         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1416             GURL(std::string("http://a.b.") + kGood3Url1)));
1417
1418     if (test_case.TestStrings()) {
1419       EXPECT_FALSE(
1420           (database_.get()->*test_case.test_list_contains_whitelisted_string)(
1421               "asdf"));
1422       EXPECT_TRUE(
1423           (database_.get()->*test_case.test_list_contains_whitelisted_string)(
1424               kGoodString));
1425     }
1426
1427     EXPECT_FALSE(
1428         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1429             GURL(std::string("http://www.google.com/"))));
1430
1431     // The malware kill switch is for the CSD whitelist only.
1432     if (test_case.test_list_name == safe_browsing_util::kCsdWhiteList) {
1433       // The CSD whitelist killswitch is not present.
1434       EXPECT_FALSE(database_->IsCsdWhitelistKillSwitchOn());
1435
1436       // Test only add the malware IP killswitch
1437       chunks.clear();
1438       chunks.push_back(AddChunkFullHashValue(
1439           15, "sb-ssl.google.com/safebrowsing/csd/killswitch_malware"));
1440
1441       ASSERT_TRUE(database_->UpdateStarted(&lists));
1442       database_->InsertChunks(safe_browsing_util::kCsdWhiteList, chunks.get());
1443       database_->UpdateFinished(true);
1444
1445       EXPECT_TRUE(database_->IsMalwareIPMatchKillSwitchOn());
1446       // The CSD whitelist killswitch is not present.
1447       EXPECT_FALSE(database_->IsCsdWhitelistKillSwitchOn());
1448     }
1449
1450     // Test that the generic whitelist kill-switch works as intended.
1451     chunks.clear();
1452     lists.clear();
1453     chunks.push_back(AddChunkFullHashValue(
1454         5, "sb-ssl.google.com/safebrowsing/csd/killswitch"));
1455
1456     ASSERT_TRUE(database_->UpdateStarted(&lists));
1457     database_->InsertChunks(test_case.test_list_name, chunks.get());
1458     database_->UpdateFinished(true);
1459
1460     // Test CSD whitelist specific methods.
1461     if (test_case.test_list_name == safe_browsing_util::kCsdWhiteList) {
1462       // The CSD whitelist killswitch is present.
1463       EXPECT_TRUE(database_->IsCsdWhitelistKillSwitchOn());
1464       EXPECT_TRUE(database_->IsMalwareIPMatchKillSwitchOn());
1465     }
1466
1467     EXPECT_TRUE(
1468         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1469             GURL(std::string("https://") + kGood1Url2 + "/c.html")));
1470     EXPECT_TRUE(
1471         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1472             GURL(std::string("http://www.google.com/"))));
1473     EXPECT_TRUE(
1474         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1475             GURL(std::string("http://www.phishing_url.com/"))));
1476
1477     if (test_case.TestStrings()) {
1478       EXPECT_TRUE(
1479           (database_.get()->*test_case.test_list_contains_whitelisted_string)(
1480               "asdf"));
1481       EXPECT_TRUE(
1482           (database_.get()->*test_case.test_list_contains_whitelisted_string)(
1483               kGoodString));
1484     }
1485
1486     // Remove the kill-switch and verify that we can recover.
1487     chunks.clear();
1488     lists.clear();
1489     chunks.push_back(SubChunkFullHashValue(
1490         1, "sb-ssl.google.com/safebrowsing/csd/killswitch", 5));
1491     if (test_case.test_list_name == safe_browsing_util::kCsdWhiteList) {
1492       chunks.push_back(SubChunkFullHashValue(
1493           10, "sb-ssl.google.com/safebrowsing/csd/killswitch_malware", 15));
1494     }
1495
1496     ASSERT_TRUE(database_->UpdateStarted(&lists));
1497     database_->InsertChunks(test_case.test_list_name, chunks.get());
1498     database_->UpdateFinished(true);
1499
1500     if (test_case.test_list_name == safe_browsing_util::kCsdWhiteList) {
1501       EXPECT_FALSE(database_->IsMalwareIPMatchKillSwitchOn());
1502       EXPECT_FALSE(database_->IsCsdWhitelistKillSwitchOn());
1503     }
1504     EXPECT_TRUE(
1505         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1506             GURL(std::string("https://") + kGood1Url2 + "/c.html")));
1507     EXPECT_TRUE(
1508         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1509             GURL(std::string("https://") + kGood2Url1 + "/c/bla")));
1510     EXPECT_TRUE(
1511         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1512             GURL(std::string("https://") + kGood3Url1)));
1513     EXPECT_FALSE(
1514         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1515             GURL(std::string("http://www.google.com/"))));
1516     EXPECT_FALSE(
1517         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1518             GURL(std::string("http://www.phishing_url.com/"))));
1519     if (test_case.TestStrings()) {
1520       EXPECT_TRUE(
1521           (database_.get()->*test_case.test_list_contains_whitelisted_string)(
1522               kGoodString));
1523       EXPECT_FALSE(
1524           (database_.get()->*test_case.test_list_contains_whitelisted_string)(
1525               "asdf"));
1526     }
1527   }
1528 }
1529
1530 // Test to make sure we could insert chunk list that
1531 // contains entries for the same host.
1532 TEST_F(SafeBrowsingDatabaseTest, SameHostEntriesOkay) {
1533   ScopedVector<SBChunkData> chunks;
1534
1535   // Add a malware add chunk with two entries of the same host.
1536   chunks.push_back(AddChunkPrefix2Value(1,
1537                                         "www.evil.com/malware1.html",
1538                                         "www.evil.com/malware2.html"));
1539
1540   // Insert the testing chunks into database.
1541   std::vector<SBListChunkRanges> lists;
1542   ASSERT_TRUE(database_->UpdateStarted(&lists));
1543   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1544   database_->UpdateFinished(true);
1545
1546   GetListsInfo(&lists);
1547   ASSERT_LE(1U, lists.size());
1548   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
1549   EXPECT_EQ("1", lists[0].adds);
1550   EXPECT_TRUE(lists[0].subs.empty());
1551
1552   // Add a phishing add chunk with two entries of the same host.
1553   chunks.clear();
1554   chunks.push_back(AddChunkPrefix2Value(47,
1555                                         "www.evil.com/phishing1.html",
1556                                         "www.evil.com/phishing2.html"));
1557
1558   ASSERT_TRUE(database_->UpdateStarted(&lists));
1559   database_->InsertChunks(safe_browsing_util::kPhishingList, chunks.get());
1560   database_->UpdateFinished(true);
1561
1562   GetListsInfo(&lists);
1563   ASSERT_LE(2U, lists.size());
1564   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
1565   EXPECT_EQ("1", lists[0].adds);
1566   EXPECT_TRUE(lists[0].subs.empty());
1567   EXPECT_EQ(safe_browsing_util::kPhishingList, lists[1].name);
1568   EXPECT_EQ("47", lists[1].adds);
1569   EXPECT_TRUE(lists[1].subs.empty());
1570
1571   std::vector<SBPrefix> prefix_hits;
1572   std::vector<SBFullHashResult> cache_hits;
1573
1574   EXPECT_TRUE(database_->ContainsBrowseUrl(
1575       GURL("http://www.evil.com/malware1.html"), &prefix_hits, &cache_hits));
1576   EXPECT_TRUE(database_->ContainsBrowseUrl(
1577       GURL("http://www.evil.com/malware2.html"), &prefix_hits, &cache_hits));
1578   EXPECT_TRUE(database_->ContainsBrowseUrl(
1579       GURL("http://www.evil.com/phishing1.html"), &prefix_hits, &cache_hits));
1580   EXPECT_TRUE(database_->ContainsBrowseUrl(
1581       GURL("http://www.evil.com/phishing2.html"), &prefix_hits, &cache_hits));
1582
1583   // Test removing a single prefix from the add chunk.
1584   // Remove the prefix that added first.
1585   chunks.clear();
1586   chunks.push_back(SubChunkPrefixValue(4, "www.evil.com/malware1.html", 1));
1587   ASSERT_TRUE(database_->UpdateStarted(&lists));
1588   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1589   database_->UpdateFinished(true);
1590
1591   // Remove the prefix that added last.
1592   chunks.clear();
1593   chunks.push_back(SubChunkPrefixValue(5, "www.evil.com/phishing2.html", 47));
1594   ASSERT_TRUE(database_->UpdateStarted(&lists));
1595   database_->InsertChunks(safe_browsing_util::kPhishingList, chunks.get());
1596   database_->UpdateFinished(true);
1597
1598   // Verify that the database contains urls expected.
1599   EXPECT_FALSE(database_->ContainsBrowseUrl(
1600       GURL("http://www.evil.com/malware1.html"), &prefix_hits, &cache_hits));
1601   EXPECT_TRUE(database_->ContainsBrowseUrl(
1602       GURL("http://www.evil.com/malware2.html"), &prefix_hits, &cache_hits));
1603   EXPECT_TRUE(database_->ContainsBrowseUrl(
1604       GURL("http://www.evil.com/phishing1.html"), &prefix_hits, &cache_hits));
1605   EXPECT_FALSE(database_->ContainsBrowseUrl(
1606       GURL("http://www.evil.com/phishing2.html"), &prefix_hits, &cache_hits));
1607 }
1608
1609 // Test that an empty update doesn't actually update the database.
1610 // This isn't a functionality requirement, but it is a useful
1611 // optimization.
1612 TEST_F(SafeBrowsingDatabaseTest, EmptyUpdate) {
1613   ScopedVector<SBChunkData> chunks;
1614
1615   base::FilePath filename = database_->BrowseDBFilename(database_filename_);
1616
1617   // Prime the database.
1618   std::vector<SBListChunkRanges> lists;
1619   ASSERT_TRUE(database_->UpdateStarted(&lists));
1620   chunks.push_back(AddChunkPrefixValue(1, "www.evil.com/malware.html"));
1621   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1622   database_->UpdateFinished(true);
1623
1624   // Get an older time to reset the lastmod time for detecting whether
1625   // the file has been updated.
1626   base::File::Info before_info, after_info;
1627   ASSERT_TRUE(base::GetFileInfo(filename, &before_info));
1628   const Time old_last_modified =
1629       before_info.last_modified - TimeDelta::FromSeconds(10);
1630
1631   // Inserting another chunk updates the database file.  The sleep is
1632   // needed because otherwise the entire test can finish w/in the
1633   // resolution of the lastmod time.
1634   ASSERT_TRUE(base::TouchFile(filename, old_last_modified, old_last_modified));
1635   ASSERT_TRUE(base::GetFileInfo(filename, &before_info));
1636   ASSERT_TRUE(database_->UpdateStarted(&lists));
1637   chunks.push_back(AddChunkPrefixValue(2, "www.foo.com/malware.html"));
1638   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1639   database_->UpdateFinished(true);
1640   ASSERT_TRUE(base::GetFileInfo(filename, &after_info));
1641   EXPECT_LT(before_info.last_modified, after_info.last_modified);
1642
1643   // Deleting a chunk updates the database file.
1644   ASSERT_TRUE(base::TouchFile(filename, old_last_modified, old_last_modified));
1645   ASSERT_TRUE(base::GetFileInfo(filename, &before_info));
1646   ASSERT_TRUE(database_->UpdateStarted(&lists));
1647   AddDelChunk(safe_browsing_util::kMalwareList, 2);
1648   database_->UpdateFinished(true);
1649   ASSERT_TRUE(base::GetFileInfo(filename, &after_info));
1650   EXPECT_LT(before_info.last_modified, after_info.last_modified);
1651
1652   // Simply calling |UpdateStarted()| then |UpdateFinished()| does not
1653   // update the database file.
1654   ASSERT_TRUE(base::TouchFile(filename, old_last_modified, old_last_modified));
1655   ASSERT_TRUE(base::GetFileInfo(filename, &before_info));
1656   ASSERT_TRUE(database_->UpdateStarted(&lists));
1657   database_->UpdateFinished(true);
1658   ASSERT_TRUE(base::GetFileInfo(filename, &after_info));
1659   EXPECT_EQ(before_info.last_modified, after_info.last_modified);
1660 }
1661
1662 // Test that a filter file is written out during update and read back
1663 // in during setup.
1664 TEST_F(SafeBrowsingDatabaseTest, FilterFile) {
1665   // Create a database with trivial example data and write it out.
1666   {
1667     // Prime the database.
1668     std::vector<SBListChunkRanges> lists;
1669     ASSERT_TRUE(database_->UpdateStarted(&lists));
1670
1671     ScopedVector<SBChunkData> chunks;
1672     chunks.push_back(AddChunkPrefixValue(1, "www.evil.com/malware.html"));
1673     database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1674     database_->UpdateFinished(true);
1675   }
1676
1677   // Find the malware url in the database, don't find a good url.
1678   std::vector<SBPrefix> prefix_hits;
1679   std::vector<SBFullHashResult> cache_hits;
1680   EXPECT_TRUE(database_->ContainsBrowseUrl(
1681       GURL("http://www.evil.com/malware.html"), &prefix_hits, &cache_hits));
1682   EXPECT_FALSE(database_->ContainsBrowseUrl(
1683       GURL("http://www.good.com/goodware.html"), &prefix_hits, &cache_hits));
1684
1685   base::FilePath filter_file = database_->PrefixSetForFilename(
1686       database_->BrowseDBFilename(database_filename_));
1687
1688   // After re-creating the database, it should have a filter read from
1689   // a file, so it should find the same results.
1690   ASSERT_TRUE(base::PathExists(filter_file));
1691   ResetAndReloadFullDatabase();
1692   EXPECT_TRUE(database_->ContainsBrowseUrl(
1693       GURL("http://www.evil.com/malware.html"), &prefix_hits, &cache_hits));
1694   EXPECT_FALSE(database_->ContainsBrowseUrl(
1695       GURL("http://www.good.com/goodware.html"), &prefix_hits, &cache_hits));
1696
1697   // If there is no filter file, the database cannot find malware urls.
1698   base::DeleteFile(filter_file, false);
1699   ASSERT_FALSE(base::PathExists(filter_file));
1700   ResetAndReloadFullDatabase();
1701   EXPECT_FALSE(database_->ContainsBrowseUrl(
1702       GURL("http://www.evil.com/malware.html"), &prefix_hits, &cache_hits));
1703   EXPECT_FALSE(database_->ContainsBrowseUrl(
1704       GURL("http://www.good.com/goodware.html"), &prefix_hits, &cache_hits));
1705 }
1706
1707 TEST_F(SafeBrowsingDatabaseTest, CachedFullMiss) {
1708   const SBPrefix kPrefix1 = 1001U;
1709   const SBFullHash kFullHash1_1 =
1710       SBFullHashForPrefixAndSuffix(kPrefix1, "\x01");
1711
1712   const SBPrefix kPrefix2 = 1002U;
1713   const SBFullHash kFullHash2_1 =
1714       SBFullHashForPrefixAndSuffix(kPrefix2, "\x01");
1715
1716   // Insert prefix kPrefix1 and kPrefix2 into database.
1717   ScopedVector<SBChunkData> chunks;
1718   chunks.push_back(AddChunkPrefix(1, kPrefix1));
1719   chunks.push_back(AddChunkPrefix(2, kPrefix2));
1720
1721   std::vector<SBListChunkRanges> lists;
1722   ASSERT_TRUE(database_->UpdateStarted(&lists));
1723   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1724   database_->UpdateFinished(true);
1725
1726   {
1727     // Cache a full miss result for kPrefix1.
1728     std::vector<SBPrefix> prefixes(1, kPrefix1);
1729     std::vector<SBFullHashResult> cache_results;
1730     database_->CacheHashResults(prefixes, cache_results, kCacheLifetime);
1731   }
1732
1733   {
1734     // kFullHash1_1 gets no prefix hit because of the cached item, and also does
1735     // not have a cache hit.
1736     std::vector<SBFullHash> full_hashes(1, kFullHash1_1);
1737     std::vector<SBPrefix> prefix_hits;
1738     std::vector<SBFullHashResult> cache_hits;
1739     EXPECT_FALSE(database_->ContainsBrowseUrlHashesForTesting(
1740         full_hashes, &prefix_hits, &cache_hits));
1741
1742     // kFullHash2_1 gets a hit from the prefix in the database.
1743     full_hashes.push_back(kFullHash2_1);
1744     prefix_hits.clear();
1745     cache_hits.clear();
1746     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
1747         full_hashes, &prefix_hits, &cache_hits));
1748     ASSERT_EQ(1U, prefix_hits.size());
1749     EXPECT_EQ(kPrefix2, prefix_hits[0]);
1750     EXPECT_TRUE(cache_hits.empty());
1751   }
1752 }
1753
1754 TEST_F(SafeBrowsingDatabaseTest, CachedPrefixHitFullMiss) {
1755   const SBPrefix kPrefix1 = 1001U;
1756   const SBFullHash kFullHash1_1 =
1757       SBFullHashForPrefixAndSuffix(kPrefix1, "\x01");
1758   const SBFullHash kFullHash1_2 =
1759       SBFullHashForPrefixAndSuffix(kPrefix1, "\x02");
1760   const SBFullHash kFullHash1_3 =
1761       SBFullHashForPrefixAndSuffix(kPrefix1, "\x03");
1762
1763   const SBPrefix kPrefix2 = 1002U;
1764   const SBFullHash kFullHash2_1 =
1765       SBFullHashForPrefixAndSuffix(kPrefix2, "\x01");
1766
1767   const SBPrefix kPrefix3 = 1003U;
1768   const SBFullHash kFullHash3_1 =
1769       SBFullHashForPrefixAndSuffix(kPrefix3, "\x01");
1770
1771   // Insert prefix kPrefix1 and kPrefix2 into database.
1772   ScopedVector<SBChunkData> chunks;
1773   chunks.push_back(AddChunkPrefix(1, kPrefix1));
1774   chunks.push_back(AddChunkPrefix(2, kPrefix2));
1775
1776   std::vector<SBListChunkRanges> lists;
1777   ASSERT_TRUE(database_->UpdateStarted(&lists));
1778   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1779   database_->UpdateFinished(true);
1780
1781   {
1782     // kFullHash1_1 has a prefix hit of kPrefix1.
1783     std::vector<SBFullHash> full_hashes;
1784     full_hashes.push_back(kFullHash1_1);
1785     std::vector<SBPrefix> prefix_hits;
1786     std::vector<SBFullHashResult> cache_hits;
1787     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
1788         full_hashes, &prefix_hits, &cache_hits));
1789     ASSERT_EQ(1U, prefix_hits.size());
1790     EXPECT_EQ(kPrefix1, prefix_hits[0]);
1791     EXPECT_TRUE(cache_hits.empty());
1792
1793     // kFullHash2_1 has a prefix hit of kPrefix2.
1794     full_hashes.push_back(kFullHash2_1);
1795     prefix_hits.clear();
1796     cache_hits.clear();
1797     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
1798         full_hashes, &prefix_hits, &cache_hits));
1799     ASSERT_EQ(2U, prefix_hits.size());
1800     EXPECT_EQ(kPrefix1, prefix_hits[0]);
1801     EXPECT_EQ(kPrefix2, prefix_hits[1]);
1802     EXPECT_TRUE(cache_hits.empty());
1803
1804     // kFullHash3_1 has no hits.
1805     full_hashes.push_back(kFullHash3_1);
1806     prefix_hits.clear();
1807     cache_hits.clear();
1808     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
1809         full_hashes, &prefix_hits, &cache_hits));
1810     ASSERT_EQ(2U, prefix_hits.size());
1811     EXPECT_EQ(kPrefix1, prefix_hits[0]);
1812     EXPECT_EQ(kPrefix2, prefix_hits[1]);
1813     EXPECT_TRUE(cache_hits.empty());
1814   }
1815
1816   {
1817     // Cache a fullhash result for two kPrefix1 full hashes.
1818     std::vector<SBPrefix> prefixes(1, kPrefix1);
1819     std::vector<SBFullHashResult> cache_results;
1820
1821     SBFullHashResult full_hash_result;
1822     full_hash_result.list_id = safe_browsing_util::MALWARE;
1823
1824     full_hash_result.hash = kFullHash1_1;
1825     cache_results.push_back(full_hash_result);
1826
1827     full_hash_result.hash = kFullHash1_3;
1828     cache_results.push_back(full_hash_result);
1829
1830     database_->CacheHashResults(prefixes, cache_results, kCacheLifetime);
1831   }
1832
1833   {
1834     // kFullHash1_1 should now see a cache hit.
1835     std::vector<SBFullHash> full_hashes(1, kFullHash1_1);
1836     std::vector<SBPrefix> prefix_hits;
1837     std::vector<SBFullHashResult> cache_hits;
1838     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
1839         full_hashes, &prefix_hits, &cache_hits));
1840     EXPECT_TRUE(prefix_hits.empty());
1841     ASSERT_EQ(1U, cache_hits.size());
1842     EXPECT_TRUE(SBFullHashEqual(kFullHash1_1, cache_hits[0].hash));
1843
1844     // Adding kFullHash2_1 will see the existing cache hit plus the prefix hit
1845     // for kPrefix2.
1846     full_hashes.push_back(kFullHash2_1);
1847     prefix_hits.clear();
1848     cache_hits.clear();
1849     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
1850         full_hashes, &prefix_hits, &cache_hits));
1851     ASSERT_EQ(1U, prefix_hits.size());
1852     EXPECT_EQ(kPrefix2, prefix_hits[0]);
1853     ASSERT_EQ(1U, cache_hits.size());
1854     EXPECT_TRUE(SBFullHashEqual(kFullHash1_1, cache_hits[0].hash));
1855
1856     // kFullHash1_3 also gets a cache hit.
1857     full_hashes.push_back(kFullHash1_3);
1858     prefix_hits.clear();
1859     cache_hits.clear();
1860     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
1861         full_hashes, &prefix_hits, &cache_hits));
1862     ASSERT_EQ(1U, prefix_hits.size());
1863     EXPECT_EQ(kPrefix2, prefix_hits[0]);
1864     ASSERT_EQ(2U, cache_hits.size());
1865     EXPECT_TRUE(SBFullHashEqual(kFullHash1_1, cache_hits[0].hash));
1866     EXPECT_TRUE(SBFullHashEqual(kFullHash1_3, cache_hits[1].hash));
1867   }
1868
1869   {
1870     // Check if DB contains only kFullHash1_3. Should return a cache hit.
1871     std::vector<SBFullHash> full_hashes(1, kFullHash1_3);
1872     std::vector<SBPrefix> prefix_hits;
1873     std::vector<SBFullHashResult> cache_hits;
1874     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
1875         full_hashes, &prefix_hits, &cache_hits));
1876     EXPECT_TRUE(prefix_hits.empty());
1877     ASSERT_EQ(1U, cache_hits.size());
1878     EXPECT_TRUE(SBFullHashEqual(kFullHash1_3, cache_hits[0].hash));
1879   }
1880
1881   {
1882     // kFullHash1_2 has no cache hit, and no prefix hit because of the cache for
1883     // kPrefix1.
1884     std::vector<SBFullHash> full_hashes(1, kFullHash1_2);
1885     std::vector<SBPrefix> prefix_hits;
1886     std::vector<SBFullHashResult> cache_hits;
1887     EXPECT_FALSE(database_->ContainsBrowseUrlHashesForTesting(
1888         full_hashes, &prefix_hits, &cache_hits));
1889
1890     // Other prefix hits possible when kFullHash1_2 hits nothing.
1891     full_hashes.push_back(kFullHash2_1);
1892     prefix_hits.clear();
1893     cache_hits.clear();
1894     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
1895         full_hashes, &prefix_hits, &cache_hits));
1896     ASSERT_EQ(1U, prefix_hits.size());
1897     EXPECT_EQ(kPrefix2, prefix_hits[0]);
1898     EXPECT_TRUE(cache_hits.empty());
1899   }
1900 }
1901
1902 TEST_F(SafeBrowsingDatabaseTest, BrowseFullHashMatching) {
1903   const SBPrefix kPrefix1 = 1001U;
1904   const SBFullHash kFullHash1_1 =
1905       SBFullHashForPrefixAndSuffix(kPrefix1, "\x01");
1906   const SBFullHash kFullHash1_2 =
1907       SBFullHashForPrefixAndSuffix(kPrefix1, "\x02");
1908   const SBFullHash kFullHash1_3 =
1909       SBFullHashForPrefixAndSuffix(kPrefix1, "\x03");
1910
1911   // Insert two full hashes with a shared prefix.
1912   ScopedVector<SBChunkData> chunks;
1913   chunks.push_back(AddChunkFullHash(1, kFullHash1_1));
1914   chunks.push_back(AddChunkFullHash(2, kFullHash1_2));
1915
1916   std::vector<SBListChunkRanges> lists;
1917   ASSERT_TRUE(database_->UpdateStarted(&lists));
1918   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1919   database_->UpdateFinished(true);
1920
1921   {
1922     // Check a full hash which isn't present.
1923     std::vector<SBFullHash> full_hashes(1, kFullHash1_3);
1924     std::vector<SBPrefix> prefix_hits;
1925     std::vector<SBFullHashResult> cache_hits;
1926     EXPECT_FALSE(database_->ContainsBrowseUrlHashesForTesting(
1927         full_hashes, &prefix_hits, &cache_hits));
1928
1929     // Also one which is present, should have a prefix hit.
1930     full_hashes.push_back(kFullHash1_1);
1931     prefix_hits.clear();
1932     cache_hits.clear();
1933     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
1934         full_hashes, &prefix_hits, &cache_hits));
1935     ASSERT_EQ(1U, prefix_hits.size());
1936     EXPECT_EQ(kPrefix1, prefix_hits[0]);
1937     EXPECT_TRUE(cache_hits.empty());
1938
1939     // Two full hash matches with the same prefix should return one prefix hit.
1940     full_hashes.push_back(kFullHash1_2);
1941     prefix_hits.clear();
1942     cache_hits.clear();
1943     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
1944         full_hashes, &prefix_hits, &cache_hits));
1945     ASSERT_EQ(1U, prefix_hits.size());
1946     EXPECT_EQ(kPrefix1, prefix_hits[0]);
1947     EXPECT_TRUE(cache_hits.empty());
1948   }
1949
1950   {
1951     // Cache a gethash result for kFullHash1_2.
1952     SBFullHashResult full_hash_result;
1953     full_hash_result.list_id = safe_browsing_util::MALWARE;
1954     full_hash_result.hash = kFullHash1_2;
1955
1956     std::vector<SBPrefix> prefixes(1, kPrefix1);
1957     std::vector<SBFullHashResult> cache_results(1, full_hash_result);
1958
1959     database_->CacheHashResults(prefixes, cache_results, kCacheLifetime);
1960   }
1961
1962   {
1963     // kFullHash1_3 should still return false, because the cached
1964     // result for kPrefix1 doesn't contain kFullHash1_3.
1965     std::vector<SBFullHash> full_hashes(1, kFullHash1_3);
1966     std::vector<SBPrefix> prefix_hits;
1967     std::vector<SBFullHashResult> cache_hits;
1968     EXPECT_FALSE(database_->ContainsBrowseUrlHashesForTesting(
1969         full_hashes, &prefix_hits, &cache_hits));
1970
1971     // kFullHash1_1 is also not in the cached result, which takes
1972     // priority over the database.
1973     prefix_hits.clear();
1974     full_hashes.push_back(kFullHash1_1);
1975     cache_hits.clear();
1976     EXPECT_FALSE(database_->ContainsBrowseUrlHashesForTesting(
1977         full_hashes, &prefix_hits, &cache_hits));
1978
1979     // kFullHash1_2 is in the cached result.
1980     full_hashes.push_back(kFullHash1_2);
1981     prefix_hits.clear();
1982     cache_hits.clear();
1983     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
1984         full_hashes, &prefix_hits, &cache_hits));
1985     EXPECT_TRUE(prefix_hits.empty());
1986     ASSERT_EQ(1U, cache_hits.size());
1987     EXPECT_TRUE(SBFullHashEqual(kFullHash1_2, cache_hits[0].hash));
1988   }
1989
1990   // Remove kFullHash1_1 from the database.
1991   chunks.clear();
1992   chunks.push_back(SubChunkFullHash(11, kFullHash1_1, 1));
1993
1994   ASSERT_TRUE(database_->UpdateStarted(&lists));
1995   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1996   database_->UpdateFinished(true);
1997
1998   // Cache should be cleared after updating.
1999   EXPECT_TRUE(
2000       database_->GetUnsynchronizedPrefixGetHashCacheForTesting()->empty());
2001
2002   {
2003     // Now the database doesn't contain kFullHash1_1.
2004     std::vector<SBFullHash> full_hashes(1, kFullHash1_1);
2005     std::vector<SBPrefix> prefix_hits;
2006     std::vector<SBFullHashResult> cache_hits;
2007     EXPECT_FALSE(database_->ContainsBrowseUrlHashesForTesting(
2008         full_hashes, &prefix_hits, &cache_hits));
2009
2010     // Nor kFullHash1_3.
2011     full_hashes.push_back(kFullHash1_3);
2012     prefix_hits.clear();
2013     cache_hits.clear();
2014     EXPECT_FALSE(database_->ContainsBrowseUrlHashesForTesting(
2015         full_hashes, &prefix_hits, &cache_hits));
2016
2017     // Still has kFullHash1_2.
2018     full_hashes.push_back(kFullHash1_2);
2019     prefix_hits.clear();
2020     cache_hits.clear();
2021     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
2022         full_hashes, &prefix_hits, &cache_hits));
2023     ASSERT_EQ(1U, prefix_hits.size());
2024     EXPECT_EQ(kPrefix1, prefix_hits[0]);
2025     EXPECT_TRUE(cache_hits.empty());
2026   }
2027
2028   // Remove kFullHash1_2 from the database.
2029   chunks.clear();
2030   chunks.push_back(SubChunkFullHash(12, kFullHash1_2, 2));
2031
2032   ASSERT_TRUE(database_->UpdateStarted(&lists));
2033   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
2034   database_->UpdateFinished(true);
2035
2036   // Cache should be cleared after updating.
2037   EXPECT_TRUE(
2038       database_->GetUnsynchronizedPrefixGetHashCacheForTesting()->empty());
2039
2040   {
2041     // None are present.
2042     std::vector<SBFullHash> full_hashes;
2043     std::vector<SBPrefix> prefix_hits;
2044     std::vector<SBFullHashResult> cache_hits;
2045     full_hashes.push_back(kFullHash1_1);
2046     full_hashes.push_back(kFullHash1_2);
2047     full_hashes.push_back(kFullHash1_3);
2048     EXPECT_FALSE(database_->ContainsBrowseUrlHashesForTesting(
2049         full_hashes, &prefix_hits, &cache_hits));
2050   }
2051 }
2052
2053 TEST_F(SafeBrowsingDatabaseTest, BrowseFullHashAndPrefixMatching) {
2054   const SBPrefix kPrefix1 = 1001U;
2055   const SBFullHash kFullHash1_1 =
2056       SBFullHashForPrefixAndSuffix(kPrefix1, "\x01");
2057   const SBFullHash kFullHash1_2 =
2058       SBFullHashForPrefixAndSuffix(kPrefix1, "\x02");
2059
2060   ScopedVector<SBChunkData> chunks;
2061   chunks.push_back(AddChunkFullHash(1, kFullHash1_1));
2062
2063   std::vector<SBListChunkRanges> lists;
2064   ASSERT_TRUE(database_->UpdateStarted(&lists));
2065   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
2066   database_->UpdateFinished(true);
2067
2068   {
2069     // kFullHash1_2 does not match kFullHash1_1.
2070     std::vector<SBFullHash> full_hashes(1, kFullHash1_2);
2071     std::vector<SBPrefix> prefix_hits;
2072     std::vector<SBFullHashResult> cache_hits;
2073     EXPECT_FALSE(database_->ContainsBrowseUrlHashesForTesting(
2074         full_hashes, &prefix_hits, &cache_hits));
2075   }
2076
2077   // Add a prefix match.
2078   chunks.clear();
2079   chunks.push_back(AddChunkPrefix(2, kPrefix1));
2080
2081   ASSERT_TRUE(database_->UpdateStarted(&lists));
2082   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
2083   database_->UpdateFinished(true);
2084
2085   {
2086     // kFullHash1_2 does match kPrefix1.
2087     std::vector<SBFullHash> full_hashes(1, kFullHash1_2);
2088     std::vector<SBPrefix> prefix_hits;
2089     std::vector<SBFullHashResult> cache_hits;
2090     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
2091         full_hashes, &prefix_hits, &cache_hits));
2092     ASSERT_EQ(1U, prefix_hits.size());
2093     EXPECT_EQ(kPrefix1, prefix_hits[0]);
2094     EXPECT_TRUE(cache_hits.empty());
2095   }
2096
2097   // Remove the full hash.
2098   chunks.clear();
2099   chunks.push_back(SubChunkFullHash(11, kFullHash1_1, 1));
2100
2101   ASSERT_TRUE(database_->UpdateStarted(&lists));
2102   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
2103   database_->UpdateFinished(true);
2104
2105   {
2106     // kFullHash1_2 still returns true due to the prefix hit.
2107     std::vector<SBFullHash> full_hashes(1, kFullHash1_2);
2108     std::vector<SBPrefix> prefix_hits;
2109     std::vector<SBFullHashResult> cache_hits;
2110     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
2111         full_hashes, &prefix_hits, &cache_hits));
2112     ASSERT_EQ(1U, prefix_hits.size());
2113     EXPECT_EQ(kPrefix1, prefix_hits[0]);
2114     EXPECT_TRUE(cache_hits.empty());
2115   }
2116 }
2117
2118 TEST_F(SafeBrowsingDatabaseTest, MalwareIpBlacklist) {
2119   std::vector<SBListChunkRanges> lists;
2120   ASSERT_TRUE(database_->UpdateStarted(&lists));
2121
2122   ScopedVector<SBChunkData> chunks;
2123
2124   // IPv4 prefix match for ::ffff:192.168.1.0/120.
2125   chunks.push_back(AddChunkHashedIpValue(1, "::ffff:192.168.1.0", 120));
2126
2127   // IPv4 exact match for ::ffff:192.1.1.1.
2128   chunks.push_back(AddChunkHashedIpValue(2, "::ffff:192.1.1.1", 128));
2129
2130   // IPv6 exact match for: fe80::31a:a0ff:fe10:786e/128.
2131   chunks.push_back(AddChunkHashedIpValue(3, "fe80::31a:a0ff:fe10:786e", 128));
2132
2133   // IPv6 prefix match for: 2620:0:1000:3103::/64.
2134   chunks.push_back(AddChunkHashedIpValue(4, "2620:0:1000:3103::", 64));
2135
2136   // IPv4 prefix match for ::ffff:192.1.122.0/119.
2137   chunks.push_back(AddChunkHashedIpValue(5, "::ffff:192.1.122.0", 119));
2138
2139   // IPv4 prefix match for ::ffff:192.1.128.0/113.
2140   chunks.push_back(AddChunkHashedIpValue(6, "::ffff:192.1.128.0", 113));
2141
2142   database_->InsertChunks(safe_browsing_util::kIPBlacklist, chunks.get());
2143   database_->UpdateFinished(true);
2144
2145   EXPECT_FALSE(database_->ContainsMalwareIP("192.168.0.255"));
2146   EXPECT_TRUE(database_->ContainsMalwareIP("192.168.1.0"));
2147   EXPECT_TRUE(database_->ContainsMalwareIP("192.168.1.255"));
2148   EXPECT_TRUE(database_->ContainsMalwareIP("192.168.1.10"));
2149   EXPECT_TRUE(database_->ContainsMalwareIP("::ffff:192.168.1.2"));
2150   EXPECT_FALSE(database_->ContainsMalwareIP("192.168.2.0"));
2151
2152   EXPECT_FALSE(database_->ContainsMalwareIP("192.1.1.0"));
2153   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.1.1"));
2154   EXPECT_FALSE(database_->ContainsMalwareIP("192.1.1.2"));
2155
2156   EXPECT_FALSE(database_->ContainsMalwareIP(
2157       "2620:0:1000:3102:ffff:ffff:ffff:ffff"));
2158   EXPECT_TRUE(database_->ContainsMalwareIP("2620:0:1000:3103::"));
2159   EXPECT_TRUE(database_->ContainsMalwareIP(
2160       "2620:0:1000:3103:ffff:ffff:ffff:ffff"));
2161   EXPECT_FALSE(database_->ContainsMalwareIP("2620:0:1000:3104::"));
2162
2163   EXPECT_FALSE(database_->ContainsMalwareIP("fe80::21a:a0ff:fe10:786d"));
2164   EXPECT_TRUE(database_->ContainsMalwareIP("fe80::31a:a0ff:fe10:786e"));
2165   EXPECT_FALSE(database_->ContainsMalwareIP("fe80::21a:a0ff:fe10:786f"));
2166
2167   EXPECT_FALSE(database_->ContainsMalwareIP("192.1.121.255"));
2168   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.122.0"));
2169   EXPECT_TRUE(database_->ContainsMalwareIP("::ffff:192.1.122.1"));
2170   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.122.255"));
2171   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.123.0"));
2172   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.123.255"));
2173   EXPECT_FALSE(database_->ContainsMalwareIP("192.1.124.0"));
2174
2175   EXPECT_FALSE(database_->ContainsMalwareIP("192.1.127.255"));
2176   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.128.0"));
2177   EXPECT_TRUE(database_->ContainsMalwareIP("::ffff:192.1.128.1"));
2178   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.128.255"));
2179   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.255.0"));
2180   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.255.255"));
2181   EXPECT_FALSE(database_->ContainsMalwareIP("192.2.0.0"));
2182 }
2183
2184 TEST_F(SafeBrowsingDatabaseTest, ContainsBrowseURL) {
2185   std::vector<SBListChunkRanges> lists;
2186   ASSERT_TRUE(database_->UpdateStarted(&lists));
2187
2188   // Add a host-level hit.
2189   {
2190     ScopedVector<SBChunkData> chunks;
2191     chunks.push_back(AddChunkPrefixValue(1, "www.evil.com/"));
2192     database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
2193   }
2194
2195   // Add a specific fullhash.
2196   static const char kWhateverMalware[] = "www.whatever.com/malware.html";
2197   {
2198     ScopedVector<SBChunkData> chunks;
2199     chunks.push_back(AddChunkFullHashValue(2, kWhateverMalware));
2200     database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
2201   }
2202
2203   // Add a fullhash which has a prefix collision for a known url.
2204   static const char kExampleFine[] = "www.example.com/fine.html";
2205   static const char kExampleCollision[] =
2206       "www.example.com/3123364814/malware.htm";
2207   ASSERT_EQ(SBPrefixForString(kExampleFine),
2208             SBPrefixForString(kExampleCollision));
2209   {
2210     ScopedVector<SBChunkData> chunks;
2211     chunks.push_back(AddChunkFullHashValue(3, kExampleCollision));
2212     database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
2213   }
2214
2215   database_->UpdateFinished(true);
2216
2217   std::vector<SBPrefix> prefix_hits;
2218   std::vector<SBFullHashResult> cache_hits;
2219
2220   // Anything will hit the host prefix.
2221   EXPECT_TRUE(database_->ContainsBrowseUrl(
2222       GURL("http://www.evil.com/malware.html"), &prefix_hits, &cache_hits));
2223   ASSERT_EQ(1U, prefix_hits.size());
2224   EXPECT_EQ(SBPrefixForString("www.evil.com/"), prefix_hits[0]);
2225   EXPECT_TRUE(cache_hits.empty());
2226
2227   // Hit the specific URL prefix.
2228   EXPECT_TRUE(database_->ContainsBrowseUrl(
2229       GURL(std::string("http://") + kWhateverMalware),
2230       &prefix_hits, &cache_hits));
2231   ASSERT_EQ(1U, prefix_hits.size());
2232   EXPECT_EQ(SBPrefixForString(kWhateverMalware), prefix_hits[0]);
2233   EXPECT_TRUE(cache_hits.empty());
2234
2235   // Other URLs at that host are fine.
2236   EXPECT_FALSE(database_->ContainsBrowseUrl(
2237       GURL("http://www.whatever.com/fine.html"), &prefix_hits, &cache_hits));
2238   EXPECT_TRUE(prefix_hits.empty());
2239   EXPECT_TRUE(cache_hits.empty());
2240
2241   // Hit the specific URL full hash.
2242   EXPECT_TRUE(database_->ContainsBrowseUrl(
2243       GURL(std::string("http://") + kExampleCollision),
2244       &prefix_hits, &cache_hits));
2245   ASSERT_EQ(1U, prefix_hits.size());
2246   EXPECT_EQ(SBPrefixForString(kExampleCollision), prefix_hits[0]);
2247   EXPECT_TRUE(cache_hits.empty());
2248
2249   // This prefix collides, but no full hash match.
2250   EXPECT_FALSE(database_->ContainsBrowseUrl(
2251       GURL(std::string("http://") + kExampleFine), &prefix_hits, &cache_hits));
2252 }