chrome/browser/safe_browsing/safe_browsing_database_unittest.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4 //
   5 // Unit tests for the SafeBrowsing storage system.
   6
   7 #include "chrome/browser/safe_browsing/safe_browsing_database.h"
   8
   9 #include "base/files/file_util.h"
  10 #include "base/files/scoped_temp_dir.h"
  11 #include "base/logging.h"
  12 #include "base/memory/scoped_vector.h"
  13 #include "base/message_loop/message_loop.h"
  14 #include "base/sha1.h"
  15 #include "base/strings/string_number_conversions.h"
  16 #include "base/strings/string_split.h"
  17 #include "base/test/test_simple_task_runner.h"
  18 #include "base/time/time.h"
  19 #include "chrome/browser/safe_browsing/chunk.pb.h"
  20 #include "chrome/browser/safe_browsing/safe_browsing_store_file.h"
  21 #include "crypto/sha2.h"
  22 #include "net/base/net_util.h"
  23 #include "testing/gtest/include/gtest/gtest.h"
  24 #include "testing/platform_test.h"
  25 #include "url/gurl.h"
  26
  27 using base::Time;
  28 using base::TimeDelta;
  29
  30 namespace {
  31
  32 const TimeDelta kCacheLifetime = TimeDelta::FromMinutes(45);
  33
  34 SBPrefix SBPrefixForString(const std::string& str) {
  35   return SBFullHashForString(str).prefix;
  36 }
  37
  38 // Construct a full hash which has the given prefix, with the given
  39 // suffix data coming after the prefix.
  40 SBFullHash SBFullHashForPrefixAndSuffix(SBPrefix prefix,
  41                                         const base::StringPiece& suffix) {
  42   SBFullHash full_hash;
  43   memset(&full_hash, 0, sizeof(SBFullHash));
  44   full_hash.prefix = prefix;
  45   CHECK_LE(suffix.size() + sizeof(SBPrefix), sizeof(SBFullHash));
  46   memcpy(full_hash.full_hash + sizeof(SBPrefix), suffix.data(), suffix.size());
  47   return full_hash;
  48 }
  49
  50 std::string HashedIpPrefix(const std::string& ip_prefix, size_t prefix_size) {
  51   net::IPAddressNumber ip_number;
  52   EXPECT_TRUE(net::ParseIPLiteralToNumber(ip_prefix, &ip_number));
  53   EXPECT_EQ(net::kIPv6AddressSize, ip_number.size());
  54   const std::string hashed_ip_prefix = base::SHA1HashString(
  55       net::IPAddressToPackedString(ip_number));
  56   std::string hash(crypto::kSHA256Length, '\0');
  57   hash.replace(0, hashed_ip_prefix.size(), hashed_ip_prefix);
  58   hash[base::kSHA1Length] = static_cast<char>(prefix_size);
  59   return hash;
  60 }
  61
  62 // Helper to build a chunk.  Caller takes ownership.
  63 SBChunkData* BuildChunk(int chunk_number,
  64                         safe_browsing::ChunkData::ChunkType chunk_type,
  65                         safe_browsing::ChunkData::PrefixType prefix_type,
  66                         const void* data, size_t data_size,
  67                         const std::vector<int>& add_chunk_numbers) {
  68   scoped_ptr<safe_browsing::ChunkData> raw_data(new safe_browsing::ChunkData);
  69   raw_data->set_chunk_number(chunk_number);
  70   raw_data->set_chunk_type(chunk_type);
  71   raw_data->set_prefix_type(prefix_type);
  72   raw_data->set_hashes(data, data_size);
  73   raw_data->clear_add_numbers();
  74   for (size_t i = 0; i < add_chunk_numbers.size(); ++i) {
  75     raw_data->add_add_numbers(add_chunk_numbers[i]);
  76   }
  77
  78   return new SBChunkData(raw_data.release());
  79 }
  80
  81 // Create add chunk with a single prefix.
  82 SBChunkData* AddChunkPrefix(int chunk_number, SBPrefix prefix) {
  83   return BuildChunk(chunk_number, safe_browsing::ChunkData::ADD,
  84                     safe_browsing::ChunkData::PREFIX_4B,
  85                     &prefix, sizeof(prefix),
  86                     std::vector<int>());
  87 }
  88
  89 // Create add chunk with a single prefix generated from |value|.
  90 SBChunkData* AddChunkPrefixValue(int chunk_number,
  91                                  const std::string& value) {
  92   return AddChunkPrefix(chunk_number, SBPrefixForString(value));
  93 }
  94
  95 // Generate an add chunk with two prefixes.
  96 SBChunkData* AddChunkPrefix2Value(int chunk_number,
  97                                   const std::string& value1,
  98                                   const std::string& value2) {
  99   const SBPrefix prefixes[2] = {
 100     SBPrefixForString(value1),
 101     SBPrefixForString(value2),
 102   };
 103   return BuildChunk(chunk_number, safe_browsing::ChunkData::ADD,
 104                     safe_browsing::ChunkData::PREFIX_4B,
 105                     &prefixes[0], sizeof(prefixes),
 106                     std::vector<int>());
 107 }
 108
 109 // Generate an add chunk with four prefixes.
 110 SBChunkData* AddChunkPrefix4Value(int chunk_number,
 111                                   const std::string& value1,
 112                                   const std::string& value2,
 113                                   const std::string& value3,
 114                                   const std::string& value4) {
 115   const SBPrefix prefixes[4] = {
 116     SBPrefixForString(value1),
 117     SBPrefixForString(value2),
 118     SBPrefixForString(value3),
 119     SBPrefixForString(value4),
 120   };
 121   return BuildChunk(chunk_number, safe_browsing::ChunkData::ADD,
 122                     safe_browsing::ChunkData::PREFIX_4B,
 123                     &prefixes[0], sizeof(prefixes),
 124                     std::vector<int>());
 125 }
 126
 127 // Generate an add chunk with a full hash.
 128 SBChunkData* AddChunkFullHash(int chunk_number, SBFullHash full_hash) {
 129   return BuildChunk(chunk_number, safe_browsing::ChunkData::ADD,
 130                     safe_browsing::ChunkData::FULL_32B,
 131                     &full_hash, sizeof(full_hash),
 132                     std::vector<int>());
 133 }
 134
 135 // Generate an add chunk with a full hash generated from |value|.
 136 SBChunkData* AddChunkFullHashValue(int chunk_number,
 137                                    const std::string& value) {
 138   return AddChunkFullHash(chunk_number, SBFullHashForString(value));
 139 }
 140
 141 // Generate an add chunk with two full hashes.
 142 SBChunkData* AddChunkFullHash2Value(int chunk_number,
 143                                    const std::string& value1,
 144                                    const std::string& value2) {
 145   const SBFullHash full_hashes[2] = {
 146     SBFullHashForString(value1),
 147     SBFullHashForString(value2),
 148   };
 149   return BuildChunk(chunk_number, safe_browsing::ChunkData::ADD,
 150                     safe_browsing::ChunkData::FULL_32B,
 151                     &full_hashes[0], sizeof(full_hashes),
 152                     std::vector<int>());
 153 }
 154
 155 // Generate a sub chunk with a prefix generated from |value|.
 156 SBChunkData* SubChunkPrefixValue(int chunk_number,
 157                                  const std::string& value,
 158                                  int add_chunk_number) {
 159   const SBPrefix prefix = SBPrefixForString(value);
 160   return BuildChunk(chunk_number, safe_browsing::ChunkData::SUB,
 161                     safe_browsing::ChunkData::PREFIX_4B,
 162                     &prefix, sizeof(prefix),
 163                     std::vector<int>(1, add_chunk_number));
 164 }
 165
 166 // Generate a sub chunk with two prefixes.
 167 SBChunkData* SubChunkPrefix2Value(int chunk_number,
 168                                   const std::string& value1,
 169                                   int add_chunk_number1,
 170                                   const std::string& value2,
 171                                   int add_chunk_number2) {
 172   const SBPrefix prefixes[2] = {
 173     SBPrefixForString(value1),
 174     SBPrefixForString(value2),
 175   };
 176   std::vector<int> add_chunk_numbers;
 177   add_chunk_numbers.push_back(add_chunk_number1);
 178   add_chunk_numbers.push_back(add_chunk_number2);
 179   return BuildChunk(chunk_number, safe_browsing::ChunkData::SUB,
 180                     safe_browsing::ChunkData::PREFIX_4B,
 181                     &prefixes[0], sizeof(prefixes),
 182                     add_chunk_numbers);
 183 }
 184
 185 // Generate a sub chunk with a full hash.
 186 SBChunkData* SubChunkFullHash(int chunk_number,
 187                               SBFullHash full_hash,
 188                               int add_chunk_number) {
 189   return BuildChunk(chunk_number, safe_browsing::ChunkData::SUB,
 190                     safe_browsing::ChunkData::FULL_32B,
 191                     &full_hash, sizeof(full_hash),
 192                     std::vector<int>(1, add_chunk_number));
 193 }
 194
 195 // Generate a sub chunk with a full hash generated from |value|.
 196 SBChunkData* SubChunkFullHashValue(int chunk_number,
 197                                    const std::string& value,
 198                                    int add_chunk_number) {
 199   return SubChunkFullHash(chunk_number,
 200                           SBFullHashForString(value),
 201                           add_chunk_number);
 202 }
 203
 204 // Generate an add chunk with a single full hash for the ip blacklist.
 205 SBChunkData* AddChunkHashedIpValue(int chunk_number,
 206                                    const std::string& ip_str,
 207                                    size_t prefix_size) {
 208   const std::string full_hash_str = HashedIpPrefix(ip_str, prefix_size);
 209   EXPECT_EQ(sizeof(SBFullHash), full_hash_str.size());
 210   SBFullHash full_hash;
 211   std::memcpy(&(full_hash.full_hash), full_hash_str.data(), sizeof(SBFullHash));
 212   return BuildChunk(chunk_number, safe_browsing::ChunkData::ADD,
 213                     safe_browsing::ChunkData::FULL_32B,
 214                     &full_hash, sizeof(full_hash),
 215                     std::vector<int>());
 216 }
 217
 218 // Prevent DCHECK from killing tests.
 219 // TODO(shess): Pawel disputes the use of this, so the test which uses
 220 // it is DISABLED.  http://crbug.com/56448
 221 class ScopedLogMessageIgnorer {
 222  public:
 223   ScopedLogMessageIgnorer() {
 224     logging::SetLogMessageHandler(&LogMessageIgnorer);
 225   }
 226   ~ScopedLogMessageIgnorer() {
 227     // TODO(shess): Would be better to verify whether anyone else
 228     // changed it, and then restore it to the previous value.
 229     logging::SetLogMessageHandler(NULL);
 230   }
 231
 232  private:
 233   static bool LogMessageIgnorer(int severity, const char* file, int line,
 234       size_t message_start, const std::string& str) {
 235     // Intercept FATAL, strip the stack backtrace, and log it without
 236     // the crash part.
 237     if (severity == logging::LOG_FATAL) {
 238       size_t newline = str.find('\n');
 239       if (newline != std::string::npos) {
 240         const std::string msg = str.substr(0, newline + 1);
 241         fprintf(stderr, "%s", msg.c_str());
 242         fflush(stderr);
 243       }
 244       return true;
 245     }
 246
 247     return false;
 248   }
 249 };
 250
 251 }  // namespace
 252
 253 class SafeBrowsingDatabaseTest : public PlatformTest {
 254  public:
 255   SafeBrowsingDatabaseTest() : task_runner_(new base::TestSimpleTaskRunner) {}
 256
 257   void SetUp() override {
 258     PlatformTest::SetUp();
 259
 260     // Setup a database in a temporary directory.
 261     ASSERT_TRUE(temp_dir_.CreateUniqueTempDir());
 262     database_filename_ =
 263         temp_dir_.path().AppendASCII("SafeBrowsingTestDatabase");
 264
 265     ResetAndReloadFullDatabase();
 266   }
 267
 268   void TearDown() override {
 269     database_.reset();
 270
 271     PlatformTest::TearDown();
 272   }
 273
 274   // Reloads the |database_| in a new SafeBrowsingDatabaseNew object with all
 275   // stores enabled.
 276   void ResetAndReloadFullDatabase() {
 277     SafeBrowsingStoreFile* browse_store =
 278         new SafeBrowsingStoreFile(task_runner_);
 279     SafeBrowsingStoreFile* download_store =
 280         new SafeBrowsingStoreFile(task_runner_);
 281     SafeBrowsingStoreFile* csd_whitelist_store =
 282         new SafeBrowsingStoreFile(task_runner_);
 283     SafeBrowsingStoreFile* download_whitelist_store =
 284         new SafeBrowsingStoreFile(task_runner_);
 285     SafeBrowsingStoreFile* inclusion_whitelist_store =
 286         new SafeBrowsingStoreFile(task_runner_);
 287     SafeBrowsingStoreFile* extension_blacklist_store =
 288         new SafeBrowsingStoreFile(task_runner_);
 289     SafeBrowsingStoreFile* side_effect_free_whitelist_store =
 290         new SafeBrowsingStoreFile(task_runner_);
 291     SafeBrowsingStoreFile* ip_blacklist_store =
 292         new SafeBrowsingStoreFile(task_runner_);
 293     SafeBrowsingStoreFile* unwanted_software_store =
 294         new SafeBrowsingStoreFile(task_runner_);
 295     database_.reset(new SafeBrowsingDatabaseNew(
 296         task_runner_, browse_store, download_store, csd_whitelist_store,
 297         download_whitelist_store, inclusion_whitelist_store,
 298         extension_blacklist_store, side_effect_free_whitelist_store,
 299         ip_blacklist_store, unwanted_software_store));
 300     database_->Init(database_filename_);
 301   }
 302
 303   bool ContainsDownloadUrl(const std::vector<GURL>& urls,
 304                            std::vector<SBPrefix>* prefix_hits) {
 305     std::vector<SBPrefix> prefixes;
 306     SafeBrowsingDatabase::GetDownloadUrlPrefixes(urls, &prefixes);
 307     return database_->ContainsDownloadUrlPrefixes(prefixes, prefix_hits);
 308   }
 309
 310   void GetListsInfo(std::vector<SBListChunkRanges>* lists) {
 311     lists->clear();
 312     ASSERT_TRUE(database_->UpdateStarted(lists));
 313     database_->UpdateFinished(true);
 314   }
 315
 316   // Helper function to do an AddDel or SubDel command.
 317   void DelChunk(const std::string& list,
 318                 int chunk_id,
 319                 bool is_sub_del) {
 320     std::vector<SBChunkDelete> deletes;
 321     SBChunkDelete chunk_delete;
 322     chunk_delete.list_name = list;
 323     chunk_delete.is_sub_del = is_sub_del;
 324     chunk_delete.chunk_del.push_back(ChunkRange(chunk_id));
 325     deletes.push_back(chunk_delete);
 326     database_->DeleteChunks(deletes);
 327   }
 328
 329   void AddDelChunk(const std::string& list, int chunk_id) {
 330     DelChunk(list, chunk_id, false);
 331   }
 332
 333   void SubDelChunk(const std::string& list, int chunk_id) {
 334     DelChunk(list, chunk_id, true);
 335   }
 336
 337   // Utility function for setting up the database for the caching test.
 338   void PopulateDatabaseForCacheTest();
 339
 340   scoped_refptr<base::TestSimpleTaskRunner> task_runner_;
 341   scoped_ptr<SafeBrowsingDatabaseNew> database_;
 342   base::FilePath database_filename_;
 343   base::ScopedTempDir temp_dir_;
 344 };
 345
 346 // Tests retrieving list name information.
 347 TEST_F(SafeBrowsingDatabaseTest, BrowseListsInfo) {
 348   std::vector<SBListChunkRanges> lists;
 349   ScopedVector<SBChunkData> chunks;
 350
 351   chunks.push_back(AddChunkPrefixValue(1, "www.evil.com/malware.html"));
 352   chunks.push_back(AddChunkPrefixValue(2, "www.foo.com/malware.html"));
 353   chunks.push_back(AddChunkPrefixValue(3, "www.whatever.com/malware.html"));
 354
 355   ASSERT_TRUE(database_->UpdateStarted(&lists));
 356   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
 357   database_->UpdateFinished(true);
 358
 359   GetListsInfo(&lists);
 360   ASSERT_LE(1U, lists.size());
 361   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
 362   EXPECT_EQ("1-3", lists[0].adds);
 363   EXPECT_TRUE(lists[0].subs.empty());
 364
 365   // Insert a malware sub chunk.
 366   chunks.clear();
 367   chunks.push_back(SubChunkPrefixValue(7, "www.subbed.com/noteveil1.html", 19));
 368
 369   ASSERT_TRUE(database_->UpdateStarted(&lists));
 370   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
 371   database_->UpdateFinished(true);
 372
 373   GetListsInfo(&lists);
 374   ASSERT_LE(1U, lists.size());
 375   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
 376   EXPECT_EQ("1-3", lists[0].adds);
 377   EXPECT_EQ("7", lists[0].subs);
 378   if (lists.size() == 2) {
 379     // Old style database won't have the second entry since it creates the lists
 380     // when it receives an update containing that list. The filter-based
 381     // database has these values hard coded.
 382     EXPECT_EQ(safe_browsing_util::kPhishingList, lists[1].name);
 383     EXPECT_TRUE(lists[1].adds.empty());
 384     EXPECT_TRUE(lists[1].subs.empty());
 385   }
 386
 387   // Add phishing chunks.
 388   chunks.clear();
 389   chunks.push_back(AddChunkPrefixValue(47, "www.evil.com/phishing.html"));
 390   chunks.push_back(
 391       SubChunkPrefixValue(200, "www.phishy.com/notevil1.html", 1999));
 392   chunks.push_back(
 393       SubChunkPrefixValue(201, "www.phishy2.com/notevil1.html", 1999));
 394
 395   ASSERT_TRUE(database_->UpdateStarted(&lists));
 396   database_->InsertChunks(safe_browsing_util::kPhishingList, chunks.get());
 397   database_->UpdateFinished(true);
 398
 399   GetListsInfo(&lists);
 400   ASSERT_LE(2U, lists.size());
 401   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
 402   EXPECT_EQ("1-3", lists[0].adds);
 403   EXPECT_EQ("7", lists[0].subs);
 404   EXPECT_EQ(safe_browsing_util::kPhishingList, lists[1].name);
 405   EXPECT_EQ("47", lists[1].adds);
 406   EXPECT_EQ("200-201", lists[1].subs);
 407 }
 408
 409 TEST_F(SafeBrowsingDatabaseTest, ListNames) {
 410   ScopedVector<SBChunkData> chunks;
 411
 412   std::vector<SBListChunkRanges> lists;
 413   ASSERT_TRUE(database_->UpdateStarted(&lists));
 414
 415   // Insert malware, phish, binurl and bindownload add chunks.
 416   chunks.push_back(AddChunkPrefixValue(1, "www.evil.com/malware.html"));
 417   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
 418
 419   chunks.clear();
 420   chunks.push_back(AddChunkPrefixValue(2, "www.foo.com/malware.html"));
 421   database_->InsertChunks(safe_browsing_util::kPhishingList, chunks.get());
 422
 423   chunks.clear();
 424   chunks.push_back(AddChunkPrefixValue(3, "www.whatever.com/download.html"));
 425   database_->InsertChunks(safe_browsing_util::kBinUrlList, chunks.get());
 426
 427   chunks.clear();
 428   chunks.push_back(AddChunkFullHashValue(5, "www.forwhitelist.com/a.html"));
 429   database_->InsertChunks(safe_browsing_util::kCsdWhiteList, chunks.get());
 430
 431   chunks.clear();
 432   chunks.push_back(AddChunkFullHashValue(6, "www.download.com/"));
 433   database_->InsertChunks(safe_browsing_util::kDownloadWhiteList, chunks.get());
 434
 435   chunks.clear();
 436   chunks.push_back(AddChunkFullHashValue(7, "www.inclusion.com/"));
 437   database_->InsertChunks(safe_browsing_util::kInclusionWhitelist,
 438                           chunks.get());
 439
 440   chunks.clear();
 441   chunks.push_back(AddChunkFullHashValue(8,
 442                                          "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
 443                                          "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"));
 444   database_->InsertChunks(safe_browsing_util::kExtensionBlacklist,
 445                           chunks.get());
 446
 447   chunks.clear();
 448   chunks.push_back(AddChunkFullHashValue(9, "www.sideeffectfree.com"));
 449   database_->InsertChunks(safe_browsing_util::kSideEffectFreeWhitelist,
 450                           chunks.get());
 451
 452   chunks.clear();
 453   chunks.push_back(AddChunkHashedIpValue(10, "::ffff:192.168.1.0", 120));
 454   database_->InsertChunks(safe_browsing_util::kIPBlacklist, chunks.get());
 455
 456   chunks.clear();
 457   chunks.push_back(AddChunkPrefixValue(11, "www.unwanted.com/software.html"));
 458   database_->InsertChunks(safe_browsing_util::kUnwantedUrlList, chunks.get());
 459
 460   database_->UpdateFinished(true);
 461
 462   GetListsInfo(&lists);
 463   ASSERT_EQ(10U, lists.size());
 464   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
 465   EXPECT_EQ("1", lists[0].adds);
 466   EXPECT_TRUE(lists[0].subs.empty());
 467   EXPECT_EQ(safe_browsing_util::kPhishingList, lists[1].name);
 468   EXPECT_EQ("2", lists[1].adds);
 469   EXPECT_TRUE(lists[1].subs.empty());
 470   EXPECT_EQ(safe_browsing_util::kBinUrlList, lists[2].name);
 471   EXPECT_EQ("3", lists[2].adds);
 472   EXPECT_TRUE(lists[2].subs.empty());
 473   EXPECT_EQ(safe_browsing_util::kCsdWhiteList, lists[3].name);
 474   EXPECT_EQ("5", lists[3].adds);
 475   EXPECT_TRUE(lists[3].subs.empty());
 476   EXPECT_EQ(safe_browsing_util::kDownloadWhiteList, lists[4].name);
 477   EXPECT_EQ("6", lists[4].adds);
 478   EXPECT_TRUE(lists[4].subs.empty());
 479   EXPECT_EQ(safe_browsing_util::kInclusionWhitelist, lists[5].name);
 480   EXPECT_EQ("7", lists[5].adds);
 481   EXPECT_TRUE(lists[5].subs.empty());
 482   EXPECT_EQ(safe_browsing_util::kExtensionBlacklist, lists[6].name);
 483   EXPECT_EQ("8", lists[6].adds);
 484   EXPECT_TRUE(lists[6].subs.empty());
 485   EXPECT_EQ(safe_browsing_util::kSideEffectFreeWhitelist, lists[7].name);
 486   EXPECT_EQ("9", lists[7].adds);
 487   EXPECT_TRUE(lists[7].subs.empty());
 488   EXPECT_EQ(safe_browsing_util::kIPBlacklist, lists[8].name);
 489   EXPECT_EQ("10", lists[8].adds);
 490   EXPECT_TRUE(lists[8].subs.empty());
 491   EXPECT_EQ(safe_browsing_util::kUnwantedUrlList, lists[9].name);
 492   EXPECT_EQ("11", lists[9].adds);
 493   EXPECT_TRUE(lists[9].subs.empty());
 494
 495   database_.reset();
 496 }
 497
 498 // Checks database reading and writing for browse and unwanted PrefixSets.
 499 TEST_F(SafeBrowsingDatabaseTest, BrowseAndUnwantedDatabasesAndPrefixSets) {
 500   struct TestCase {
 501     using TestListContainsBadUrl = bool (SafeBrowsingDatabase::*)(
 502         const GURL& url,
 503         std::vector<SBPrefix>* prefix_hits,
 504         std::vector<SBFullHashResult>* cache_hits);
 505
 506     const char* test_list_name;
 507     size_t expected_list_index;
 508     TestListContainsBadUrl test_list_contains_bad_url;
 509   } const kTestCases[] {
 510     { safe_browsing_util::kMalwareList, 0U,
 511       &SafeBrowsingDatabase::ContainsBrowseUrl },
 512     { safe_browsing_util::kPhishingList, 1U,
 513       &SafeBrowsingDatabase::ContainsBrowseUrl },
 514     { safe_browsing_util::kUnwantedUrlList, 9U,
 515       &SafeBrowsingDatabase::ContainsUnwantedSoftwareUrl },
 516   };
 517
 518   for (const auto& test_case : kTestCases) {
 519     SCOPED_TRACE(std::string("Tested list at fault => ") +
 520                  test_case.test_list_name);
 521
 522     std::vector<SBListChunkRanges> lists;
 523     ScopedVector<SBChunkData> chunks;
 524
 525     chunks.push_back(AddChunkPrefix2Value(1,
 526                                           "www.evil.com/phishing.html",
 527                                           "www.evil.com/malware.html"));
 528     chunks.push_back(AddChunkPrefix4Value(2,
 529                                           "www.evil.com/notevil1.html",
 530                                           "www.evil.com/notevil2.html",
 531                                           "www.good.com/good1.html",
 532                                           "www.good.com/good2.html"));
 533     chunks.push_back(AddChunkPrefixValue(3, "192.168.0.1/malware.html"));
 534     chunks.push_back(AddChunkFullHashValue(7, "www.evil.com/evil.html"));
 535
 536     ASSERT_TRUE(database_->UpdateStarted(&lists));
 537     database_->InsertChunks(test_case.test_list_name, chunks.get());
 538     database_->UpdateFinished(true);
 539
 540     // Make sure they were added correctly.
 541     GetListsInfo(&lists);
 542
 543     ASSERT_LE(1U, lists.size());
 544     EXPECT_EQ(test_case.test_list_name,
 545               lists[test_case.expected_list_index].name);
 546     EXPECT_EQ("1-3,7", lists[test_case.expected_list_index].adds);
 547     EXPECT_TRUE(lists[test_case.expected_list_index].subs.empty());
 548
 549     std::vector<SBPrefix> prefix_hits;
 550     std::vector<SBFullHashResult> cache_hits;
 551     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 552         GURL("http://www.evil.com/phishing.html"), &prefix_hits, &cache_hits));
 553     ASSERT_EQ(1U, prefix_hits.size());
 554     EXPECT_EQ(SBPrefixForString("www.evil.com/phishing.html"), prefix_hits[0]);
 555     EXPECT_TRUE(cache_hits.empty());
 556
 557     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 558         GURL("http://www.evil.com/malware.html"), &prefix_hits, &cache_hits));
 559
 560     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 561         GURL("http://www.evil.com/notevil1.html"), &prefix_hits, &cache_hits));
 562
 563     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 564         GURL("http://www.evil.com/notevil2.html"), &prefix_hits, &cache_hits));
 565
 566     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 567         GURL("http://www.good.com/good1.html"), &prefix_hits, &cache_hits));
 568
 569     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 570         GURL("http://www.good.com/good2.html"), &prefix_hits, &cache_hits));
 571
 572     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 573         GURL("http://192.168.0.1/malware.html"), &prefix_hits, &cache_hits));
 574
 575     EXPECT_FALSE((database_.get()->*test_case.test_list_contains_bad_url)(
 576         GURL("http://www.evil.com/"), &prefix_hits, &cache_hits));
 577     EXPECT_TRUE(prefix_hits.empty());
 578     EXPECT_TRUE(cache_hits.empty());
 579
 580     EXPECT_FALSE((database_.get()->*test_case.test_list_contains_bad_url)(
 581         GURL("http://www.evil.com/robots.txt"), &prefix_hits, &cache_hits));
 582
 583     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 584         GURL("http://www.evil.com/evil.html"), &prefix_hits, &cache_hits));
 585     ASSERT_EQ(1U, prefix_hits.size());
 586     EXPECT_EQ(SBPrefixForString("www.evil.com/evil.html"), prefix_hits[0]);
 587
 588     // Attempt to re-add the first chunk (should be a no-op).
 589     // see bug: http://code.google.com/p/chromium/issues/detail?id=4522
 590     chunks.clear();
 591     chunks.push_back(AddChunkPrefix2Value(1,
 592                                           "www.evil.com/phishing.html",
 593                                           "www.evil.com/malware.html"));
 594     ASSERT_TRUE(database_->UpdateStarted(&lists));
 595     database_->InsertChunks(test_case.test_list_name, chunks.get());
 596     database_->UpdateFinished(true);
 597
 598     GetListsInfo(&lists);
 599     ASSERT_LE(1U, lists.size());
 600     EXPECT_EQ(test_case.test_list_name,
 601               lists[test_case.expected_list_index].name);
 602     EXPECT_EQ("1-3,7", lists[test_case.expected_list_index].adds);
 603     EXPECT_TRUE(lists[test_case.expected_list_index].subs.empty());
 604
 605     // Test removing a single prefix from the add chunk.
 606     chunks.clear();
 607     chunks.push_back(SubChunkPrefixValue(4, "www.evil.com/notevil1.html", 2));
 608     ASSERT_TRUE(database_->UpdateStarted(&lists));
 609     database_->InsertChunks(test_case.test_list_name, chunks.get());
 610     database_->UpdateFinished(true);
 611
 612     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 613         GURL("http://www.evil.com/phishing.html"), &prefix_hits, &cache_hits));
 614     ASSERT_EQ(1U, prefix_hits.size());
 615     EXPECT_EQ(SBPrefixForString("www.evil.com/phishing.html"), prefix_hits[0]);
 616     EXPECT_TRUE(cache_hits.empty());
 617
 618     EXPECT_FALSE((database_.get()->*test_case.test_list_contains_bad_url)(
 619         GURL("http://www.evil.com/notevil1.html"), &prefix_hits, &cache_hits));
 620     EXPECT_TRUE(prefix_hits.empty());
 621     EXPECT_TRUE(cache_hits.empty());
 622
 623     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 624         GURL("http://www.evil.com/notevil2.html"), &prefix_hits, &cache_hits));
 625
 626     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 627         GURL("http://www.good.com/good1.html"), &prefix_hits, &cache_hits));
 628
 629     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 630         GURL("http://www.good.com/good2.html"), &prefix_hits, &cache_hits));
 631
 632     GetListsInfo(&lists);
 633     ASSERT_LE(1U, lists.size());
 634     EXPECT_EQ(test_case.test_list_name,
 635               lists[test_case.expected_list_index].name);
 636     EXPECT_EQ("1-3,7", lists[test_case.expected_list_index].adds);
 637     EXPECT_EQ("4", lists[test_case.expected_list_index].subs);
 638
 639     // Test the same sub chunk again.  This should be a no-op.
 640     // see bug: http://code.google.com/p/chromium/issues/detail?id=4522
 641     chunks.clear();
 642     chunks.push_back(SubChunkPrefixValue(4, "www.evil.com/notevil1.html", 2));
 643
 644     ASSERT_TRUE(database_->UpdateStarted(&lists));
 645     database_->InsertChunks(test_case.test_list_name, chunks.get());
 646     database_->UpdateFinished(true);
 647
 648     GetListsInfo(&lists);
 649     ASSERT_LE(1U, lists.size());
 650     EXPECT_EQ(test_case.test_list_name,
 651               lists[test_case.expected_list_index].name);
 652     EXPECT_EQ("1-3,7", lists[test_case.expected_list_index].adds);
 653     EXPECT_EQ("4", lists[test_case.expected_list_index].subs);
 654
 655     // Test removing all the prefixes from an add chunk.
 656     ASSERT_TRUE(database_->UpdateStarted(&lists));
 657     AddDelChunk(test_case.test_list_name, 2);
 658     database_->UpdateFinished(true);
 659
 660     EXPECT_FALSE((database_.get()->*test_case.test_list_contains_bad_url)(
 661         GURL("http://www.evil.com/notevil2.html"), &prefix_hits, &cache_hits));
 662
 663     EXPECT_FALSE((database_.get()->*test_case.test_list_contains_bad_url)(
 664         GURL("http://www.good.com/good1.html"), &prefix_hits, &cache_hits));
 665
 666     EXPECT_FALSE((database_.get()->*test_case.test_list_contains_bad_url)(
 667         GURL("http://www.good.com/good2.html"), &prefix_hits, &cache_hits));
 668
 669     GetListsInfo(&lists);
 670     ASSERT_LE(1U, lists.size());
 671     EXPECT_EQ(test_case.test_list_name,
 672               lists[test_case.expected_list_index].name);
 673     EXPECT_EQ("1,3,7", lists[test_case.expected_list_index].adds);
 674     EXPECT_EQ("4", lists[test_case.expected_list_index].subs);
 675
 676     // The adddel command exposed a bug in the transaction code where any
 677     // transaction after it would fail.  Add a dummy entry and remove it to
 678     // make sure the transcation works fine.
 679     chunks.clear();
 680     chunks.push_back(AddChunkPrefixValue(44, "www.redherring.com/index.html"));
 681     ASSERT_TRUE(database_->UpdateStarted(&lists));
 682     database_->InsertChunks(test_case.test_list_name, chunks.get());
 683
 684     // Now remove the dummy entry.  If there are any problems with the
 685     // transactions, asserts will fire.
 686     AddDelChunk(test_case.test_list_name, 44);
 687
 688     // Test the subdel command.
 689     SubDelChunk(test_case.test_list_name, 4);
 690     database_->UpdateFinished(true);
 691
 692     GetListsInfo(&lists);
 693     ASSERT_LE(1U, lists.size());
 694     EXPECT_EQ(test_case.test_list_name,
 695               lists[test_case.expected_list_index].name);
 696     EXPECT_EQ("1,3,7", lists[test_case.expected_list_index].adds);
 697     EXPECT_TRUE(lists[test_case.expected_list_index].subs.empty());
 698
 699     // Test a sub command coming in before the add.
 700     chunks.clear();
 701     chunks.push_back(SubChunkPrefix2Value(5,
 702                                           "www.notevilanymore.com/index.html",
 703                                           10,
 704                                           "www.notevilanymore.com/good.html",
 705                                           10));
 706     ASSERT_TRUE(database_->UpdateStarted(&lists));
 707     database_->InsertChunks(test_case.test_list_name, chunks.get());
 708     database_->UpdateFinished(true);
 709
 710     EXPECT_FALSE((database_.get()->*test_case.test_list_contains_bad_url)(
 711         GURL("http://www.notevilanymore.com/index.html"),
 712         &prefix_hits,
 713         &cache_hits));
 714
 715     // Now insert the tardy add chunk and we don't expect them to appear
 716     // in database because of the previous sub chunk.
 717     chunks.clear();
 718     chunks.push_back(AddChunkPrefix2Value(10,
 719                                           "www.notevilanymore.com/index.html",
 720                                           "www.notevilanymore.com/good.html"));
 721     ASSERT_TRUE(database_->UpdateStarted(&lists));
 722     database_->InsertChunks(test_case.test_list_name, chunks.get());
 723     database_->UpdateFinished(true);
 724
 725     EXPECT_FALSE((database_.get()->*test_case.test_list_contains_bad_url)(
 726         GURL("http://www.notevilanymore.com/index.html"),
 727         &prefix_hits,
 728         &cache_hits));
 729
 730     EXPECT_FALSE((database_.get()->*test_case.test_list_contains_bad_url)(
 731         GURL("http://www.notevilanymore.com/good.html"),
 732         &prefix_hits,
 733         &cache_hits));
 734
 735     // Reset and reload the database.  The database will rely on the prefix set.
 736     ResetAndReloadFullDatabase();
 737
 738     // Check that a prefix still hits.
 739     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 740         GURL("http://www.evil.com/phishing.html"), &prefix_hits, &cache_hits));
 741     ASSERT_EQ(1U, prefix_hits.size());
 742     EXPECT_EQ(SBPrefixForString("www.evil.com/phishing.html"), prefix_hits[0]);
 743
 744     // Also check that it's not just always returning true in this case.
 745     EXPECT_FALSE((database_.get()->*test_case.test_list_contains_bad_url)(
 746         GURL("http://www.evil.com/"), &prefix_hits, &cache_hits));
 747
 748     // Check that the full hash is still present.
 749     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 750         GURL("http://www.evil.com/evil.html"), &prefix_hits, &cache_hits));
 751     ASSERT_EQ(1U, prefix_hits.size());
 752     EXPECT_EQ(SBPrefixForString("www.evil.com/evil.html"), prefix_hits[0]);
 753   }
 754 }
 755
 756 // Test adding zero length chunks to the database.
 757 TEST_F(SafeBrowsingDatabaseTest, ZeroSizeChunk) {
 758   std::vector<SBListChunkRanges> lists;
 759   ScopedVector<SBChunkData> chunks;
 760
 761   // Populate with a couple of normal chunks.
 762   chunks.push_back(AddChunkPrefix2Value(1,
 763                                         "www.test.com/test1.html",
 764                                         "www.test.com/test2.html"));
 765   chunks.push_back(AddChunkPrefix2Value(10,
 766                                         "www.random.com/random1.html",
 767                                         "www.random.com/random2.html"));
 768
 769   ASSERT_TRUE(database_->UpdateStarted(&lists));
 770   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
 771   database_->UpdateFinished(true);
 772
 773   // Add an empty ADD and SUB chunk.
 774   GetListsInfo(&lists);
 775   ASSERT_LE(1U, lists.size());
 776   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
 777   EXPECT_EQ("1,10", lists[0].adds);
 778   EXPECT_TRUE(lists[0].subs.empty());
 779
 780   chunks.clear();
 781   chunks.push_back(BuildChunk(19, safe_browsing::ChunkData::ADD,
 782                               safe_browsing::ChunkData::PREFIX_4B,
 783                               NULL, 0, std::vector<int>()));
 784   chunks.push_back(BuildChunk(7, safe_browsing::ChunkData::SUB,
 785                               safe_browsing::ChunkData::PREFIX_4B,
 786                               NULL, 0, std::vector<int>()));
 787
 788   ASSERT_TRUE(database_->UpdateStarted(&lists));
 789   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
 790   database_->UpdateFinished(true);
 791
 792   GetListsInfo(&lists);
 793   ASSERT_LE(1U, lists.size());
 794   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
 795   EXPECT_EQ("1,10,19", lists[0].adds);
 796   EXPECT_EQ("7", lists[0].subs);
 797
 798   // Add an empty chunk along with a couple that contain data. This should
 799   // result in the chunk range being reduced in size.
 800   chunks.clear();
 801   chunks.push_back(AddChunkPrefixValue(20, "www.notempty.com/full1.html"));
 802   chunks.push_back(BuildChunk(21, safe_browsing::ChunkData::ADD,
 803                               safe_browsing::ChunkData::PREFIX_4B,
 804                               NULL, 0, std::vector<int>()));
 805   chunks.push_back(AddChunkPrefixValue(22, "www.notempty.com/full2.html"));
 806
 807   ASSERT_TRUE(database_->UpdateStarted(&lists));
 808   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
 809   database_->UpdateFinished(true);
 810
 811   std::vector<SBPrefix> prefix_hits;
 812   std::vector<SBFullHashResult> cache_hits;
 813   EXPECT_TRUE(database_->ContainsBrowseUrl(
 814       GURL("http://www.notempty.com/full1.html"), &prefix_hits, &cache_hits));
 815   EXPECT_TRUE(database_->ContainsBrowseUrl(
 816       GURL("http://www.notempty.com/full2.html"), &prefix_hits, &cache_hits));
 817
 818   GetListsInfo(&lists);
 819   ASSERT_LE(1U, lists.size());
 820   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
 821   EXPECT_EQ("1,10,19-22", lists[0].adds);
 822   EXPECT_EQ("7", lists[0].subs);
 823
 824   // Handle AddDel and SubDel commands for empty chunks.
 825   ASSERT_TRUE(database_->UpdateStarted(&lists));
 826   AddDelChunk(safe_browsing_util::kMalwareList, 21);
 827   database_->UpdateFinished(true);
 828
 829   GetListsInfo(&lists);
 830   ASSERT_LE(1U, lists.size());
 831   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
 832   EXPECT_EQ("1,10,19-20,22", lists[0].adds);
 833   EXPECT_EQ("7", lists[0].subs);
 834
 835   ASSERT_TRUE(database_->UpdateStarted(&lists));
 836   SubDelChunk(safe_browsing_util::kMalwareList, 7);
 837   database_->UpdateFinished(true);
 838
 839   GetListsInfo(&lists);
 840   ASSERT_LE(1U, lists.size());
 841   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
 842   EXPECT_EQ("1,10,19-20,22", lists[0].adds);
 843   EXPECT_TRUE(lists[0].subs.empty());
 844 }
 845
 846 // Utility function for setting up the database for the caching test.
 847 void SafeBrowsingDatabaseTest::PopulateDatabaseForCacheTest() {
 848   // Add a couple prefixes.
 849   ScopedVector<SBChunkData> chunks;
 850   chunks.push_back(AddChunkPrefix2Value(1,
 851                                         "www.evil.com/phishing.html",
 852                                         "www.evil.com/malware.html"));
 853
 854   std::vector<SBListChunkRanges> lists;
 855   ASSERT_TRUE(database_->UpdateStarted(&lists));
 856   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
 857   database_->UpdateFinished(true);
 858
 859   // Cache should be cleared after updating.
 860   EXPECT_TRUE(
 861       database_->GetUnsynchronizedPrefixGetHashCacheForTesting()->empty());
 862
 863   SBFullHashResult full_hash;
 864   full_hash.list_id = safe_browsing_util::MALWARE;
 865
 866   std::vector<SBFullHashResult> results;
 867   std::vector<SBPrefix> prefixes;
 868
 869   // Add a fullhash result for each prefix.
 870   full_hash.hash = SBFullHashForString("www.evil.com/phishing.html");
 871   results.push_back(full_hash);
 872   prefixes.push_back(full_hash.hash.prefix);
 873
 874   full_hash.hash = SBFullHashForString("www.evil.com/malware.html");
 875   results.push_back(full_hash);
 876   prefixes.push_back(full_hash.hash.prefix);
 877
 878   database_->CacheHashResults(prefixes, results, kCacheLifetime);
 879 }
 880
 881 TEST_F(SafeBrowsingDatabaseTest, HashCaching) {
 882   PopulateDatabaseForCacheTest();
 883
 884   // We should have both full hashes in the cache.
 885   EXPECT_EQ(2U,
 886             database_->GetUnsynchronizedPrefixGetHashCacheForTesting()->size());
 887
 888   // Test the cache lookup for the first prefix.
 889   std::vector<SBPrefix> prefix_hits;
 890   std::vector<SBFullHashResult> cache_hits;
 891   EXPECT_TRUE(database_->ContainsBrowseUrl(
 892       GURL("http://www.evil.com/phishing.html"), &prefix_hits, &cache_hits));
 893   EXPECT_TRUE(prefix_hits.empty());
 894   ASSERT_EQ(1U, cache_hits.size());
 895   EXPECT_TRUE(SBFullHashEqual(
 896       cache_hits[0].hash, SBFullHashForString("www.evil.com/phishing.html")));
 897
 898   prefix_hits.clear();
 899   cache_hits.clear();
 900
 901   // Test the cache lookup for the second prefix.
 902   EXPECT_TRUE(database_->ContainsBrowseUrl(
 903       GURL("http://www.evil.com/malware.html"), &prefix_hits, &cache_hits));
 904   EXPECT_TRUE(prefix_hits.empty());
 905   ASSERT_EQ(1U, cache_hits.size());
 906   EXPECT_TRUE(SBFullHashEqual(
 907       cache_hits[0].hash, SBFullHashForString("www.evil.com/malware.html")));
 908
 909   prefix_hits.clear();
 910   cache_hits.clear();
 911
 912   // Test removing a prefix via a sub chunk.
 913   ScopedVector<SBChunkData> chunks;
 914   chunks.push_back(SubChunkPrefixValue(2, "www.evil.com/phishing.html", 1));
 915
 916   std::vector<SBListChunkRanges> lists;
 917   ASSERT_TRUE(database_->UpdateStarted(&lists));
 918   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
 919   database_->UpdateFinished(true);
 920
 921   // This prefix should still be there, but cached fullhash should be gone.
 922   EXPECT_TRUE(database_->ContainsBrowseUrl(
 923       GURL("http://www.evil.com/malware.html"), &prefix_hits, &cache_hits));
 924   ASSERT_EQ(1U, prefix_hits.size());
 925   EXPECT_EQ(SBPrefixForString("www.evil.com/malware.html"), prefix_hits[0]);
 926   EXPECT_TRUE(cache_hits.empty());
 927   prefix_hits.clear();
 928   cache_hits.clear();
 929
 930   // This prefix should be gone.
 931   EXPECT_FALSE(database_->ContainsBrowseUrl(
 932       GURL("http://www.evil.com/phishing.html"), &prefix_hits, &cache_hits));
 933   prefix_hits.clear();
 934   cache_hits.clear();
 935
 936   // Test that an AddDel for the original chunk removes the last cached entry.
 937   ASSERT_TRUE(database_->UpdateStarted(&lists));
 938   AddDelChunk(safe_browsing_util::kMalwareList, 1);
 939   database_->UpdateFinished(true);
 940   EXPECT_FALSE(database_->ContainsBrowseUrl(
 941       GURL("http://www.evil.com/malware.html"), &prefix_hits, &cache_hits));
 942   EXPECT_TRUE(
 943       database_->GetUnsynchronizedPrefixGetHashCacheForTesting()->empty());
 944   prefix_hits.clear();
 945   cache_hits.clear();
 946
 947   // Test that the cache won't return expired values. First we have to adjust
 948   // the cached entries' received time to make them older, since the database
 949   // cache insert uses Time::Now(). First, store some entries.
 950   PopulateDatabaseForCacheTest();
 951
 952   SafeBrowsingDatabaseNew::PrefixGetHashCache* hash_cache =
 953       database_->GetUnsynchronizedPrefixGetHashCacheForTesting();
 954   EXPECT_EQ(2U, hash_cache->size());
 955
 956   // Now adjust one of the entries times to be in the past.
 957   const SBPrefix key = SBPrefixForString("www.evil.com/malware.html");
 958   SafeBrowsingDatabaseNew::PrefixGetHashCache::iterator iter =
 959       hash_cache->find(key);
 960   ASSERT_TRUE(iter != hash_cache->end());
 961   iter->second.expire_after = Time::Now() - TimeDelta::FromMinutes(1);
 962
 963   EXPECT_TRUE(database_->ContainsBrowseUrl(
 964       GURL("http://www.evil.com/malware.html"), &prefix_hits, &cache_hits));
 965   EXPECT_EQ(1U, prefix_hits.size());
 966   EXPECT_TRUE(cache_hits.empty());
 967   // Expired entry should have been removed from cache.
 968   EXPECT_EQ(1U, hash_cache->size());
 969
 970   // This entry should still exist.
 971   EXPECT_TRUE(database_->ContainsBrowseUrl(
 972       GURL("http://www.evil.com/phishing.html"), &prefix_hits, &cache_hits));
 973   EXPECT_TRUE(prefix_hits.empty());
 974   EXPECT_EQ(1U, cache_hits.size());
 975
 976   // Testing prefix miss caching. First, we clear out the existing database,
 977   // Since PopulateDatabaseForCacheTest() doesn't handle adding duplicate
 978   // chunks.
 979   ASSERT_TRUE(database_->UpdateStarted(&lists));
 980   AddDelChunk(safe_browsing_util::kMalwareList, 1);
 981   database_->UpdateFinished(true);
 982
 983   // Cache should be cleared after updating.
 984   EXPECT_TRUE(hash_cache->empty());
 985
 986   std::vector<SBPrefix> prefix_misses;
 987   std::vector<SBFullHashResult> empty_full_hash;
 988   prefix_misses.push_back(SBPrefixForString("http://www.bad.com/malware.html"));
 989   prefix_misses.push_back(
 990       SBPrefixForString("http://www.bad.com/phishing.html"));
 991   database_->CacheHashResults(prefix_misses, empty_full_hash, kCacheLifetime);
 992
 993   // Prefixes with no full results are misses.
 994   EXPECT_EQ(hash_cache->size(), prefix_misses.size());
 995   ASSERT_TRUE(
 996       hash_cache->count(SBPrefixForString("http://www.bad.com/malware.html")));
 997   EXPECT_TRUE(
 998       hash_cache->find(SBPrefixForString("http://www.bad.com/malware.html"))
 999           ->second.full_hashes.empty());
1000   ASSERT_TRUE(
1001       hash_cache->count(SBPrefixForString("http://www.bad.com/phishing.html")));
1002   EXPECT_TRUE(
1003       hash_cache->find(SBPrefixForString("http://www.bad.com/phishing.html"))
1004           ->second.full_hashes.empty());
1005
1006   // Update the database.
1007   PopulateDatabaseForCacheTest();
1008
1009   // Cache a GetHash miss for a particular prefix, and even though the prefix is
1010   // in the database, it is flagged as a miss so looking up the associated URL
1011   // will not succeed.
1012   prefix_hits.clear();
1013   cache_hits.clear();
1014   prefix_misses.clear();
1015   empty_full_hash.clear();
1016   prefix_misses.push_back(SBPrefixForString("www.evil.com/phishing.html"));
1017   database_->CacheHashResults(prefix_misses, empty_full_hash, kCacheLifetime);
1018   EXPECT_FALSE(database_->ContainsBrowseUrl(
1019       GURL("http://www.evil.com/phishing.html"), &prefix_hits, &cache_hits));
1020   prefix_hits.clear();
1021   cache_hits.clear();
1022
1023   // Test receiving a full add chunk.
1024   chunks.clear();
1025   chunks.push_back(AddChunkFullHash2Value(20,
1026                                           "www.fullevil.com/bad1.html",
1027                                           "www.fullevil.com/bad2.html"));
1028   ASSERT_TRUE(database_->UpdateStarted(&lists));
1029   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1030   database_->UpdateFinished(true);
1031
1032   EXPECT_TRUE(database_->ContainsBrowseUrl(
1033       GURL("http://www.fullevil.com/bad1.html"), &prefix_hits, &cache_hits));
1034   ASSERT_EQ(1U, prefix_hits.size());
1035   EXPECT_EQ(SBPrefixForString("www.fullevil.com/bad1.html"), prefix_hits[0]);
1036   EXPECT_TRUE(cache_hits.empty());
1037   prefix_hits.clear();
1038   cache_hits.clear();
1039
1040   EXPECT_TRUE(database_->ContainsBrowseUrl(
1041       GURL("http://www.fullevil.com/bad2.html"), &prefix_hits, &cache_hits));
1042   ASSERT_EQ(1U, prefix_hits.size());
1043   EXPECT_EQ(SBPrefixForString("www.fullevil.com/bad2.html"), prefix_hits[0]);
1044   EXPECT_TRUE(cache_hits.empty());
1045   prefix_hits.clear();
1046   cache_hits.clear();
1047
1048   // Test receiving a full sub chunk, which will remove one of the full adds.
1049   chunks.clear();
1050   chunks.push_back(SubChunkFullHashValue(200,
1051                                          "www.fullevil.com/bad1.html",
1052                                          20));
1053   ASSERT_TRUE(database_->UpdateStarted(&lists));
1054   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1055   database_->UpdateFinished(true);
1056
1057   EXPECT_FALSE(database_->ContainsBrowseUrl(
1058       GURL("http://www.fullevil.com/bad1.html"), &prefix_hits, &cache_hits));
1059
1060   // There should be one remaining full add.
1061   EXPECT_TRUE(database_->ContainsBrowseUrl(
1062       GURL("http://www.fullevil.com/bad2.html"), &prefix_hits, &cache_hits));
1063   ASSERT_EQ(1U, prefix_hits.size());
1064   EXPECT_EQ(SBPrefixForString("www.fullevil.com/bad2.html"), prefix_hits[0]);
1065   EXPECT_TRUE(cache_hits.empty());
1066   prefix_hits.clear();
1067   cache_hits.clear();
1068
1069   // Now test an AddDel for the remaining full add.
1070   ASSERT_TRUE(database_->UpdateStarted(&lists));
1071   AddDelChunk(safe_browsing_util::kMalwareList, 20);
1072   database_->UpdateFinished(true);
1073
1074   EXPECT_FALSE(database_->ContainsBrowseUrl(
1075       GURL("http://www.fullevil.com/bad1.html"), &prefix_hits, &cache_hits));
1076   EXPECT_FALSE(database_->ContainsBrowseUrl(
1077       GURL("http://www.fullevil.com/bad2.html"), &prefix_hits, &cache_hits));
1078
1079   // Add a fullhash which has a prefix collision for a known url.
1080   static const char kExampleFine[] = "www.example.com/fine.html";
1081   static const char kExampleCollision[] =
1082       "www.example.com/3123364814/malware.htm";
1083   ASSERT_EQ(SBPrefixForString(kExampleFine),
1084             SBPrefixForString(kExampleCollision));
1085   ASSERT_TRUE(database_->UpdateStarted(&lists));
1086   {
1087     ScopedVector<SBChunkData> chunks;
1088     chunks.push_back(AddChunkPrefixValue(21, kExampleCollision));
1089     database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1090   }
1091   database_->UpdateFinished(true);
1092
1093   // Expect a prefix hit due to the collision between |kExampleFine| and
1094   // |kExampleCollision|.
1095   EXPECT_TRUE(database_->ContainsBrowseUrl(
1096       GURL(std::string("http://") + kExampleFine), &prefix_hits, &cache_hits));
1097   ASSERT_EQ(1U, prefix_hits.size());
1098   EXPECT_EQ(SBPrefixForString(kExampleFine), prefix_hits[0]);
1099   EXPECT_TRUE(cache_hits.empty());
1100
1101   // Cache gethash response for |kExampleCollision|.
1102   {
1103     SBFullHashResult result;
1104     result.hash = SBFullHashForString(kExampleCollision);
1105     result.list_id = safe_browsing_util::MALWARE;
1106     database_->CacheHashResults(std::vector<SBPrefix>(1, result.hash.prefix),
1107                                 std::vector<SBFullHashResult>(1, result),
1108                                 kCacheLifetime);
1109   }
1110
1111   // The cached response means the collision no longer causes a hit.
1112   EXPECT_FALSE(database_->ContainsBrowseUrl(
1113       GURL(std::string("http://") + kExampleFine), &prefix_hits, &cache_hits));
1114 }
1115
1116 // Test that corrupt databases are appropriately handled, even if the
1117 // corruption is detected in the midst of the update.
1118 // TODO(shess): Disabled until ScopedLogMessageIgnorer resolved.
1119 // http://crbug.com/56448
1120 TEST_F(SafeBrowsingDatabaseTest, DISABLED_FileCorruptionHandling) {
1121   // Re-create the database in a captive message loop so that we can
1122   // influence task-posting.  Database specifically needs to the
1123   // file-backed.
1124   database_.reset();
1125   base::MessageLoop loop;
1126   SafeBrowsingStoreFile* store = new SafeBrowsingStoreFile(task_runner_);
1127   database_.reset(new SafeBrowsingDatabaseNew(
1128       task_runner_, store, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL));
1129   database_->Init(database_filename_);
1130
1131   // This will cause an empty database to be created.
1132   std::vector<SBListChunkRanges> lists;
1133   ASSERT_TRUE(database_->UpdateStarted(&lists));
1134   database_->UpdateFinished(true);
1135
1136   // Create a sub chunk to insert.
1137   ScopedVector<SBChunkData> chunks;
1138   chunks.push_back(SubChunkPrefixValue(7,
1139                                        "www.subbed.com/notevil1.html",
1140                                        19));
1141
1142   // Corrupt the file by corrupting the checksum, which is not checked
1143   // until the entire table is read in |UpdateFinished()|.
1144   FILE* fp = base::OpenFile(database_filename_, "r+");
1145   ASSERT_TRUE(fp);
1146   ASSERT_NE(-1, fseek(fp, -8, SEEK_END));
1147   for (size_t i = 0; i < 8; ++i) {
1148     fputc('!', fp);
1149   }
1150   fclose(fp);
1151
1152   {
1153     // The following code will cause DCHECKs, so suppress the crashes.
1154     ScopedLogMessageIgnorer ignorer;
1155
1156     // Start an update.  The insert will fail due to corruption.
1157     ASSERT_TRUE(database_->UpdateStarted(&lists));
1158     database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1159     database_->UpdateFinished(true);
1160
1161     // Database file still exists until the corruption handler has run.
1162     EXPECT_TRUE(base::PathExists(database_filename_));
1163
1164     // Flush through the corruption-handler task.
1165     DVLOG(1) << "Expect failed check on: SafeBrowsing database reset";
1166     base::MessageLoop::current()->RunUntilIdle();
1167   }
1168
1169   // Database file should not exist.
1170   EXPECT_FALSE(base::PathExists(database_filename_));
1171
1172   // Run the update again successfully.
1173   ASSERT_TRUE(database_->UpdateStarted(&lists));
1174   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1175   database_->UpdateFinished(true);
1176   EXPECT_TRUE(base::PathExists(database_filename_));
1177
1178   database_.reset();
1179 }
1180
1181 // Checks database reading and writing.
1182 TEST_F(SafeBrowsingDatabaseTest, ContainsDownloadUrlPrefixes) {
1183   const char kEvil1Url1[] = "www.evil1.com/download1/";
1184   const char kEvil1Url2[] = "www.evil1.com/download2.html";
1185
1186   // Add a simple chunk with one hostkey for download url list.
1187   ScopedVector<SBChunkData> chunks;
1188   chunks.push_back(AddChunkPrefix2Value(1, kEvil1Url1, kEvil1Url2));
1189
1190   std::vector<SBListChunkRanges> lists;
1191   ASSERT_TRUE(database_->UpdateStarted(&lists));
1192   database_->InsertChunks(safe_browsing_util::kBinUrlList, chunks.get());
1193   database_->UpdateFinished(true);
1194
1195   std::vector<SBPrefix> prefix_hits;
1196   std::vector<GURL> urls(1);
1197
1198   urls[0] = GURL(std::string("http://") + kEvil1Url1);
1199   EXPECT_TRUE(ContainsDownloadUrl(urls, &prefix_hits));
1200   ASSERT_EQ(1U, prefix_hits.size());
1201   EXPECT_EQ(SBPrefixForString(kEvil1Url1), prefix_hits[0]);
1202
1203   urls[0] = GURL(std::string("http://") + kEvil1Url2);
1204   EXPECT_TRUE(ContainsDownloadUrl(urls, &prefix_hits));
1205   ASSERT_EQ(1U, prefix_hits.size());
1206   EXPECT_EQ(SBPrefixForString(kEvil1Url2), prefix_hits[0]);
1207
1208   urls[0] = GURL(std::string("https://") + kEvil1Url2);
1209   EXPECT_TRUE(ContainsDownloadUrl(urls, &prefix_hits));
1210   ASSERT_EQ(1U, prefix_hits.size());
1211   EXPECT_EQ(SBPrefixForString(kEvil1Url2), prefix_hits[0]);
1212
1213   urls[0] = GURL(std::string("ftp://") + kEvil1Url2);
1214   EXPECT_TRUE(ContainsDownloadUrl(urls, &prefix_hits));
1215   ASSERT_EQ(1U, prefix_hits.size());
1216   EXPECT_EQ(SBPrefixForString(kEvil1Url2), prefix_hits[0]);
1217
1218   urls[0] = GURL("http://www.randomevil.com");
1219   EXPECT_FALSE(ContainsDownloadUrl(urls, &prefix_hits));
1220
1221   // Should match with query args stripped.
1222   urls[0] = GURL(std::string("http://") + kEvil1Url2 + "?blah");
1223   EXPECT_TRUE(ContainsDownloadUrl(urls, &prefix_hits));
1224   ASSERT_EQ(1U, prefix_hits.size());
1225   EXPECT_EQ(SBPrefixForString(kEvil1Url2), prefix_hits[0]);
1226
1227   // Should match with extra path stuff and query args stripped.
1228   urls[0] = GURL(std::string("http://") + kEvil1Url1 + "foo/bar?blah");
1229   EXPECT_TRUE(ContainsDownloadUrl(urls, &prefix_hits));
1230   ASSERT_EQ(1U, prefix_hits.size());
1231   EXPECT_EQ(SBPrefixForString(kEvil1Url1), prefix_hits[0]);
1232
1233   // First hit in redirect chain is malware.
1234   urls.clear();
1235   urls.push_back(GURL(std::string("http://") + kEvil1Url1));
1236   urls.push_back(GURL("http://www.randomevil.com"));
1237   EXPECT_TRUE(ContainsDownloadUrl(urls, &prefix_hits));
1238   ASSERT_EQ(1U, prefix_hits.size());
1239   EXPECT_EQ(SBPrefixForString(kEvil1Url1), prefix_hits[0]);
1240
1241   // Middle hit in redirect chain is malware.
1242   urls.clear();
1243   urls.push_back(GURL("http://www.randomevil.com"));
1244   urls.push_back(GURL(std::string("http://") + kEvil1Url1));
1245   urls.push_back(GURL("http://www.randomevil2.com"));
1246   EXPECT_TRUE(ContainsDownloadUrl(urls, &prefix_hits));
1247   ASSERT_EQ(1U, prefix_hits.size());
1248   EXPECT_EQ(SBPrefixForString(kEvil1Url1), prefix_hits[0]);
1249
1250   // Final hit in redirect chain is malware.
1251   urls.clear();
1252   urls.push_back(GURL("http://www.randomevil.com"));
1253   urls.push_back(GURL(std::string("http://") + kEvil1Url1));
1254   EXPECT_TRUE(ContainsDownloadUrl(urls, &prefix_hits));
1255   ASSERT_EQ(1U, prefix_hits.size());
1256   EXPECT_EQ(SBPrefixForString(kEvil1Url1), prefix_hits[0]);
1257
1258   // Multiple hits in redirect chain are in malware list.
1259   urls.clear();
1260   urls.push_back(GURL(std::string("http://") + kEvil1Url1));
1261   urls.push_back(GURL(std::string("https://") + kEvil1Url2));
1262   EXPECT_TRUE(ContainsDownloadUrl(urls, &prefix_hits));
1263   ASSERT_EQ(2U, prefix_hits.size());
1264   EXPECT_EQ(SBPrefixForString(kEvil1Url1), prefix_hits[0]);
1265   EXPECT_EQ(SBPrefixForString(kEvil1Url2), prefix_hits[1]);
1266   database_.reset();
1267 }
1268
1269 // Checks that the whitelists are handled properly.
1270 TEST_F(SafeBrowsingDatabaseTest, Whitelists) {
1271   struct TestCase {
1272     using TestListContainsWhitelistedUrl =
1273         bool (SafeBrowsingDatabase::*)(const GURL& url);
1274     using TestListContainsWhitelistedString =
1275         bool (SafeBrowsingDatabase::*)(const std::string& str);
1276
1277     // Returns true if strings should be tested in this test case (i.e.
1278     // |test_list_contains_whitelisted_string| is not null).
1279     bool TestStrings() const {
1280       return test_list_contains_whitelisted_string != nullptr;
1281     }
1282
1283     const char* test_list_name;
1284     TestListContainsWhitelistedUrl test_list_contains_whitelisted_url;
1285     // Optional test case field, if set the tested whitelist will also be tested
1286     // for strings.
1287     TestListContainsWhitelistedString test_list_contains_whitelisted_string;
1288   } const kTestCases[]{
1289       {safe_browsing_util::kCsdWhiteList,
1290        &SafeBrowsingDatabase::ContainsCsdWhitelistedUrl,
1291        nullptr},
1292       {safe_browsing_util::kDownloadWhiteList,
1293        &SafeBrowsingDatabase::ContainsDownloadWhitelistedUrl,
1294        &SafeBrowsingDatabase::ContainsDownloadWhitelistedString},
1295       {safe_browsing_util::kInclusionWhitelist,
1296        &SafeBrowsingDatabase::ContainsInclusionWhitelistedUrl,
1297        nullptr},
1298   };
1299
1300   // If the whitelist is disabled everything should match the whitelist.
1301   database_.reset(new SafeBrowsingDatabaseNew(
1302       task_runner_, new SafeBrowsingStoreFile(task_runner_), NULL, NULL, NULL,
1303       NULL, NULL, NULL, NULL, NULL));
1304   database_->Init(database_filename_);
1305   for (const auto& test_case : kTestCases) {
1306     SCOPED_TRACE(std::string("Tested list at fault => ") +
1307                  test_case.test_list_name);
1308
1309     EXPECT_TRUE(
1310         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1311             GURL(std::string("http://www.phishing.com/"))));
1312     if (test_case.TestStrings()) {
1313       EXPECT_TRUE(
1314           (database_.get()->*test_case.test_list_contains_whitelisted_string)(
1315               "asdf"));
1316     }
1317   }
1318   ResetAndReloadFullDatabase();
1319
1320   // Now test every whitelist one-by-one; intentionally not resetting the
1321   // database in-between to further stress potential inter-dependencies.
1322   for (const auto& test_case : kTestCases) {
1323     SCOPED_TRACE(std::string("Tested list at fault => ") +
1324                  test_case.test_list_name);
1325
1326     const char kGood1Host[] = "www.good1.com/";
1327     const char kGood1Url1[] = "www.good1.com/a/b.html";
1328     const char kGood1Url2[] = "www.good1.com/b/";
1329
1330     const char kGood2Url1[] = "www.good2.com/c";  // Should match '/c/bla'.
1331
1332     // good3.com/a/b/c/d/e/f/g/ should match because it's a whitelist.
1333     const char kGood3Url1[] = "good3.com/";
1334
1335     const char kGoodString[] = "good_string";
1336
1337     // Nothing should be whitelisted before the database receives the chunks.
1338     EXPECT_FALSE(
1339         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1340             GURL(std::string("http://") + kGood1Host)));
1341     EXPECT_FALSE(
1342         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1343             GURL(std::string("http://") + kGood1Url1)));
1344     EXPECT_FALSE(
1345         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1346             GURL(std::string("http://") + kGood1Url2)));
1347     EXPECT_FALSE(
1348         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1349             GURL(std::string("http://") + kGood2Url1)));
1350     EXPECT_FALSE(
1351         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1352             GURL(std::string("http://") + kGood3Url1)));
1353     if (test_case.TestStrings()) {
1354       EXPECT_FALSE(
1355           (database_.get()->*test_case.test_list_contains_whitelisted_string)(
1356               kGoodString));
1357     }
1358
1359     ScopedVector<SBChunkData> chunks;
1360
1361     // Add a few test chunks to the whitelist under test.
1362     chunks.push_back(AddChunkFullHash2Value(1, kGood1Url1, kGood1Url2));
1363     chunks.push_back(AddChunkFullHashValue(2, kGood2Url1));
1364     if (test_case.TestStrings())
1365       chunks.push_back(AddChunkFullHashValue(3, kGoodString));
1366     chunks.push_back(AddChunkFullHashValue(4, kGood3Url1));
1367
1368     std::vector<SBListChunkRanges> lists;
1369     ASSERT_TRUE(database_->UpdateStarted(&lists));
1370     database_->InsertChunks(test_case.test_list_name, chunks.get());
1371     database_->UpdateFinished(true);
1372
1373     EXPECT_FALSE(
1374         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1375             GURL(std::string("http://") + kGood1Host)));
1376
1377     EXPECT_TRUE(
1378         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1379             GURL(std::string("http://") + kGood1Url1)));
1380     EXPECT_TRUE(
1381         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1382             GURL(std::string("http://") + kGood1Url1 + "?a=b")));
1383
1384     EXPECT_TRUE(
1385         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1386             GURL(std::string("http://") + kGood1Url2)));
1387     EXPECT_TRUE(
1388         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1389             GURL(std::string("http://") + kGood1Url2 + "/c.html")));
1390
1391     EXPECT_TRUE(
1392         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1393             GURL(std::string("https://") + kGood1Url2 + "/c.html")));
1394
1395     EXPECT_TRUE(
1396         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1397             GURL(std::string("http://") + kGood2Url1 + "/c")));
1398     EXPECT_TRUE(
1399         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1400             GURL(std::string("http://") + kGood2Url1 + "/c?bla")));
1401     EXPECT_TRUE(
1402         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1403             GURL(std::string("http://") + kGood2Url1 + "/c/bla")));
1404
1405     EXPECT_FALSE(
1406         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1407             GURL(std::string("http://www.google.com/"))));
1408
1409     EXPECT_TRUE(
1410         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1411             GURL(std::string("http://") + kGood3Url1 + "a/b/c/d/e/f/g/")));
1412     EXPECT_TRUE(
1413         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1414             GURL(std::string("http://a.b.") + kGood3Url1)));
1415
1416     if (test_case.TestStrings()) {
1417       EXPECT_FALSE(
1418           (database_.get()->*test_case.test_list_contains_whitelisted_string)(
1419               "asdf"));
1420       EXPECT_TRUE(
1421           (database_.get()->*test_case.test_list_contains_whitelisted_string)(
1422               kGoodString));
1423     }
1424
1425     EXPECT_FALSE(
1426         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1427             GURL(std::string("http://www.google.com/"))));
1428
1429     // The malware kill switch is for the CSD whitelist only.
1430     if (test_case.test_list_name == safe_browsing_util::kCsdWhiteList) {
1431       // The CSD whitelist killswitch is not present.
1432       EXPECT_FALSE(database_->IsCsdWhitelistKillSwitchOn());
1433
1434       // Test only add the malware IP killswitch
1435       chunks.clear();
1436       chunks.push_back(AddChunkFullHashValue(
1437           15, "sb-ssl.google.com/safebrowsing/csd/killswitch_malware"));
1438
1439       ASSERT_TRUE(database_->UpdateStarted(&lists));
1440       database_->InsertChunks(safe_browsing_util::kCsdWhiteList, chunks.get());
1441       database_->UpdateFinished(true);
1442
1443       EXPECT_TRUE(database_->IsMalwareIPMatchKillSwitchOn());
1444       // The CSD whitelist killswitch is not present.
1445       EXPECT_FALSE(database_->IsCsdWhitelistKillSwitchOn());
1446     }
1447
1448     // Test that the generic whitelist kill-switch works as intended.
1449     chunks.clear();
1450     lists.clear();
1451     chunks.push_back(AddChunkFullHashValue(
1452         5, "sb-ssl.google.com/safebrowsing/csd/killswitch"));
1453
1454     ASSERT_TRUE(database_->UpdateStarted(&lists));
1455     database_->InsertChunks(test_case.test_list_name, chunks.get());
1456     database_->UpdateFinished(true);
1457
1458     // Test CSD whitelist specific methods.
1459     if (test_case.test_list_name == safe_browsing_util::kCsdWhiteList) {
1460       // The CSD whitelist killswitch is present.
1461       EXPECT_TRUE(database_->IsCsdWhitelistKillSwitchOn());
1462       EXPECT_TRUE(database_->IsMalwareIPMatchKillSwitchOn());
1463     }
1464
1465     EXPECT_TRUE(
1466         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1467             GURL(std::string("https://") + kGood1Url2 + "/c.html")));
1468     EXPECT_TRUE(
1469         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1470             GURL(std::string("http://www.google.com/"))));
1471     EXPECT_TRUE(
1472         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1473             GURL(std::string("http://www.phishing_url.com/"))));
1474
1475     if (test_case.TestStrings()) {
1476       EXPECT_TRUE(
1477           (database_.get()->*test_case.test_list_contains_whitelisted_string)(
1478               "asdf"));
1479       EXPECT_TRUE(
1480           (database_.get()->*test_case.test_list_contains_whitelisted_string)(
1481               kGoodString));
1482     }
1483
1484     // Remove the kill-switch and verify that we can recover.
1485     chunks.clear();
1486     lists.clear();
1487     chunks.push_back(SubChunkFullHashValue(
1488         1, "sb-ssl.google.com/safebrowsing/csd/killswitch", 5));
1489     if (test_case.test_list_name == safe_browsing_util::kCsdWhiteList) {
1490       chunks.push_back(SubChunkFullHashValue(
1491           10, "sb-ssl.google.com/safebrowsing/csd/killswitch_malware", 15));
1492     }
1493
1494     ASSERT_TRUE(database_->UpdateStarted(&lists));
1495     database_->InsertChunks(test_case.test_list_name, chunks.get());
1496     database_->UpdateFinished(true);
1497
1498     if (test_case.test_list_name == safe_browsing_util::kCsdWhiteList) {
1499       EXPECT_FALSE(database_->IsMalwareIPMatchKillSwitchOn());
1500       EXPECT_FALSE(database_->IsCsdWhitelistKillSwitchOn());
1501     }
1502     EXPECT_TRUE(
1503         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1504             GURL(std::string("https://") + kGood1Url2 + "/c.html")));
1505     EXPECT_TRUE(
1506         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1507             GURL(std::string("https://") + kGood2Url1 + "/c/bla")));
1508     EXPECT_TRUE(
1509         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1510             GURL(std::string("https://") + kGood3Url1)));
1511     EXPECT_FALSE(
1512         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1513             GURL(std::string("http://www.google.com/"))));
1514     EXPECT_FALSE(
1515         (database_.get()->*test_case.test_list_contains_whitelisted_url)(
1516             GURL(std::string("http://www.phishing_url.com/"))));
1517     if (test_case.TestStrings()) {
1518       EXPECT_TRUE(
1519           (database_.get()->*test_case.test_list_contains_whitelisted_string)(
1520               kGoodString));
1521       EXPECT_FALSE(
1522           (database_.get()->*test_case.test_list_contains_whitelisted_string)(
1523               "asdf"));
1524     }
1525   }
1526 }
1527
1528 // Test to make sure we could insert chunk list that
1529 // contains entries for the same host.
1530 TEST_F(SafeBrowsingDatabaseTest, SameHostEntriesOkay) {
1531   ScopedVector<SBChunkData> chunks;
1532
1533   // Add a malware add chunk with two entries of the same host.
1534   chunks.push_back(AddChunkPrefix2Value(1,
1535                                         "www.evil.com/malware1.html",
1536                                         "www.evil.com/malware2.html"));
1537
1538   // Insert the testing chunks into database.
1539   std::vector<SBListChunkRanges> lists;
1540   ASSERT_TRUE(database_->UpdateStarted(&lists));
1541   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1542   database_->UpdateFinished(true);
1543
1544   GetListsInfo(&lists);
1545   ASSERT_LE(1U, lists.size());
1546   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
1547   EXPECT_EQ("1", lists[0].adds);
1548   EXPECT_TRUE(lists[0].subs.empty());
1549
1550   // Add a phishing add chunk with two entries of the same host.
1551   chunks.clear();
1552   chunks.push_back(AddChunkPrefix2Value(47,
1553                                         "www.evil.com/phishing1.html",
1554                                         "www.evil.com/phishing2.html"));
1555
1556   ASSERT_TRUE(database_->UpdateStarted(&lists));
1557   database_->InsertChunks(safe_browsing_util::kPhishingList, chunks.get());
1558   database_->UpdateFinished(true);
1559
1560   GetListsInfo(&lists);
1561   ASSERT_LE(2U, lists.size());
1562   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
1563   EXPECT_EQ("1", lists[0].adds);
1564   EXPECT_TRUE(lists[0].subs.empty());
1565   EXPECT_EQ(safe_browsing_util::kPhishingList, lists[1].name);
1566   EXPECT_EQ("47", lists[1].adds);
1567   EXPECT_TRUE(lists[1].subs.empty());
1568
1569   std::vector<SBPrefix> prefix_hits;
1570   std::vector<SBFullHashResult> cache_hits;
1571
1572   EXPECT_TRUE(database_->ContainsBrowseUrl(
1573       GURL("http://www.evil.com/malware1.html"), &prefix_hits, &cache_hits));
1574   EXPECT_TRUE(database_->ContainsBrowseUrl(
1575       GURL("http://www.evil.com/malware2.html"), &prefix_hits, &cache_hits));
1576   EXPECT_TRUE(database_->ContainsBrowseUrl(
1577       GURL("http://www.evil.com/phishing1.html"), &prefix_hits, &cache_hits));
1578   EXPECT_TRUE(database_->ContainsBrowseUrl(
1579       GURL("http://www.evil.com/phishing2.html"), &prefix_hits, &cache_hits));
1580
1581   // Test removing a single prefix from the add chunk.
1582   // Remove the prefix that added first.
1583   chunks.clear();
1584   chunks.push_back(SubChunkPrefixValue(4, "www.evil.com/malware1.html", 1));
1585   ASSERT_TRUE(database_->UpdateStarted(&lists));
1586   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1587   database_->UpdateFinished(true);
1588
1589   // Remove the prefix that added last.
1590   chunks.clear();
1591   chunks.push_back(SubChunkPrefixValue(5, "www.evil.com/phishing2.html", 47));
1592   ASSERT_TRUE(database_->UpdateStarted(&lists));
1593   database_->InsertChunks(safe_browsing_util::kPhishingList, chunks.get());
1594   database_->UpdateFinished(true);
1595
1596   // Verify that the database contains urls expected.
1597   EXPECT_FALSE(database_->ContainsBrowseUrl(
1598       GURL("http://www.evil.com/malware1.html"), &prefix_hits, &cache_hits));
1599   EXPECT_TRUE(database_->ContainsBrowseUrl(
1600       GURL("http://www.evil.com/malware2.html"), &prefix_hits, &cache_hits));
1601   EXPECT_TRUE(database_->ContainsBrowseUrl(
1602       GURL("http://www.evil.com/phishing1.html"), &prefix_hits, &cache_hits));
1603   EXPECT_FALSE(database_->ContainsBrowseUrl(
1604       GURL("http://www.evil.com/phishing2.html"), &prefix_hits, &cache_hits));
1605 }
1606
1607 // Test that an empty update doesn't actually update the database.
1608 // This isn't a functionality requirement, but it is a useful
1609 // optimization.
1610 TEST_F(SafeBrowsingDatabaseTest, EmptyUpdate) {
1611   ScopedVector<SBChunkData> chunks;
1612
1613   base::FilePath filename = database_->BrowseDBFilename(database_filename_);
1614
1615   // Prime the database.
1616   std::vector<SBListChunkRanges> lists;
1617   ASSERT_TRUE(database_->UpdateStarted(&lists));
1618   chunks.push_back(AddChunkPrefixValue(1, "www.evil.com/malware.html"));
1619   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1620   database_->UpdateFinished(true);
1621
1622   // Get an older time to reset the lastmod time for detecting whether
1623   // the file has been updated.
1624   base::File::Info before_info, after_info;
1625   ASSERT_TRUE(base::GetFileInfo(filename, &before_info));
1626   const Time old_last_modified =
1627       before_info.last_modified - TimeDelta::FromSeconds(10);
1628
1629   // Inserting another chunk updates the database file.  The sleep is
1630   // needed because otherwise the entire test can finish w/in the
1631   // resolution of the lastmod time.
1632   ASSERT_TRUE(base::TouchFile(filename, old_last_modified, old_last_modified));
1633   ASSERT_TRUE(base::GetFileInfo(filename, &before_info));
1634   ASSERT_TRUE(database_->UpdateStarted(&lists));
1635   chunks.push_back(AddChunkPrefixValue(2, "www.foo.com/malware.html"));
1636   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1637   database_->UpdateFinished(true);
1638   ASSERT_TRUE(base::GetFileInfo(filename, &after_info));
1639   EXPECT_LT(before_info.last_modified, after_info.last_modified);
1640
1641   // Deleting a chunk updates the database file.
1642   ASSERT_TRUE(base::TouchFile(filename, old_last_modified, old_last_modified));
1643   ASSERT_TRUE(base::GetFileInfo(filename, &before_info));
1644   ASSERT_TRUE(database_->UpdateStarted(&lists));
1645   AddDelChunk(safe_browsing_util::kMalwareList, 2);
1646   database_->UpdateFinished(true);
1647   ASSERT_TRUE(base::GetFileInfo(filename, &after_info));
1648   EXPECT_LT(before_info.last_modified, after_info.last_modified);
1649
1650   // Simply calling |UpdateStarted()| then |UpdateFinished()| does not
1651   // update the database file.
1652   ASSERT_TRUE(base::TouchFile(filename, old_last_modified, old_last_modified));
1653   ASSERT_TRUE(base::GetFileInfo(filename, &before_info));
1654   ASSERT_TRUE(database_->UpdateStarted(&lists));
1655   database_->UpdateFinished(true);
1656   ASSERT_TRUE(base::GetFileInfo(filename, &after_info));
1657   EXPECT_EQ(before_info.last_modified, after_info.last_modified);
1658 }
1659
1660 // Test that a filter file is written out during update and read back
1661 // in during setup.
1662 TEST_F(SafeBrowsingDatabaseTest, FilterFile) {
1663   // Create a database with trivial example data and write it out.
1664   {
1665     // Prime the database.
1666     std::vector<SBListChunkRanges> lists;
1667     ASSERT_TRUE(database_->UpdateStarted(&lists));
1668
1669     ScopedVector<SBChunkData> chunks;
1670     chunks.push_back(AddChunkPrefixValue(1, "www.evil.com/malware.html"));
1671     database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1672     database_->UpdateFinished(true);
1673   }
1674
1675   // Find the malware url in the database, don't find a good url.
1676   std::vector<SBPrefix> prefix_hits;
1677   std::vector<SBFullHashResult> cache_hits;
1678   EXPECT_TRUE(database_->ContainsBrowseUrl(
1679       GURL("http://www.evil.com/malware.html"), &prefix_hits, &cache_hits));
1680   EXPECT_FALSE(database_->ContainsBrowseUrl(
1681       GURL("http://www.good.com/goodware.html"), &prefix_hits, &cache_hits));
1682
1683   base::FilePath filter_file = database_->PrefixSetForFilename(
1684       database_->BrowseDBFilename(database_filename_));
1685
1686   // After re-creating the database, it should have a filter read from
1687   // a file, so it should find the same results.
1688   ASSERT_TRUE(base::PathExists(filter_file));
1689   ResetAndReloadFullDatabase();
1690   EXPECT_TRUE(database_->ContainsBrowseUrl(
1691       GURL("http://www.evil.com/malware.html"), &prefix_hits, &cache_hits));
1692   EXPECT_FALSE(database_->ContainsBrowseUrl(
1693       GURL("http://www.good.com/goodware.html"), &prefix_hits, &cache_hits));
1694
1695   // If there is no filter file, the database cannot find malware urls.
1696   base::DeleteFile(filter_file, false);
1697   ASSERT_FALSE(base::PathExists(filter_file));
1698   ResetAndReloadFullDatabase();
1699   EXPECT_FALSE(database_->ContainsBrowseUrl(
1700       GURL("http://www.evil.com/malware.html"), &prefix_hits, &cache_hits));
1701   EXPECT_FALSE(database_->ContainsBrowseUrl(
1702       GURL("http://www.good.com/goodware.html"), &prefix_hits, &cache_hits));
1703 }
1704
1705 TEST_F(SafeBrowsingDatabaseTest, CachedFullMiss) {
1706   const SBPrefix kPrefix1 = 1001U;
1707   const SBFullHash kFullHash1_1 =
1708       SBFullHashForPrefixAndSuffix(kPrefix1, "\x01");
1709
1710   const SBPrefix kPrefix2 = 1002U;
1711   const SBFullHash kFullHash2_1 =
1712       SBFullHashForPrefixAndSuffix(kPrefix2, "\x01");
1713
1714   // Insert prefix kPrefix1 and kPrefix2 into database.
1715   ScopedVector<SBChunkData> chunks;
1716   chunks.push_back(AddChunkPrefix(1, kPrefix1));
1717   chunks.push_back(AddChunkPrefix(2, kPrefix2));
1718
1719   std::vector<SBListChunkRanges> lists;
1720   ASSERT_TRUE(database_->UpdateStarted(&lists));
1721   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1722   database_->UpdateFinished(true);
1723
1724   {
1725     // Cache a full miss result for kPrefix1.
1726     std::vector<SBPrefix> prefixes(1, kPrefix1);
1727     std::vector<SBFullHashResult> cache_results;
1728     database_->CacheHashResults(prefixes, cache_results, kCacheLifetime);
1729   }
1730
1731   {
1732     // kFullHash1_1 gets no prefix hit because of the cached item, and also does
1733     // not have a cache hit.
1734     std::vector<SBFullHash> full_hashes(1, kFullHash1_1);
1735     std::vector<SBPrefix> prefix_hits;
1736     std::vector<SBFullHashResult> cache_hits;
1737     EXPECT_FALSE(database_->ContainsBrowseUrlHashesForTesting(
1738         full_hashes, &prefix_hits, &cache_hits));
1739
1740     // kFullHash2_1 gets a hit from the prefix in the database.
1741     full_hashes.push_back(kFullHash2_1);
1742     prefix_hits.clear();
1743     cache_hits.clear();
1744     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
1745         full_hashes, &prefix_hits, &cache_hits));
1746     ASSERT_EQ(1U, prefix_hits.size());
1747     EXPECT_EQ(kPrefix2, prefix_hits[0]);
1748     EXPECT_TRUE(cache_hits.empty());
1749   }
1750 }
1751
1752 TEST_F(SafeBrowsingDatabaseTest, CachedPrefixHitFullMiss) {
1753   const SBPrefix kPrefix1 = 1001U;
1754   const SBFullHash kFullHash1_1 =
1755       SBFullHashForPrefixAndSuffix(kPrefix1, "\x01");
1756   const SBFullHash kFullHash1_2 =
1757       SBFullHashForPrefixAndSuffix(kPrefix1, "\x02");
1758   const SBFullHash kFullHash1_3 =
1759       SBFullHashForPrefixAndSuffix(kPrefix1, "\x03");
1760
1761   const SBPrefix kPrefix2 = 1002U;
1762   const SBFullHash kFullHash2_1 =
1763       SBFullHashForPrefixAndSuffix(kPrefix2, "\x01");
1764
1765   const SBPrefix kPrefix3 = 1003U;
1766   const SBFullHash kFullHash3_1 =
1767       SBFullHashForPrefixAndSuffix(kPrefix3, "\x01");
1768
1769   // Insert prefix kPrefix1 and kPrefix2 into database.
1770   ScopedVector<SBChunkData> chunks;
1771   chunks.push_back(AddChunkPrefix(1, kPrefix1));
1772   chunks.push_back(AddChunkPrefix(2, kPrefix2));
1773
1774   std::vector<SBListChunkRanges> lists;
1775   ASSERT_TRUE(database_->UpdateStarted(&lists));
1776   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1777   database_->UpdateFinished(true);
1778
1779   {
1780     // kFullHash1_1 has a prefix hit of kPrefix1.
1781     std::vector<SBFullHash> full_hashes;
1782     full_hashes.push_back(kFullHash1_1);
1783     std::vector<SBPrefix> prefix_hits;
1784     std::vector<SBFullHashResult> cache_hits;
1785     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
1786         full_hashes, &prefix_hits, &cache_hits));
1787     ASSERT_EQ(1U, prefix_hits.size());
1788     EXPECT_EQ(kPrefix1, prefix_hits[0]);
1789     EXPECT_TRUE(cache_hits.empty());
1790
1791     // kFullHash2_1 has a prefix hit of kPrefix2.
1792     full_hashes.push_back(kFullHash2_1);
1793     prefix_hits.clear();
1794     cache_hits.clear();
1795     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
1796         full_hashes, &prefix_hits, &cache_hits));
1797     ASSERT_EQ(2U, prefix_hits.size());
1798     EXPECT_EQ(kPrefix1, prefix_hits[0]);
1799     EXPECT_EQ(kPrefix2, prefix_hits[1]);
1800     EXPECT_TRUE(cache_hits.empty());
1801
1802     // kFullHash3_1 has no hits.
1803     full_hashes.push_back(kFullHash3_1);
1804     prefix_hits.clear();
1805     cache_hits.clear();
1806     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
1807         full_hashes, &prefix_hits, &cache_hits));
1808     ASSERT_EQ(2U, prefix_hits.size());
1809     EXPECT_EQ(kPrefix1, prefix_hits[0]);
1810     EXPECT_EQ(kPrefix2, prefix_hits[1]);
1811     EXPECT_TRUE(cache_hits.empty());
1812   }
1813
1814   {
1815     // Cache a fullhash result for two kPrefix1 full hashes.
1816     std::vector<SBPrefix> prefixes(1, kPrefix1);
1817     std::vector<SBFullHashResult> cache_results;
1818
1819     SBFullHashResult full_hash_result;
1820     full_hash_result.list_id = safe_browsing_util::MALWARE;
1821
1822     full_hash_result.hash = kFullHash1_1;
1823     cache_results.push_back(full_hash_result);
1824
1825     full_hash_result.hash = kFullHash1_3;
1826     cache_results.push_back(full_hash_result);
1827
1828     database_->CacheHashResults(prefixes, cache_results, kCacheLifetime);
1829   }
1830
1831   {
1832     // kFullHash1_1 should now see a cache hit.
1833     std::vector<SBFullHash> full_hashes(1, kFullHash1_1);
1834     std::vector<SBPrefix> prefix_hits;
1835     std::vector<SBFullHashResult> cache_hits;
1836     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
1837         full_hashes, &prefix_hits, &cache_hits));
1838     EXPECT_TRUE(prefix_hits.empty());
1839     ASSERT_EQ(1U, cache_hits.size());
1840     EXPECT_TRUE(SBFullHashEqual(kFullHash1_1, cache_hits[0].hash));
1841
1842     // Adding kFullHash2_1 will see the existing cache hit plus the prefix hit
1843     // for kPrefix2.
1844     full_hashes.push_back(kFullHash2_1);
1845     prefix_hits.clear();
1846     cache_hits.clear();
1847     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
1848         full_hashes, &prefix_hits, &cache_hits));
1849     ASSERT_EQ(1U, prefix_hits.size());
1850     EXPECT_EQ(kPrefix2, prefix_hits[0]);
1851     ASSERT_EQ(1U, cache_hits.size());
1852     EXPECT_TRUE(SBFullHashEqual(kFullHash1_1, cache_hits[0].hash));
1853
1854     // kFullHash1_3 also gets a cache hit.
1855     full_hashes.push_back(kFullHash1_3);
1856     prefix_hits.clear();
1857     cache_hits.clear();
1858     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
1859         full_hashes, &prefix_hits, &cache_hits));
1860     ASSERT_EQ(1U, prefix_hits.size());
1861     EXPECT_EQ(kPrefix2, prefix_hits[0]);
1862     ASSERT_EQ(2U, cache_hits.size());
1863     EXPECT_TRUE(SBFullHashEqual(kFullHash1_1, cache_hits[0].hash));
1864     EXPECT_TRUE(SBFullHashEqual(kFullHash1_3, cache_hits[1].hash));
1865   }
1866
1867   {
1868     // Check if DB contains only kFullHash1_3. Should return a cache hit.
1869     std::vector<SBFullHash> full_hashes(1, kFullHash1_3);
1870     std::vector<SBPrefix> prefix_hits;
1871     std::vector<SBFullHashResult> cache_hits;
1872     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
1873         full_hashes, &prefix_hits, &cache_hits));
1874     EXPECT_TRUE(prefix_hits.empty());
1875     ASSERT_EQ(1U, cache_hits.size());
1876     EXPECT_TRUE(SBFullHashEqual(kFullHash1_3, cache_hits[0].hash));
1877   }
1878
1879   {
1880     // kFullHash1_2 has no cache hit, and no prefix hit because of the cache for
1881     // kPrefix1.
1882     std::vector<SBFullHash> full_hashes(1, kFullHash1_2);
1883     std::vector<SBPrefix> prefix_hits;
1884     std::vector<SBFullHashResult> cache_hits;
1885     EXPECT_FALSE(database_->ContainsBrowseUrlHashesForTesting(
1886         full_hashes, &prefix_hits, &cache_hits));
1887
1888     // Other prefix hits possible when kFullHash1_2 hits nothing.
1889     full_hashes.push_back(kFullHash2_1);
1890     prefix_hits.clear();
1891     cache_hits.clear();
1892     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
1893         full_hashes, &prefix_hits, &cache_hits));
1894     ASSERT_EQ(1U, prefix_hits.size());
1895     EXPECT_EQ(kPrefix2, prefix_hits[0]);
1896     EXPECT_TRUE(cache_hits.empty());
1897   }
1898 }
1899
1900 TEST_F(SafeBrowsingDatabaseTest, BrowseFullHashMatching) {
1901   const SBPrefix kPrefix1 = 1001U;
1902   const SBFullHash kFullHash1_1 =
1903       SBFullHashForPrefixAndSuffix(kPrefix1, "\x01");
1904   const SBFullHash kFullHash1_2 =
1905       SBFullHashForPrefixAndSuffix(kPrefix1, "\x02");
1906   const SBFullHash kFullHash1_3 =
1907       SBFullHashForPrefixAndSuffix(kPrefix1, "\x03");
1908
1909   // Insert two full hashes with a shared prefix.
1910   ScopedVector<SBChunkData> chunks;
1911   chunks.push_back(AddChunkFullHash(1, kFullHash1_1));
1912   chunks.push_back(AddChunkFullHash(2, kFullHash1_2));
1913
1914   std::vector<SBListChunkRanges> lists;
1915   ASSERT_TRUE(database_->UpdateStarted(&lists));
1916   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1917   database_->UpdateFinished(true);
1918
1919   {
1920     // Check a full hash which isn't present.
1921     std::vector<SBFullHash> full_hashes(1, kFullHash1_3);
1922     std::vector<SBPrefix> prefix_hits;
1923     std::vector<SBFullHashResult> cache_hits;
1924     EXPECT_FALSE(database_->ContainsBrowseUrlHashesForTesting(
1925         full_hashes, &prefix_hits, &cache_hits));
1926
1927     // Also one which is present, should have a prefix hit.
1928     full_hashes.push_back(kFullHash1_1);
1929     prefix_hits.clear();
1930     cache_hits.clear();
1931     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
1932         full_hashes, &prefix_hits, &cache_hits));
1933     ASSERT_EQ(1U, prefix_hits.size());
1934     EXPECT_EQ(kPrefix1, prefix_hits[0]);
1935     EXPECT_TRUE(cache_hits.empty());
1936
1937     // Two full hash matches with the same prefix should return one prefix hit.
1938     full_hashes.push_back(kFullHash1_2);
1939     prefix_hits.clear();
1940     cache_hits.clear();
1941     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
1942         full_hashes, &prefix_hits, &cache_hits));
1943     ASSERT_EQ(1U, prefix_hits.size());
1944     EXPECT_EQ(kPrefix1, prefix_hits[0]);
1945     EXPECT_TRUE(cache_hits.empty());
1946   }
1947
1948   {
1949     // Cache a gethash result for kFullHash1_2.
1950     SBFullHashResult full_hash_result;
1951     full_hash_result.list_id = safe_browsing_util::MALWARE;
1952     full_hash_result.hash = kFullHash1_2;
1953
1954     std::vector<SBPrefix> prefixes(1, kPrefix1);
1955     std::vector<SBFullHashResult> cache_results(1, full_hash_result);
1956
1957     database_->CacheHashResults(prefixes, cache_results, kCacheLifetime);
1958   }
1959
1960   {
1961     // kFullHash1_3 should still return false, because the cached
1962     // result for kPrefix1 doesn't contain kFullHash1_3.
1963     std::vector<SBFullHash> full_hashes(1, kFullHash1_3);
1964     std::vector<SBPrefix> prefix_hits;
1965     std::vector<SBFullHashResult> cache_hits;
1966     EXPECT_FALSE(database_->ContainsBrowseUrlHashesForTesting(
1967         full_hashes, &prefix_hits, &cache_hits));
1968
1969     // kFullHash1_1 is also not in the cached result, which takes
1970     // priority over the database.
1971     prefix_hits.clear();
1972     full_hashes.push_back(kFullHash1_1);
1973     cache_hits.clear();
1974     EXPECT_FALSE(database_->ContainsBrowseUrlHashesForTesting(
1975         full_hashes, &prefix_hits, &cache_hits));
1976
1977     // kFullHash1_2 is in the cached result.
1978     full_hashes.push_back(kFullHash1_2);
1979     prefix_hits.clear();
1980     cache_hits.clear();
1981     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
1982         full_hashes, &prefix_hits, &cache_hits));
1983     EXPECT_TRUE(prefix_hits.empty());
1984     ASSERT_EQ(1U, cache_hits.size());
1985     EXPECT_TRUE(SBFullHashEqual(kFullHash1_2, cache_hits[0].hash));
1986   }
1987
1988   // Remove kFullHash1_1 from the database.
1989   chunks.clear();
1990   chunks.push_back(SubChunkFullHash(11, kFullHash1_1, 1));
1991
1992   ASSERT_TRUE(database_->UpdateStarted(&lists));
1993   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1994   database_->UpdateFinished(true);
1995
1996   // Cache should be cleared after updating.
1997   EXPECT_TRUE(
1998       database_->GetUnsynchronizedPrefixGetHashCacheForTesting()->empty());
1999
2000   {
2001     // Now the database doesn't contain kFullHash1_1.
2002     std::vector<SBFullHash> full_hashes(1, kFullHash1_1);
2003     std::vector<SBPrefix> prefix_hits;
2004     std::vector<SBFullHashResult> cache_hits;
2005     EXPECT_FALSE(database_->ContainsBrowseUrlHashesForTesting(
2006         full_hashes, &prefix_hits, &cache_hits));
2007
2008     // Nor kFullHash1_3.
2009     full_hashes.push_back(kFullHash1_3);
2010     prefix_hits.clear();
2011     cache_hits.clear();
2012     EXPECT_FALSE(database_->ContainsBrowseUrlHashesForTesting(
2013         full_hashes, &prefix_hits, &cache_hits));
2014
2015     // Still has kFullHash1_2.
2016     full_hashes.push_back(kFullHash1_2);
2017     prefix_hits.clear();
2018     cache_hits.clear();
2019     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
2020         full_hashes, &prefix_hits, &cache_hits));
2021     ASSERT_EQ(1U, prefix_hits.size());
2022     EXPECT_EQ(kPrefix1, prefix_hits[0]);
2023     EXPECT_TRUE(cache_hits.empty());
2024   }
2025
2026   // Remove kFullHash1_2 from the database.
2027   chunks.clear();
2028   chunks.push_back(SubChunkFullHash(12, kFullHash1_2, 2));
2029
2030   ASSERT_TRUE(database_->UpdateStarted(&lists));
2031   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
2032   database_->UpdateFinished(true);
2033
2034   // Cache should be cleared after updating.
2035   EXPECT_TRUE(
2036       database_->GetUnsynchronizedPrefixGetHashCacheForTesting()->empty());
2037
2038   {
2039     // None are present.
2040     std::vector<SBFullHash> full_hashes;
2041     std::vector<SBPrefix> prefix_hits;
2042     std::vector<SBFullHashResult> cache_hits;
2043     full_hashes.push_back(kFullHash1_1);
2044     full_hashes.push_back(kFullHash1_2);
2045     full_hashes.push_back(kFullHash1_3);
2046     EXPECT_FALSE(database_->ContainsBrowseUrlHashesForTesting(
2047         full_hashes, &prefix_hits, &cache_hits));
2048   }
2049 }
2050
2051 TEST_F(SafeBrowsingDatabaseTest, BrowseFullHashAndPrefixMatching) {
2052   const SBPrefix kPrefix1 = 1001U;
2053   const SBFullHash kFullHash1_1 =
2054       SBFullHashForPrefixAndSuffix(kPrefix1, "\x01");
2055   const SBFullHash kFullHash1_2 =
2056       SBFullHashForPrefixAndSuffix(kPrefix1, "\x02");
2057
2058   ScopedVector<SBChunkData> chunks;
2059   chunks.push_back(AddChunkFullHash(1, kFullHash1_1));
2060
2061   std::vector<SBListChunkRanges> lists;
2062   ASSERT_TRUE(database_->UpdateStarted(&lists));
2063   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
2064   database_->UpdateFinished(true);
2065
2066   {
2067     // kFullHash1_2 does not match kFullHash1_1.
2068     std::vector<SBFullHash> full_hashes(1, kFullHash1_2);
2069     std::vector<SBPrefix> prefix_hits;
2070     std::vector<SBFullHashResult> cache_hits;
2071     EXPECT_FALSE(database_->ContainsBrowseUrlHashesForTesting(
2072         full_hashes, &prefix_hits, &cache_hits));
2073   }
2074
2075   // Add a prefix match.
2076   chunks.clear();
2077   chunks.push_back(AddChunkPrefix(2, kPrefix1));
2078
2079   ASSERT_TRUE(database_->UpdateStarted(&lists));
2080   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
2081   database_->UpdateFinished(true);
2082
2083   {
2084     // kFullHash1_2 does match kPrefix1.
2085     std::vector<SBFullHash> full_hashes(1, kFullHash1_2);
2086     std::vector<SBPrefix> prefix_hits;
2087     std::vector<SBFullHashResult> cache_hits;
2088     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
2089         full_hashes, &prefix_hits, &cache_hits));
2090     ASSERT_EQ(1U, prefix_hits.size());
2091     EXPECT_EQ(kPrefix1, prefix_hits[0]);
2092     EXPECT_TRUE(cache_hits.empty());
2093   }
2094
2095   // Remove the full hash.
2096   chunks.clear();
2097   chunks.push_back(SubChunkFullHash(11, kFullHash1_1, 1));
2098
2099   ASSERT_TRUE(database_->UpdateStarted(&lists));
2100   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
2101   database_->UpdateFinished(true);
2102
2103   {
2104     // kFullHash1_2 still returns true due to the prefix hit.
2105     std::vector<SBFullHash> full_hashes(1, kFullHash1_2);
2106     std::vector<SBPrefix> prefix_hits;
2107     std::vector<SBFullHashResult> cache_hits;
2108     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
2109         full_hashes, &prefix_hits, &cache_hits));
2110     ASSERT_EQ(1U, prefix_hits.size());
2111     EXPECT_EQ(kPrefix1, prefix_hits[0]);
2112     EXPECT_TRUE(cache_hits.empty());
2113   }
2114 }
2115
2116 TEST_F(SafeBrowsingDatabaseTest, MalwareIpBlacklist) {
2117   std::vector<SBListChunkRanges> lists;
2118   ASSERT_TRUE(database_->UpdateStarted(&lists));
2119
2120   ScopedVector<SBChunkData> chunks;
2121
2122   // IPv4 prefix match for ::ffff:192.168.1.0/120.
2123   chunks.push_back(AddChunkHashedIpValue(1, "::ffff:192.168.1.0", 120));
2124
2125   // IPv4 exact match for ::ffff:192.1.1.1.
2126   chunks.push_back(AddChunkHashedIpValue(2, "::ffff:192.1.1.1", 128));
2127
2128   // IPv6 exact match for: fe80::31a:a0ff:fe10:786e/128.
2129   chunks.push_back(AddChunkHashedIpValue(3, "fe80::31a:a0ff:fe10:786e", 128));
2130
2131   // IPv6 prefix match for: 2620:0:1000:3103::/64.
2132   chunks.push_back(AddChunkHashedIpValue(4, "2620:0:1000:3103::", 64));
2133
2134   // IPv4 prefix match for ::ffff:192.1.122.0/119.
2135   chunks.push_back(AddChunkHashedIpValue(5, "::ffff:192.1.122.0", 119));
2136
2137   // IPv4 prefix match for ::ffff:192.1.128.0/113.
2138   chunks.push_back(AddChunkHashedIpValue(6, "::ffff:192.1.128.0", 113));
2139
2140   database_->InsertChunks(safe_browsing_util::kIPBlacklist, chunks.get());
2141   database_->UpdateFinished(true);
2142
2143   EXPECT_FALSE(database_->ContainsMalwareIP("192.168.0.255"));
2144   EXPECT_TRUE(database_->ContainsMalwareIP("192.168.1.0"));
2145   EXPECT_TRUE(database_->ContainsMalwareIP("192.168.1.255"));
2146   EXPECT_TRUE(database_->ContainsMalwareIP("192.168.1.10"));
2147   EXPECT_TRUE(database_->ContainsMalwareIP("::ffff:192.168.1.2"));
2148   EXPECT_FALSE(database_->ContainsMalwareIP("192.168.2.0"));
2149
2150   EXPECT_FALSE(database_->ContainsMalwareIP("192.1.1.0"));
2151   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.1.1"));
2152   EXPECT_FALSE(database_->ContainsMalwareIP("192.1.1.2"));
2153
2154   EXPECT_FALSE(database_->ContainsMalwareIP(
2155       "2620:0:1000:3102:ffff:ffff:ffff:ffff"));
2156   EXPECT_TRUE(database_->ContainsMalwareIP("2620:0:1000:3103::"));
2157   EXPECT_TRUE(database_->ContainsMalwareIP(
2158       "2620:0:1000:3103:ffff:ffff:ffff:ffff"));
2159   EXPECT_FALSE(database_->ContainsMalwareIP("2620:0:1000:3104::"));
2160
2161   EXPECT_FALSE(database_->ContainsMalwareIP("fe80::21a:a0ff:fe10:786d"));
2162   EXPECT_TRUE(database_->ContainsMalwareIP("fe80::31a:a0ff:fe10:786e"));
2163   EXPECT_FALSE(database_->ContainsMalwareIP("fe80::21a:a0ff:fe10:786f"));
2164
2165   EXPECT_FALSE(database_->ContainsMalwareIP("192.1.121.255"));
2166   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.122.0"));
2167   EXPECT_TRUE(database_->ContainsMalwareIP("::ffff:192.1.122.1"));
2168   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.122.255"));
2169   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.123.0"));
2170   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.123.255"));
2171   EXPECT_FALSE(database_->ContainsMalwareIP("192.1.124.0"));
2172
2173   EXPECT_FALSE(database_->ContainsMalwareIP("192.1.127.255"));
2174   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.128.0"));
2175   EXPECT_TRUE(database_->ContainsMalwareIP("::ffff:192.1.128.1"));
2176   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.128.255"));
2177   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.255.0"));
2178   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.255.255"));
2179   EXPECT_FALSE(database_->ContainsMalwareIP("192.2.0.0"));
2180 }
2181
2182 TEST_F(SafeBrowsingDatabaseTest, ContainsBrowseURL) {
2183   std::vector<SBListChunkRanges> lists;
2184   ASSERT_TRUE(database_->UpdateStarted(&lists));
2185
2186   // Add a host-level hit.
2187   {
2188     ScopedVector<SBChunkData> chunks;
2189     chunks.push_back(AddChunkPrefixValue(1, "www.evil.com/"));
2190     database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
2191   }
2192
2193   // Add a specific fullhash.
2194   static const char kWhateverMalware[] = "www.whatever.com/malware.html";
2195   {
2196     ScopedVector<SBChunkData> chunks;
2197     chunks.push_back(AddChunkFullHashValue(2, kWhateverMalware));
2198     database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
2199   }
2200
2201   // Add a fullhash which has a prefix collision for a known url.
2202   static const char kExampleFine[] = "www.example.com/fine.html";
2203   static const char kExampleCollision[] =
2204       "www.example.com/3123364814/malware.htm";
2205   ASSERT_EQ(SBPrefixForString(kExampleFine),
2206             SBPrefixForString(kExampleCollision));
2207   {
2208     ScopedVector<SBChunkData> chunks;
2209     chunks.push_back(AddChunkFullHashValue(3, kExampleCollision));
2210     database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
2211   }
2212
2213   database_->UpdateFinished(true);
2214
2215   std::vector<SBPrefix> prefix_hits;
2216   std::vector<SBFullHashResult> cache_hits;
2217
2218   // Anything will hit the host prefix.
2219   EXPECT_TRUE(database_->ContainsBrowseUrl(
2220       GURL("http://www.evil.com/malware.html"), &prefix_hits, &cache_hits));
2221   ASSERT_EQ(1U, prefix_hits.size());
2222   EXPECT_EQ(SBPrefixForString("www.evil.com/"), prefix_hits[0]);
2223   EXPECT_TRUE(cache_hits.empty());
2224
2225   // Hit the specific URL prefix.
2226   EXPECT_TRUE(database_->ContainsBrowseUrl(
2227       GURL(std::string("http://") + kWhateverMalware),
2228       &prefix_hits, &cache_hits));
2229   ASSERT_EQ(1U, prefix_hits.size());
2230   EXPECT_EQ(SBPrefixForString(kWhateverMalware), prefix_hits[0]);
2231   EXPECT_TRUE(cache_hits.empty());
2232
2233   // Other URLs at that host are fine.
2234   EXPECT_FALSE(database_->ContainsBrowseUrl(
2235       GURL("http://www.whatever.com/fine.html"), &prefix_hits, &cache_hits));
2236   EXPECT_TRUE(prefix_hits.empty());
2237   EXPECT_TRUE(cache_hits.empty());
2238
2239   // Hit the specific URL full hash.
2240   EXPECT_TRUE(database_->ContainsBrowseUrl(
2241       GURL(std::string("http://") + kExampleCollision),
2242       &prefix_hits, &cache_hits));
2243   ASSERT_EQ(1U, prefix_hits.size());
2244   EXPECT_EQ(SBPrefixForString(kExampleCollision), prefix_hits[0]);
2245   EXPECT_TRUE(cache_hits.empty());
2246
2247   // This prefix collides, but no full hash match.
2248   EXPECT_FALSE(database_->ContainsBrowseUrl(
2249       GURL(std::string("http://") + kExampleFine), &prefix_hits, &cache_hits));
2250 }