?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Knock out injected safe-browsing prefixes.
[chromium-blink-merge.git] / chrome / browser / safe_browsing / safe_browsing_database_unittest.cc
blob559987cc6c2a0ce7a648c934193b7347a56d6f9c
1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 //
5 // Unit tests for the SafeBrowsing storage system.
6
7 #include "base/file_util.h"
8 #include "base/files/scoped_temp_dir.h"
9 #include "base/logging.h"
10 #include "base/message_loop/message_loop.h"
11 #include "base/sha1.h"
12 #include "base/time/time.h"
13 #include "chrome/browser/safe_browsing/safe_browsing_database.h"
14 #include "chrome/browser/safe_browsing/safe_browsing_store_file.h"
15 #include "content/public/test/test_browser_thread_bundle.h"
16 #include "crypto/sha2.h"
17 #include "net/base/net_util.h"
18 #include "sql/connection.h"
19 #include "sql/statement.h"
20 #include "testing/gtest/include/gtest/gtest.h"
21 #include "testing/platform_test.h"
22 #include "url/gurl.h"
23
24 using base::Time;
25
26 namespace {
27
28 SBPrefix SBPrefixForString(const std::string& str) {
29 return SBFullHashForString(str).prefix;
30 }
31
32 std::string HashedIpPrefix(const std::string& ip_prefix, size_t prefix_size) {
33 net::IPAddressNumber ip_number;
34 EXPECT_TRUE(net::ParseIPLiteralToNumber(ip_prefix, &ip_number));
35 EXPECT_EQ(net::kIPv6AddressSize, ip_number.size());
36 const std::string hashed_ip_prefix = base::SHA1HashString(
37 net::IPAddressToPackedString(ip_number));
38 std::string hash(crypto::kSHA256Length, '\0');
39 hash.replace(0, hashed_ip_prefix.size(), hashed_ip_prefix);
40 hash[base::kSHA1Length] = static_cast<char>(prefix_size);
41 return hash;
42 }
43
44 // Add a host-level entry.
45 void InsertAddChunkHostPrefix(SBChunk* chunk,
46 int chunk_number,
47 const std::string& host_name) {
48 chunk->chunk_number = chunk_number;
49 chunk->is_add = true;
50 SBChunkHost host;
51 host.host = SBPrefixForString(host_name);
52 host.entry = SBEntry::Create(SBEntry::ADD_PREFIX, 0);
53 host.entry->set_chunk_id(chunk->chunk_number);
54 chunk->hosts.push_back(host);
55 }
56
57 // Same as InsertAddChunkHostPrefixUrl, but with pre-computed
58 // prefix values.
59 void InsertAddChunkHostPrefixValue(SBChunk* chunk,
60 int chunk_number,
61 const SBPrefix& host_prefix,
62 const SBPrefix& url_prefix) {
63 chunk->chunk_number = chunk_number;
64 chunk->is_add = true;
65 SBChunkHost host;
66 host.host = host_prefix;
67 host.entry = SBEntry::Create(SBEntry::ADD_PREFIX, 1);
68 host.entry->set_chunk_id(chunk->chunk_number);
69 host.entry->SetPrefixAt(0, url_prefix);
70 chunk->hosts.push_back(host);
71 }
72
73 // A helper function that appends one AddChunkHost to chunk with
74 // one url for prefix.
75 void InsertAddChunkHostPrefixUrl(SBChunk* chunk,
76 int chunk_number,
77 const std::string& host_name,
78 const std::string& url) {
79 InsertAddChunkHostPrefixValue(chunk, chunk_number,
80 SBPrefixForString(host_name),
81 SBPrefixForString(url));
82 }
83
84 // Same as InsertAddChunkHostPrefixUrl, but with full hashes.
85 void InsertAddChunkHostFullHashes(SBChunk* chunk,
86 int chunk_number,
87 const std::string& host_name,
88 const std::string& url) {
89 chunk->chunk_number = chunk_number;
90 chunk->is_add = true;
91 SBChunkHost host;
92 host.host = SBPrefixForString(host_name);
93 host.entry = SBEntry::Create(SBEntry::ADD_FULL_HASH, 1);
94 host.entry->set_chunk_id(chunk->chunk_number);
95 host.entry->SetFullHashAt(0, SBFullHashForString(url));
96 chunk->hosts.push_back(host);
97 }
98
99 // TODO(shess): This sounds like something to insert a full-hash chunk, but it's
100 // actually specific to IP blacklist.
101 void InsertAddChunkFullHash(SBChunk* chunk,
102 int chunk_number,
103 const std::string& ip_str,
104 size_t prefix_size) {
105 const std::string full_hash_str = HashedIpPrefix(ip_str, prefix_size);
106 EXPECT_EQ(sizeof(SBFullHash), full_hash_str.size());
107 SBFullHash full_hash;
108 std::memcpy(&(full_hash.full_hash), full_hash_str.data(), sizeof(SBFullHash));
109
110 chunk->chunk_number = chunk_number;
111 chunk->is_add = true;
112 SBChunkHost host;
113 host.host = full_hash.prefix;
114 host.entry = SBEntry::Create(SBEntry::ADD_FULL_HASH, 1);
115 host.entry->set_chunk_id(chunk->chunk_number);
116 host.entry->SetFullHashAt(0, full_hash);
117 chunk->hosts.push_back(host);
118 }
119
120 // Same as InsertAddChunkHostPrefixUrl, but with two urls for prefixes.
121 void InsertAddChunkHost2PrefixUrls(SBChunk* chunk,
122 int chunk_number,
123 const std::string& host_name,
124 const std::string& url1,
125 const std::string& url2) {
126 chunk->chunk_number = chunk_number;
127 chunk->is_add = true;
128 SBChunkHost host;
129 host.host = SBPrefixForString(host_name);
130 host.entry = SBEntry::Create(SBEntry::ADD_PREFIX, 2);
131 host.entry->set_chunk_id(chunk->chunk_number);
132 host.entry->SetPrefixAt(0, SBPrefixForString(url1));
133 host.entry->SetPrefixAt(1, SBPrefixForString(url2));
134 chunk->hosts.push_back(host);
135 }
136
137 // Same as InsertAddChunkHost2PrefixUrls, but with full hashes.
138 void InsertAddChunkHost2FullHashes(SBChunk* chunk,
139 int chunk_number,
140 const std::string& host_name,
141 const std::string& url1,
142 const std::string& url2) {
143 chunk->chunk_number = chunk_number;
144 chunk->is_add = true;
145 SBChunkHost host;
146 host.host = SBPrefixForString(host_name);
147 host.entry = SBEntry::Create(SBEntry::ADD_FULL_HASH, 2);
148 host.entry->set_chunk_id(chunk->chunk_number);
149 host.entry->SetFullHashAt(0, SBFullHashForString(url1));
150 host.entry->SetFullHashAt(1, SBFullHashForString(url2));
151 chunk->hosts.push_back(host);
152 }
153
154 // Same as InsertSubChunkHostPrefixUrl, but with pre-computed
155 // prefix values.
156 void InsertSubChunkHostPrefixValue(SBChunk* chunk,
157 int chunk_number,
158 int chunk_id_to_sub,
159 const SBPrefix& host_prefix,
160 const SBPrefix& url_prefix) {
161 chunk->chunk_number = chunk_number;
162 chunk->is_add = false;
163 SBChunkHost host;
164 host.host = host_prefix;
165 host.entry = SBEntry::Create(SBEntry::SUB_PREFIX, 1);
166 host.entry->set_chunk_id(chunk->chunk_number);
167 host.entry->SetChunkIdAtPrefix(0, chunk_id_to_sub);
168 host.entry->SetPrefixAt(0, url_prefix);
169 chunk->hosts.push_back(host);
170 }
171
172 // A helper function that adds one SubChunkHost to chunk with
173 // one url for prefix.
174 void InsertSubChunkHostPrefixUrl(SBChunk* chunk,
175 int chunk_number,
176 int chunk_id_to_sub,
177 const std::string& host_name,
178 const std::string& url) {
179 InsertSubChunkHostPrefixValue(chunk, chunk_number,
180 chunk_id_to_sub,
181 SBPrefixForString(host_name),
182 SBPrefixForString(url));
183 }
184
185 // Same as InsertSubChunkHostPrefixUrl, but with two urls for prefixes.
186 void InsertSubChunkHost2PrefixUrls(SBChunk* chunk,
187 int chunk_number,
188 int chunk_id_to_sub,
189 const std::string& host_name,
190 const std::string& url1,
191 const std::string& url2) {
192 chunk->chunk_number = chunk_number;
193 chunk->is_add = false;
194 SBChunkHost host;
195 host.host = SBPrefixForString(host_name);
196 host.entry = SBEntry::Create(SBEntry::SUB_PREFIX, 2);
197 host.entry->set_chunk_id(chunk->chunk_number);
198 host.entry->SetPrefixAt(0, SBPrefixForString(url1));
199 host.entry->SetChunkIdAtPrefix(0, chunk_id_to_sub);
200 host.entry->SetPrefixAt(1, SBPrefixForString(url2));
201 host.entry->SetChunkIdAtPrefix(1, chunk_id_to_sub);
202 chunk->hosts.push_back(host);
203 }
204
205 // Same as InsertSubChunkHost2PrefixUrls, but with full hashes.
206 void InsertSubChunkHostFullHash(SBChunk* chunk,
207 int chunk_number,
208 int chunk_id_to_sub,
209 const std::string& host_name,
210 const std::string& url) {
211 chunk->chunk_number = chunk_number;
212 chunk->is_add = false;
213 SBChunkHost host;
214 host.host = SBPrefixForString(host_name);
215 host.entry = SBEntry::Create(SBEntry::SUB_FULL_HASH, 2);
216 host.entry->set_chunk_id(chunk->chunk_number);
217 host.entry->SetFullHashAt(0, SBFullHashForString(url));
218 host.entry->SetChunkIdAtPrefix(0, chunk_id_to_sub);
219 chunk->hosts.push_back(host);
220 }
221
222 // Prevent DCHECK from killing tests.
223 // TODO(shess): Pawel disputes the use of this, so the test which uses
224 // it is DISABLED. http://crbug.com/56448
225 class ScopedLogMessageIgnorer {
226 public:
227 ScopedLogMessageIgnorer() {
228 logging::SetLogMessageHandler(&LogMessageIgnorer);
229 }
230 ~ScopedLogMessageIgnorer() {
231 // TODO(shess): Would be better to verify whether anyone else
232 // changed it, and then restore it to the previous value.
233 logging::SetLogMessageHandler(NULL);
234 }
235
236 private:
237 static bool LogMessageIgnorer(int severity, const char* file, int line,
238 size_t message_start, const std::string& str) {
239 // Intercept FATAL, strip the stack backtrace, and log it without
240 // the crash part.
241 if (severity == logging::LOG_FATAL) {
242 size_t newline = str.find('\n');
243 if (newline != std::string::npos) {
244 const std::string msg = str.substr(0, newline + 1);
245 fprintf(stderr, "%s", msg.c_str());
246 fflush(stderr);
247 }
248 return true;
249 }
250
251 return false;
252 }
253 };
254
255 } // namespace
256
257 class SafeBrowsingDatabaseTest : public PlatformTest {
258 public:
259 virtual void SetUp() {
260 PlatformTest::SetUp();
261
262 // Setup a database in a temporary directory.
263 ASSERT_TRUE(temp_dir_.CreateUniqueTempDir());
264 database_.reset(new SafeBrowsingDatabaseNew);
265 database_filename_ =
266 temp_dir_.path().AppendASCII("SafeBrowsingTestDatabase");
267 database_->Init(database_filename_);
268 }
269
270 virtual void TearDown() {
271 database_.reset();
272
273 PlatformTest::TearDown();
274 }
275
276 void GetListsInfo(std::vector<SBListChunkRanges>* lists) {
277 lists->clear();
278 EXPECT_TRUE(database_->UpdateStarted(lists));
279 database_->UpdateFinished(true);
280 }
281
282 // Helper function to do an AddDel or SubDel command.
283 void DelChunk(const std::string& list,
284 int chunk_id,
285 bool is_sub_del) {
286 std::vector<SBChunkDelete> deletes;
287 SBChunkDelete chunk_delete;
288 chunk_delete.list_name = list;
289 chunk_delete.is_sub_del = is_sub_del;
290 chunk_delete.chunk_del.push_back(ChunkRange(chunk_id));
291 deletes.push_back(chunk_delete);
292 database_->DeleteChunks(deletes);
293 }
294
295 void AddDelChunk(const std::string& list, int chunk_id) {
296 DelChunk(list, chunk_id, false);
297 }
298
299 void SubDelChunk(const std::string& list, int chunk_id) {
300 DelChunk(list, chunk_id, true);
301 }
302
303 // Utility function for setting up the database for the caching test.
304 void PopulateDatabaseForCacheTest();
305
306 scoped_ptr<SafeBrowsingDatabaseNew> database_;
307 base::FilePath database_filename_;
308 base::ScopedTempDir temp_dir_;
309 };
310
311 // Tests retrieving list name information.
312 TEST_F(SafeBrowsingDatabaseTest, ListNameForBrowse) {
313 SBChunkList chunks;
314 SBChunk chunk;
315
316 InsertAddChunkHostPrefixUrl(&chunk, 1, "www.evil.com/",
317 "www.evil.com/malware.html");
318 chunks.clear();
319 chunks.push_back(chunk);
320 std::vector<SBListChunkRanges> lists;
321 EXPECT_TRUE(database_->UpdateStarted(&lists));
322 database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
323
324 chunk.hosts.clear();
325 InsertAddChunkHostPrefixUrl(&chunk, 2, "www.foo.com/",
326 "www.foo.com/malware.html");
327 chunks.clear();
328 chunks.push_back(chunk);
329 database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
330
331 chunk.hosts.clear();
332 InsertAddChunkHostPrefixUrl(&chunk, 3, "www.whatever.com/",
333 "www.whatever.com/malware.html");
334 chunks.clear();
335 chunks.push_back(chunk);
336 database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
337 database_->UpdateFinished(true);
338
339 GetListsInfo(&lists);
340 EXPECT_TRUE(lists[0].name == safe_browsing_util::kMalwareList);
341 EXPECT_EQ(lists[0].adds, "1-3");
342 EXPECT_TRUE(lists[0].subs.empty());
343
344 // Insert a malware sub chunk.
345 chunk.hosts.clear();
346 InsertSubChunkHostPrefixUrl(&chunk, 7, 19, "www.subbed.com/",
347 "www.subbed.com/noteveil1.html");
348 chunks.clear();
349 chunks.push_back(chunk);
350
351 EXPECT_TRUE(database_->UpdateStarted(&lists));
352 database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
353 database_->UpdateFinished(true);
354
355 GetListsInfo(&lists);
356 EXPECT_TRUE(lists[0].name == safe_browsing_util::kMalwareList);
357 EXPECT_EQ(lists[0].adds, "1-3");
358 EXPECT_EQ(lists[0].subs, "7");
359 if (lists.size() == 2) {
360 // Old style database won't have the second entry since it creates the lists
361 // when it receives an update containing that list. The filter-based
362 // database has these values hard coded.
363 EXPECT_TRUE(lists[1].name == safe_browsing_util::kPhishingList);
364 EXPECT_TRUE(lists[1].adds.empty());
365 EXPECT_TRUE(lists[1].subs.empty());
366 }
367
368 // Add a phishing add chunk.
369 chunk.hosts.clear();
370 InsertAddChunkHostPrefixUrl(&chunk, 47, "www.evil.com/",
371 "www.evil.com/phishing.html");
372 chunks.clear();
373 chunks.push_back(chunk);
374 EXPECT_TRUE(database_->UpdateStarted(&lists));
375 database_->InsertChunks(safe_browsing_util::kPhishingList, chunks);
376
377 // Insert some phishing sub chunks.
378 chunk.hosts.clear();
379 InsertSubChunkHostPrefixUrl(&chunk, 200, 1999, "www.phishy.com/",
380 "www.phishy.com/notevil1.html");
381 chunks.clear();
382 chunks.push_back(chunk);
383 database_->InsertChunks(safe_browsing_util::kPhishingList, chunks);
384
385 chunk.hosts.clear();
386 InsertSubChunkHostPrefixUrl(&chunk, 201, 1999, "www.phishy2.com/",
387 "www.phishy2.com/notevil1.html");
388 chunks.clear();
389 chunks.push_back(chunk);
390 database_->InsertChunks(safe_browsing_util::kPhishingList, chunks);
391 database_->UpdateFinished(true);
392
393 GetListsInfo(&lists);
394 EXPECT_TRUE(lists[0].name == safe_browsing_util::kMalwareList);
395 EXPECT_EQ(lists[0].adds, "1-3");
396 EXPECT_EQ(lists[0].subs, "7");
397 EXPECT_TRUE(lists[1].name == safe_browsing_util::kPhishingList);
398 EXPECT_EQ(lists[1].adds, "47");
399 EXPECT_EQ(lists[1].subs, "200-201");
400 }
401
402 TEST_F(SafeBrowsingDatabaseTest, ListNameForBrowseAndDownload) {
403 database_.reset();
404 base::MessageLoop loop;
405 SafeBrowsingStoreFile* browse_store = new SafeBrowsingStoreFile();
406 SafeBrowsingStoreFile* download_store = new SafeBrowsingStoreFile();
407 SafeBrowsingStoreFile* csd_whitelist_store = new SafeBrowsingStoreFile();
408 SafeBrowsingStoreFile* download_whitelist_store = new SafeBrowsingStoreFile();
409 SafeBrowsingStoreFile* extension_blacklist_store =
410 new SafeBrowsingStoreFile();
411 SafeBrowsingStoreFile* ip_blacklist_store = new SafeBrowsingStoreFile();
412 database_.reset(new SafeBrowsingDatabaseNew(browse_store,
413 download_store,
414 csd_whitelist_store,
415 download_whitelist_store,
416 extension_blacklist_store,
417 NULL,
418 ip_blacklist_store));
419 database_->Init(database_filename_);
420
421 SBChunkList chunks;
422 SBChunk chunk;
423
424 // Insert malware, phish, binurl and bindownload add chunks.
425 InsertAddChunkHostPrefixUrl(&chunk, 1, "www.evil.com/",
426 "www.evil.com/malware.html");
427 chunks.push_back(chunk);
428 std::vector<SBListChunkRanges> lists;
429 EXPECT_TRUE(database_->UpdateStarted(&lists));
430 database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
431
432 chunk.hosts.clear();
433 InsertAddChunkHostPrefixUrl(&chunk, 2, "www.foo.com/",
434 "www.foo.com/malware.html");
435 chunks.clear();
436 chunks.push_back(chunk);
437 database_->InsertChunks(safe_browsing_util::kPhishingList, chunks);
438
439 chunk.hosts.clear();
440 InsertAddChunkHostPrefixUrl(&chunk, 3, "www.whatever.com/",
441 "www.whatever.com/download.html");
442 chunks.clear();
443 chunks.push_back(chunk);
444 database_->InsertChunks(safe_browsing_util::kBinUrlList, chunks);
445
446 chunk.hosts.clear();
447 InsertAddChunkHostFullHashes(&chunk, 5, "www.forwhitelist.com/",
448 "www.forwhitelist.com/a.html");
449 chunks.clear();
450 chunks.push_back(chunk);
451 database_->InsertChunks(safe_browsing_util::kCsdWhiteList, chunks);
452
453 chunk.hosts.clear();
454 InsertAddChunkHostFullHashes(&chunk, 6, "www.download.com/",
455 "www.download.com/");
456
457 chunks.clear();
458 chunks.push_back(chunk);
459 database_->InsertChunks(safe_browsing_util::kDownloadWhiteList, chunks);
460
461 chunk.hosts.clear();
462 InsertAddChunkHostFullHashes(&chunk, 8,
463 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
464 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
465
466 chunks.clear();
467 chunks.push_back(chunk);
468 database_->InsertChunks(safe_browsing_util::kExtensionBlacklist, chunks);
469
470 chunk.hosts.clear();
471 InsertAddChunkFullHash(&chunk, 9, "::ffff:192.168.1.0", 120);
472
473 chunks.clear();
474 chunks.push_back(chunk);
475 database_->InsertChunks(safe_browsing_util::kIPBlacklist, chunks);
476
477 database_->UpdateFinished(true);
478
479 GetListsInfo(&lists);
480 ASSERT_EQ(7U, lists.size());
481 EXPECT_TRUE(lists[0].name == safe_browsing_util::kMalwareList);
482 EXPECT_EQ(lists[0].adds, "1");
483 EXPECT_TRUE(lists[0].subs.empty());
484 EXPECT_TRUE(lists[1].name == safe_browsing_util::kPhishingList);
485 EXPECT_EQ(lists[1].adds, "2");
486 EXPECT_TRUE(lists[1].subs.empty());
487 EXPECT_TRUE(lists[2].name == safe_browsing_util::kBinUrlList);
488 EXPECT_EQ(lists[2].adds, "3");
489 EXPECT_TRUE(lists[2].subs.empty());
490 EXPECT_TRUE(lists[3].name == safe_browsing_util::kCsdWhiteList);
491 EXPECT_EQ(lists[3].adds, "5");
492 EXPECT_TRUE(lists[3].subs.empty());
493 EXPECT_TRUE(lists[4].name == safe_browsing_util::kDownloadWhiteList);
494 EXPECT_EQ(lists[4].adds, "6");
495 EXPECT_TRUE(lists[4].subs.empty());
496 EXPECT_TRUE(lists[5].name == safe_browsing_util::kExtensionBlacklist);
497 EXPECT_EQ(lists[5].adds, "8");
498 EXPECT_TRUE(lists[5].subs.empty());
499 EXPECT_TRUE(lists[6].name == safe_browsing_util::kIPBlacklist);
500 EXPECT_EQ(lists[6].adds, "9");
501 EXPECT_TRUE(lists[6].subs.empty());
502
503 database_.reset();
504 }
505
506 // Checks database reading and writing for browse.
507 TEST_F(SafeBrowsingDatabaseTest, BrowseDatabase) {
508 SBChunkList chunks;
509 SBChunk chunk;
510
511 // Add a simple chunk with one hostkey.
512 InsertAddChunkHost2PrefixUrls(&chunk, 1, "www.evil.com/",
513 "www.evil.com/phishing.html",
514 "www.evil.com/malware.html");
515 chunks.push_back(chunk);
516 std::vector<SBListChunkRanges> lists;
517 EXPECT_TRUE(database_->UpdateStarted(&lists));
518 database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
519
520 chunk.hosts.clear();
521 InsertAddChunkHost2PrefixUrls(&chunk, 2, "www.evil.com/",
522 "www.evil.com/notevil1.html",
523 "www.evil.com/notevil2.html");
524 InsertAddChunkHost2PrefixUrls(&chunk, 2, "www.good.com/",
525 "www.good.com/good1.html",
526 "www.good.com/good2.html");
527 chunks.clear();
528 chunks.push_back(chunk);
529 database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
530
531 // and a chunk with an IP-based host
532 chunk.hosts.clear();
533 InsertAddChunkHostPrefixUrl(&chunk, 3, "192.168.0.1/",
534 "192.168.0.1/malware.html");
535 chunks.clear();
536 chunks.push_back(chunk);
537 database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
538
539 // A chunk with a full hash.
540 chunk.hosts.clear();
541 InsertAddChunkHostFullHashes(&chunk, 7, "www.evil.com/",
542 "www.evil.com/evil.html");
543 chunks.clear();
544 chunks.push_back(chunk);
545 database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
546
547 database_->UpdateFinished(true);
548
549 // Make sure they were added correctly.
550 GetListsInfo(&lists);
551 EXPECT_TRUE(lists[0].name == safe_browsing_util::kMalwareList);
552 EXPECT_EQ(lists[0].adds, "1-3,7");
553 EXPECT_TRUE(lists[0].subs.empty());
554
555 const Time now = Time::Now();
556 std::vector<SBFullHashResult> cached_hashes;
557 std::vector<SBPrefix> prefix_hits;
558 EXPECT_TRUE(database_->ContainsBrowseUrl(
559 GURL("http://www.evil.com/phishing.html"),
560 &prefix_hits, &cached_hashes, now));
561 EXPECT_EQ(prefix_hits[0], SBPrefixForString("www.evil.com/phishing.html"));
562 EXPECT_EQ(prefix_hits.size(), 1U);
563
564 EXPECT_TRUE(database_->ContainsBrowseUrl(
565 GURL("http://www.evil.com/malware.html"),
566 &prefix_hits, &cached_hashes, now));
567
568 EXPECT_TRUE(database_->ContainsBrowseUrl(
569 GURL("http://www.evil.com/notevil1.html"),
570 &prefix_hits, &cached_hashes, now));
571
572 EXPECT_TRUE(database_->ContainsBrowseUrl(
573 GURL("http://www.evil.com/notevil2.html"),
574 &prefix_hits, &cached_hashes, now));
575
576 EXPECT_TRUE(database_->ContainsBrowseUrl(
577 GURL("http://www.good.com/good1.html"),
578 &prefix_hits, &cached_hashes, now));
579
580 EXPECT_TRUE(database_->ContainsBrowseUrl(
581 GURL("http://www.good.com/good2.html"),
582 &prefix_hits, &cached_hashes, now));
583
584 EXPECT_TRUE(database_->ContainsBrowseUrl(
585 GURL("http://192.168.0.1/malware.html"),
586 &prefix_hits, &cached_hashes, now));
587
588 EXPECT_FALSE(database_->ContainsBrowseUrl(
589 GURL("http://www.evil.com/"),
590 &prefix_hits, &cached_hashes, now));
591 EXPECT_TRUE(prefix_hits.empty());
592
593 EXPECT_FALSE(database_->ContainsBrowseUrl(
594 GURL("http://www.evil.com/robots.txt"),
595 &prefix_hits, &cached_hashes, now));
596
597 EXPECT_TRUE(database_->ContainsBrowseUrl(
598 GURL("http://www.evil.com/evil.html"),
599 &prefix_hits, &cached_hashes, now));
600 ASSERT_EQ(1U, prefix_hits.size());
601 EXPECT_EQ(prefix_hits[0], SBPrefixForString("www.evil.com/evil.html"));
602
603 // Attempt to re-add the first chunk (should be a no-op).
604 // see bug: http://code.google.com/p/chromium/issues/detail?id=4522
605 chunk.hosts.clear();
606 InsertAddChunkHost2PrefixUrls(&chunk, 1, "www.evil.com/",
607 "www.evil.com/phishing.html",
608 "www.evil.com/malware.html");
609 chunks.clear();
610 chunks.push_back(chunk);
611 EXPECT_TRUE(database_->UpdateStarted(&lists));
612 database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
613 database_->UpdateFinished(true);
614
615 GetListsInfo(&lists);
616 EXPECT_TRUE(lists[0].name == safe_browsing_util::kMalwareList);
617 EXPECT_EQ(lists[0].adds, "1-3,7");
618 EXPECT_TRUE(lists[0].subs.empty());
619
620 // Test removing a single prefix from the add chunk.
621 chunk.hosts.clear();
622 InsertSubChunkHostPrefixUrl(&chunk, 4, 2, "www.evil.com/",
623 "www.evil.com/notevil1.html");
624 chunks.clear();
625 chunks.push_back(chunk);
626 EXPECT_TRUE(database_->UpdateStarted(&lists));
627 database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
628
629 database_->UpdateFinished(true);
630
631 EXPECT_TRUE(database_->ContainsBrowseUrl(
632 GURL("http://www.evil.com/phishing.html"),
633 &prefix_hits, &cached_hashes, now));
634 EXPECT_EQ(prefix_hits[0], SBPrefixForString("www.evil.com/phishing.html"));
635 EXPECT_EQ(prefix_hits.size(), 1U);
636
637 EXPECT_FALSE(database_->ContainsBrowseUrl(
638 GURL("http://www.evil.com/notevil1.html"),
639 &prefix_hits, &cached_hashes, now));
640 EXPECT_TRUE(prefix_hits.empty());
641
642 EXPECT_TRUE(database_->ContainsBrowseUrl(
643 GURL("http://www.evil.com/notevil2.html"),
644 &prefix_hits, &cached_hashes, now));
645
646 EXPECT_TRUE(database_->ContainsBrowseUrl(
647 GURL("http://www.good.com/good1.html"),
648 &prefix_hits, &cached_hashes, now));
649
650 EXPECT_TRUE(database_->ContainsBrowseUrl(
651 GURL("http://www.good.com/good2.html"),
652 &prefix_hits, &cached_hashes, now));
653
654 GetListsInfo(&lists);
655 EXPECT_TRUE(lists[0].name == safe_browsing_util::kMalwareList);
656 EXPECT_EQ(lists[0].subs, "4");
657
658 // Test the same sub chunk again. This should be a no-op.
659 // see bug: http://code.google.com/p/chromium/issues/detail?id=4522
660 chunk.hosts.clear();
661 InsertSubChunkHostPrefixUrl(&chunk, 4, 2, "www.evil.com/",
662 "www.evil.com/notevil1.html");
663 chunks.clear();
664 chunks.push_back(chunk);
665
666 EXPECT_TRUE(database_->UpdateStarted(&lists));
667 database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
668 database_->UpdateFinished(true);
669
670 GetListsInfo(&lists);
671 EXPECT_TRUE(lists[0].name == safe_browsing_util::kMalwareList);
672 EXPECT_EQ(lists[0].subs, "4");
673
674 // Test removing all the prefixes from an add chunk.
675 EXPECT_TRUE(database_->UpdateStarted(&lists));
676 AddDelChunk(safe_browsing_util::kMalwareList, 2);
677 database_->UpdateFinished(true);
678
679 EXPECT_FALSE(database_->ContainsBrowseUrl(
680 GURL("http://www.evil.com/notevil2.html"),
681 &prefix_hits, &cached_hashes, now));
682
683 EXPECT_FALSE(database_->ContainsBrowseUrl(
684 GURL("http://www.good.com/good1.html"),
685 &prefix_hits, &cached_hashes, now));
686
687 EXPECT_FALSE(database_->ContainsBrowseUrl(
688 GURL("http://www.good.com/good2.html"),
689 &prefix_hits, &cached_hashes, now));
690
691 GetListsInfo(&lists);
692 EXPECT_TRUE(lists[0].name == safe_browsing_util::kMalwareList);
693 EXPECT_EQ(lists[0].adds, "1,3,7");
694 EXPECT_EQ(lists[0].subs, "4");
695
696 // The adddel command exposed a bug in the transaction code where any
697 // transaction after it would fail. Add a dummy entry and remove it to
698 // make sure the transcation works fine.
699 chunk.hosts.clear();
700 InsertAddChunkHostPrefixUrl(&chunk, 44, "www.redherring.com/",
701 "www.redherring.com/index.html");
702 chunks.clear();
703 chunks.push_back(chunk);
704 EXPECT_TRUE(database_->UpdateStarted(&lists));
705 database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
706
707 // Now remove the dummy entry. If there are any problems with the
708 // transactions, asserts will fire.
709 AddDelChunk(safe_browsing_util::kMalwareList, 44);
710
711 // Test the subdel command.
712 SubDelChunk(safe_browsing_util::kMalwareList, 4);
713 database_->UpdateFinished(true);
714
715 GetListsInfo(&lists);
716 EXPECT_TRUE(lists[0].name == safe_browsing_util::kMalwareList);
717 EXPECT_EQ(lists[0].adds, "1,3,7");
718 EXPECT_EQ(lists[0].subs, "");
719
720 // Test a sub command coming in before the add.
721 chunk.hosts.clear();
722 InsertSubChunkHost2PrefixUrls(&chunk, 5, 10,
723 "www.notevilanymore.com/",
724 "www.notevilanymore.com/index.html",
725 "www.notevilanymore.com/good.html");
726 chunks.clear();
727 chunks.push_back(chunk);
728 EXPECT_TRUE(database_->UpdateStarted(&lists));
729 database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
730 database_->UpdateFinished(true);
731
732 EXPECT_FALSE(database_->ContainsBrowseUrl(
733 GURL("http://www.notevilanymore.com/index.html"),
734 &prefix_hits, &cached_hashes, now));
735
736 // Now insert the tardy add chunk and we don't expect them to appear
737 // in database because of the previous sub chunk.
738 chunk.hosts.clear();
739 InsertAddChunkHost2PrefixUrls(&chunk, 10, "www.notevilanymore.com/",
740 "www.notevilanymore.com/index.html",
741 "www.notevilanymore.com/good.html");
742 chunks.clear();
743 chunks.push_back(chunk);
744 EXPECT_TRUE(database_->UpdateStarted(&lists));
745 database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
746 database_->UpdateFinished(true);
747
748 EXPECT_FALSE(database_->ContainsBrowseUrl(
749 GURL("http://www.notevilanymore.com/index.html"),
750 &prefix_hits, &cached_hashes, now));
751
752 EXPECT_FALSE(database_->ContainsBrowseUrl(
753 GURL("http://www.notevilanymore.com/good.html"),
754 &prefix_hits, &cached_hashes, now));
755
756 // Reset and reload the database. The database will rely on the prefix set.
757 database_.reset(new SafeBrowsingDatabaseNew);
758 database_->Init(database_filename_);
759
760 // Check that a prefix still hits.
761 EXPECT_TRUE(database_->ContainsBrowseUrl(
762 GURL("http://www.evil.com/phishing.html"),
763 &prefix_hits, &cached_hashes, now));
764 EXPECT_EQ(prefix_hits[0], SBPrefixForString("www.evil.com/phishing.html"));
765 EXPECT_EQ(prefix_hits.size(), 1U);
766
767 // Also check that it's not just always returning true in this case.
768 EXPECT_FALSE(database_->ContainsBrowseUrl(
769 GURL("http://www.evil.com/"),
770 &prefix_hits, &cached_hashes, now));
771
772 // Check that the full hash is still present.
773 EXPECT_TRUE(database_->ContainsBrowseUrl(
774 GURL("http://www.evil.com/evil.html"),
775 &prefix_hits, &cached_hashes, now));
776 ASSERT_EQ(1U, prefix_hits.size());
777 EXPECT_EQ(prefix_hits[0], SBPrefixForString("www.evil.com/evil.html"));
778 }
779
780
781 // Test adding zero length chunks to the database.
782 TEST_F(SafeBrowsingDatabaseTest, ZeroSizeChunk) {
783 SBChunkList chunks;
784 SBChunk chunk;
785
786 // Populate with a couple of normal chunks.
787 InsertAddChunkHost2PrefixUrls(&chunk, 1, "www.test.com/",
788 "www.test.com/test1.html",
789 "www.test.com/test2.html");
790 chunks.clear();
791 chunks.push_back(chunk);
792
793 chunk.hosts.clear();
794 InsertAddChunkHost2PrefixUrls(&chunk, 10, "www.random.com/",
795 "www.random.com/random1.html",
796 "www.random.com/random2.html");
797 chunks.push_back(chunk);
798
799 std::vector<SBListChunkRanges> lists;
800 EXPECT_TRUE(database_->UpdateStarted(&lists));
801 database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
802 database_->UpdateFinished(true);
803
804 // Add an empty ADD and SUB chunk.
805 GetListsInfo(&lists);
806 EXPECT_EQ(lists[0].adds, "1,10");
807
808 SBChunk empty_chunk;
809 empty_chunk.chunk_number = 19;
810 empty_chunk.is_add = true;
811 chunks.clear();
812 chunks.push_back(empty_chunk);
813 EXPECT_TRUE(database_->UpdateStarted(&lists));
814 database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
815 chunks.clear();
816 empty_chunk.chunk_number = 7;
817 empty_chunk.is_add = false;
818 chunks.push_back(empty_chunk);
819 database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
820 database_->UpdateFinished(true);
821
822 GetListsInfo(&lists);
823 EXPECT_EQ(lists[0].adds, "1,10,19");
824 EXPECT_EQ(lists[0].subs, "7");
825
826 // Add an empty chunk along with a couple that contain data. This should
827 // result in the chunk range being reduced in size.
828 empty_chunk.hosts.clear();
829 InsertAddChunkHostPrefixUrl(&empty_chunk, 20, "www.notempy.com/",
830 "www.notempty.com/full1.html");
831 chunks.clear();
832 chunks.push_back(empty_chunk);
833
834 empty_chunk.chunk_number = 21;
835 empty_chunk.is_add = true;
836 empty_chunk.hosts.clear();
837 chunks.push_back(empty_chunk);
838
839 empty_chunk.hosts.clear();
840 InsertAddChunkHostPrefixUrl(&empty_chunk, 22, "www.notempy.com/",
841 "www.notempty.com/full2.html");
842 chunks.push_back(empty_chunk);
843
844 EXPECT_TRUE(database_->UpdateStarted(&lists));
845 database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
846 database_->UpdateFinished(true);
847
848 const Time now = Time::Now();
849 std::vector<SBFullHashResult> cached_hashes;
850 std::vector<SBPrefix> prefix_hits;
851 EXPECT_TRUE(database_->ContainsBrowseUrl(
852 GURL("http://www.notempty.com/full1.html"),
853 &prefix_hits, &cached_hashes, now));
854 EXPECT_TRUE(database_->ContainsBrowseUrl(
855 GURL("http://www.notempty.com/full2.html"),
856 &prefix_hits, &cached_hashes, now));
857
858 GetListsInfo(&lists);
859 EXPECT_EQ(lists[0].adds, "1,10,19-22");
860 EXPECT_EQ(lists[0].subs, "7");
861
862 // Handle AddDel and SubDel commands for empty chunks.
863 EXPECT_TRUE(database_->UpdateStarted(&lists));
864 AddDelChunk(safe_browsing_util::kMalwareList, 21);
865 database_->UpdateFinished(true);
866
867 GetListsInfo(&lists);
868 EXPECT_EQ(lists[0].adds, "1,10,19-20,22");
869 EXPECT_EQ(lists[0].subs, "7");
870
871 EXPECT_TRUE(database_->UpdateStarted(&lists));
872 SubDelChunk(safe_browsing_util::kMalwareList, 7);
873 database_->UpdateFinished(true);
874
875 GetListsInfo(&lists);
876 EXPECT_EQ(lists[0].adds, "1,10,19-20,22");
877 EXPECT_EQ(lists[0].subs, "");
878 }
879
880 // Utility function for setting up the database for the caching test.
881 void SafeBrowsingDatabaseTest::PopulateDatabaseForCacheTest() {
882 SBChunkList chunks;
883 SBChunk chunk;
884 // Add a simple chunk with one hostkey and cache it.
885 InsertAddChunkHost2PrefixUrls(&chunk, 1, "www.evil.com/",
886 "www.evil.com/phishing.html",
887 "www.evil.com/malware.html");
888 chunks.push_back(chunk);
889
890 std::vector<SBListChunkRanges> lists;
891 EXPECT_TRUE(database_->UpdateStarted(&lists));
892 database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
893 database_->UpdateFinished(true);
894
895 // Add the GetHash results to the cache.
896 SBFullHashResult full_hash;
897 full_hash.hash = SBFullHashForString("www.evil.com/phishing.html");
898 full_hash.list_id = safe_browsing_util::MALWARE;
899
900 std::vector<SBFullHashResult> results;
901 results.push_back(full_hash);
902
903 full_hash.hash = SBFullHashForString("www.evil.com/malware.html");
904 results.push_back(full_hash);
905
906 std::vector<SBPrefix> prefixes;
907 database_->CacheHashResults(prefixes, results);
908 }
909
910 TEST_F(SafeBrowsingDatabaseTest, HashCaching) {
911 PopulateDatabaseForCacheTest();
912
913 // We should have both full hashes in the cache.
914 EXPECT_EQ(2U, database_->cached_browse_hashes_.size());
915
916 // Test the cache lookup for the first prefix.
917 std::vector<SBPrefix> prefixes;
918 std::vector<SBFullHashResult> cached_hashes;
919 database_->ContainsBrowseUrl(
920 GURL("http://www.evil.com/phishing.html"),
921 &prefixes, &cached_hashes, Time::Now());
922 ASSERT_EQ(1U, cached_hashes.size());
923 EXPECT_TRUE(
924 SBFullHashEqual(cached_hashes[0].hash,
925 SBFullHashForString("www.evil.com/phishing.html")));
926
927 prefixes.clear();
928 cached_hashes.clear();
929
930 // Test the cache lookup for the second prefix.
931 database_->ContainsBrowseUrl(
932 GURL("http://www.evil.com/malware.html"),
933 &prefixes, &cached_hashes, Time::Now());
934 ASSERT_EQ(1U, cached_hashes.size());
935 EXPECT_TRUE(
936 SBFullHashEqual(cached_hashes[0].hash,
937 SBFullHashForString("www.evil.com/malware.html")));
938
939 prefixes.clear();
940 cached_hashes.clear();
941
942 // Test removing a prefix via a sub chunk.
943 SBChunk chunk;
944 SBChunkList chunks;
945 InsertSubChunkHostPrefixUrl(&chunk, 2, 1, "www.evil.com/",
946 "www.evil.com/phishing.html");
947 chunks.push_back(chunk);
948
949 std::vector<SBListChunkRanges> lists;
950 EXPECT_TRUE(database_->UpdateStarted(&lists));
951 database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
952 database_->UpdateFinished(true);
953
954 // This prefix should still be there, but the fullhash is gone.
955 EXPECT_TRUE(database_->ContainsBrowseUrl(
956 GURL("http://www.evil.com/malware.html"),
957 &prefixes, &cached_hashes, Time::Now()));
958 ASSERT_EQ(1U, prefixes.size());
959 EXPECT_EQ(SBPrefixForString("www.evil.com/malware.html"), prefixes[0]);
960 EXPECT_TRUE(cached_hashes.empty());
961 prefixes.clear();
962 cached_hashes.clear();
963
964 // This prefix should be gone.
965 database_->ContainsBrowseUrl(
966 GURL("http://www.evil.com/phishing.html"),
967 &prefixes, &cached_hashes, Time::Now());
968 EXPECT_TRUE(cached_hashes.empty());
969
970 prefixes.clear();
971 cached_hashes.clear();
972
973 // Test that an AddDel for the original chunk removes the last cached entry.
974 EXPECT_TRUE(database_->UpdateStarted(&lists));
975 AddDelChunk(safe_browsing_util::kMalwareList, 1);
976 database_->UpdateFinished(true);
977 database_->ContainsBrowseUrl(
978 GURL("http://www.evil.com/malware.html"),
979 &prefixes, &cached_hashes, Time::Now());
980 EXPECT_TRUE(cached_hashes.empty());
981 EXPECT_TRUE(database_->cached_browse_hashes_.empty());
982
983 prefixes.clear();
984 cached_hashes.clear();
985
986 // Test that the cache won't return expired values. First we have to adjust
987 // the cached entries' received time to make them older, since the database
988 // cache insert uses Time::Now(). First, store some entries.
989 PopulateDatabaseForCacheTest();
990
991 std::vector<SBFullHashCached>* hash_cache = &database_->cached_browse_hashes_;
992 EXPECT_EQ(2U, hash_cache->size());
993
994 // Now adjust one of the entries times to be in the past.
995 base::Time expired = base::Time::Now() - base::TimeDelta::FromMinutes(60);
996 const SBPrefix key = SBPrefixForString("www.evil.com/malware.html");
997 std::vector<SBFullHashCached>::iterator iter;
998 for (iter = hash_cache->begin(); iter != hash_cache->end(); ++iter) {
999 if (iter->hash.prefix == key) {
1000 iter->received = static_cast<int32>(expired.ToTimeT());
1001 break;
1002 }
1003 }
1004 EXPECT_TRUE(iter != hash_cache->end());
1005
1006 database_->ContainsBrowseUrl(
1007 GURL("http://www.evil.com/malware.html"),
1008 &prefixes, &cached_hashes, expired);
1009 EXPECT_TRUE(cached_hashes.empty());
1010
1011 // This entry should still exist.
1012 database_->ContainsBrowseUrl(
1013 GURL("http://www.evil.com/phishing.html"),
1014 &prefixes, &cached_hashes, expired);
1015 EXPECT_EQ(1U, cached_hashes.size());
1016
1017 // Testing prefix miss caching. First, we clear out the existing database,
1018 // Since PopulateDatabaseForCacheTest() doesn't handle adding duplicate
1019 // chunks.
1020 EXPECT_TRUE(database_->UpdateStarted(&lists));
1021 AddDelChunk(safe_browsing_util::kMalwareList, 1);
1022 database_->UpdateFinished(true);
1023
1024 std::vector<SBPrefix> prefix_misses;
1025 std::vector<SBFullHashResult> empty_full_hash;
1026 prefix_misses.push_back(SBPrefixForString("http://www.bad.com/malware.html"));
1027 prefix_misses.push_back(
1028 SBPrefixForString("http://www.bad.com/phishing.html"));
1029 database_->CacheHashResults(prefix_misses, empty_full_hash);
1030
1031 // Prefixes with no full results are misses.
1032 EXPECT_EQ(2U, database_->prefix_miss_cache_.size());
1033
1034 // Update the database.
1035 PopulateDatabaseForCacheTest();
1036
1037 // Prefix miss cache should be cleared.
1038 EXPECT_TRUE(database_->prefix_miss_cache_.empty());
1039
1040 // Cache a GetHash miss for a particular prefix, and even though the prefix is
1041 // in the database, it is flagged as a miss so looking up the associated URL
1042 // will not succeed.
1043 prefixes.clear();
1044 cached_hashes.clear();
1045 prefix_misses.clear();
1046 empty_full_hash.clear();
1047 prefix_misses.push_back(SBPrefixForString("www.evil.com/phishing.html"));
1048 database_->CacheHashResults(prefix_misses, empty_full_hash);
1049 EXPECT_FALSE(database_->ContainsBrowseUrl(
1050 GURL("http://www.evil.com/phishing.html"),
1051 &prefixes, &cached_hashes, Time::Now()));
1052
1053 prefixes.clear();
1054 cached_hashes.clear();
1055
1056 // Test receiving a full add chunk.
1057 chunk.hosts.clear();
1058 InsertAddChunkHost2FullHashes(&chunk, 20, "www.fullevil.com/",
1059 "www.fullevil.com/bad1.html",
1060 "www.fullevil.com/bad2.html");
1061 chunks.clear();
1062 chunks.push_back(chunk);
1063 EXPECT_TRUE(database_->UpdateStarted(&lists));
1064 database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
1065 database_->UpdateFinished(true);
1066
1067 EXPECT_TRUE(database_->ContainsBrowseUrl(
1068 GURL("http://www.fullevil.com/bad1.html"),
1069 &prefixes, &cached_hashes, Time::Now()));
1070 EXPECT_TRUE(cached_hashes.empty());
1071 ASSERT_EQ(1U, prefixes.size());
1072 EXPECT_EQ(SBPrefixForString("www.fullevil.com/bad1.html"), prefixes[0]);
1073 prefixes.clear();
1074 cached_hashes.clear();
1075
1076 EXPECT_TRUE(database_->ContainsBrowseUrl(
1077 GURL("http://www.fullevil.com/bad2.html"),
1078 &prefixes, &cached_hashes, Time::Now()));
1079 EXPECT_TRUE(cached_hashes.empty());
1080 ASSERT_EQ(1U, prefixes.size());
1081 EXPECT_EQ(SBPrefixForString("www.fullevil.com/bad2.html"), prefixes[0]);
1082 prefixes.clear();
1083 cached_hashes.clear();
1084
1085 // Test receiving a full sub chunk, which will remove one of the full adds.
1086 chunk.hosts.clear();
1087 InsertSubChunkHostFullHash(&chunk, 200, 20,
1088 "www.fullevil.com/",
1089 "www.fullevil.com/bad1.html");
1090 chunks.clear();
1091 chunks.push_back(chunk);
1092 EXPECT_TRUE(database_->UpdateStarted(&lists));
1093 database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
1094 database_->UpdateFinished(true);
1095
1096 EXPECT_FALSE(database_->ContainsBrowseUrl(
1097 GURL("http://www.fullevil.com/bad1.html"),
1098 &prefixes, &cached_hashes, Time::Now()));
1099 EXPECT_TRUE(cached_hashes.empty());
1100
1101 // There should be one remaining full add.
1102 EXPECT_TRUE(database_->ContainsBrowseUrl(
1103 GURL("http://www.fullevil.com/bad2.html"),
1104 &prefixes, &cached_hashes, Time::Now()));
1105 EXPECT_TRUE(cached_hashes.empty());
1106 ASSERT_EQ(1U, prefixes.size());
1107 EXPECT_EQ(SBPrefixForString("www.fullevil.com/bad2.html"), prefixes[0]);
1108 prefixes.clear();
1109 cached_hashes.clear();
1110
1111 // Now test an AddDel for the remaining full add.
1112 EXPECT_TRUE(database_->UpdateStarted(&lists));
1113 AddDelChunk(safe_browsing_util::kMalwareList, 20);
1114 database_->UpdateFinished(true);
1115
1116 EXPECT_FALSE(database_->ContainsBrowseUrl(
1117 GURL("http://www.fullevil.com/bad1.html"),
1118 &prefixes, &cached_hashes, Time::Now()));
1119 EXPECT_FALSE(database_->ContainsBrowseUrl(
1120 GURL("http://www.fullevil.com/bad2.html"),
1121 &prefixes, &cached_hashes, Time::Now()));
1122
1123 // Add a fullhash which has a prefix collision for a known url.
1124 static const char kExampleFine[] = "www.example.com/fine.html";
1125 static const char kExampleCollision[] =
1126 "www.example.com/3123364814/malware.htm";
1127 ASSERT_EQ(SBPrefixForString(kExampleFine),
1128 SBPrefixForString(kExampleCollision));
1129 EXPECT_TRUE(database_->UpdateStarted(&lists));
1130 {
1131 SBChunkList chunks;
1132 SBChunk chunk;
1133 InsertAddChunkHostPrefixUrl(&chunk, 21, "www.example.com/",
1134 kExampleCollision);
1135 chunks.push_back(chunk);
1136 database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
1137 }
1138 database_->UpdateFinished(true);
1139
1140 // Cache gethash response for |kExampleCollision|.
1141 {
1142 SBFullHashResult result;
1143 result.hash = SBFullHashForString(kExampleCollision);
1144 result.list_id = safe_browsing_util::MALWARE;
1145 database_->CacheHashResults(std::vector<SBPrefix>(1, result.hash.prefix),
1146 std::vector<SBFullHashResult>(1, result));
1147 }
1148
1149 // Expect a prefix hit due to the collision between |kExampleFine| and
1150 // |kExampleCollision|, with the gethash showing only |kExampleCollision|.
1151 EXPECT_TRUE(database_->ContainsBrowseUrl(
1152 GURL(std::string("http://") + kExampleFine),
1153 &prefixes, &cached_hashes, Time::Now()));
1154 ASSERT_EQ(1U, prefixes.size());
1155 EXPECT_EQ(SBPrefixForString(kExampleFine), prefixes[0]);
1156 ASSERT_EQ(1U, cached_hashes.size());
1157 EXPECT_TRUE(SBFullHashEqual(cached_hashes[0].hash,
1158 SBFullHashForString(kExampleCollision)));
1159 }
1160
1161 // Test that corrupt databases are appropriately handled, even if the
1162 // corruption is detected in the midst of the update.
1163 // TODO(shess): Disabled until ScopedLogMessageIgnorer resolved.
1164 // http://crbug.com/56448
1165 TEST_F(SafeBrowsingDatabaseTest, DISABLED_FileCorruptionHandling) {
1166 // Re-create the database in a captive message loop so that we can
1167 // influence task-posting. Database specifically needs to the
1168 // file-backed.
1169 database_.reset();
1170 base::MessageLoop loop;
1171 SafeBrowsingStoreFile* store = new SafeBrowsingStoreFile();
1172 database_.reset(new SafeBrowsingDatabaseNew(store, NULL, NULL, NULL, NULL,
1173 NULL, NULL));
1174 database_->Init(database_filename_);
1175
1176 // This will cause an empty database to be created.
1177 std::vector<SBListChunkRanges> lists;
1178 EXPECT_TRUE(database_->UpdateStarted(&lists));
1179 database_->UpdateFinished(true);
1180
1181 // Create a sub chunk to insert.
1182 SBChunkList chunks;
1183 SBChunk chunk;
1184 SBChunkHost host;
1185 host.host = SBPrefixForString("www.subbed.com/");
1186 host.entry = SBEntry::Create(SBEntry::SUB_PREFIX, 1);
1187 host.entry->set_chunk_id(7);
1188 host.entry->SetChunkIdAtPrefix(0, 19);
1189 host.entry->SetPrefixAt(0, SBPrefixForString("www.subbed.com/notevil1.html"));
1190 chunk.chunk_number = 7;
1191 chunk.is_add = false;
1192 chunk.hosts.clear();
1193 chunk.hosts.push_back(host);
1194 chunks.clear();
1195 chunks.push_back(chunk);
1196
1197 // Corrupt the file by corrupting the checksum, which is not checked
1198 // until the entire table is read in |UpdateFinished()|.
1199 FILE* fp = base::OpenFile(database_filename_, "r+");
1200 ASSERT_TRUE(fp);
1201 ASSERT_NE(-1, fseek(fp, -8, SEEK_END));
1202 for (size_t i = 0; i < 8; ++i) {
1203 fputc('!', fp);
1204 }
1205 fclose(fp);
1206
1207 {
1208 // The following code will cause DCHECKs, so suppress the crashes.
1209 ScopedLogMessageIgnorer ignorer;
1210
1211 // Start an update. The insert will fail due to corruption.
1212 EXPECT_TRUE(database_->UpdateStarted(&lists));
1213 database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
1214 database_->UpdateFinished(true);
1215
1216 // Database file still exists until the corruption handler has run.
1217 EXPECT_TRUE(base::PathExists(database_filename_));
1218
1219 // Flush through the corruption-handler task.
1220 VLOG(1) << "Expect failed check on: SafeBrowsing database reset";
1221 base::MessageLoop::current()->RunUntilIdle();
1222 }
1223
1224 // Database file should not exist.
1225 EXPECT_FALSE(base::PathExists(database_filename_));
1226
1227 // Run the update again successfully.
1228 EXPECT_TRUE(database_->UpdateStarted(&lists));
1229 database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
1230 database_->UpdateFinished(true);
1231 EXPECT_TRUE(base::PathExists(database_filename_));
1232
1233 database_.reset();
1234 }
1235
1236 // Checks database reading and writing.
1237 TEST_F(SafeBrowsingDatabaseTest, ContainsDownloadUrl) {
1238 database_.reset();
1239 base::MessageLoop loop;
1240 SafeBrowsingStoreFile* browse_store = new SafeBrowsingStoreFile();
1241 SafeBrowsingStoreFile* download_store = new SafeBrowsingStoreFile();
1242 SafeBrowsingStoreFile* csd_whitelist_store = new SafeBrowsingStoreFile();
1243 database_.reset(new SafeBrowsingDatabaseNew(browse_store,
1244 download_store,
1245 csd_whitelist_store,
1246 NULL,
1247 NULL,
1248 NULL,
1249 NULL));
1250 database_->Init(database_filename_);
1251
1252 const char kEvil1Host[] = "www.evil1.com/";
1253 const char kEvil1Url1[] = "www.evil1.com/download1/";
1254 const char kEvil1Url2[] = "www.evil1.com/download2.html";
1255
1256 SBChunkList chunks;
1257 SBChunk chunk;
1258 // Add a simple chunk with one hostkey for download url list.
1259 InsertAddChunkHost2PrefixUrls(&chunk, 1, kEvil1Host,
1260 kEvil1Url1, kEvil1Url2);
1261 chunks.push_back(chunk);
1262 std::vector<SBListChunkRanges> lists;
1263 EXPECT_TRUE(database_->UpdateStarted(&lists));
1264 database_->InsertChunks(safe_browsing_util::kBinUrlList, chunks);
1265 database_->UpdateFinished(true);
1266
1267 std::vector<SBPrefix> prefix_hits;
1268 std::vector<GURL> urls(1);
1269
1270 urls[0] = GURL(std::string("http://") + kEvil1Url1);
1271 EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1272 ASSERT_EQ(prefix_hits.size(), 1U);
1273 EXPECT_EQ(prefix_hits[0], SBPrefixForString(kEvil1Url1));
1274
1275 urls[0] = GURL(std::string("http://") + kEvil1Url2);
1276 EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1277 ASSERT_EQ(prefix_hits.size(), 1U);
1278 EXPECT_EQ(prefix_hits[0], SBPrefixForString(kEvil1Url2));
1279
1280 urls[0] = GURL(std::string("https://") + kEvil1Url2);
1281 EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1282 ASSERT_EQ(prefix_hits.size(), 1U);
1283 EXPECT_EQ(prefix_hits[0], SBPrefixForString(kEvil1Url2));
1284
1285 urls[0] = GURL(std::string("ftp://") + kEvil1Url2);
1286 EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1287 ASSERT_EQ(prefix_hits.size(), 1U);
1288 EXPECT_EQ(prefix_hits[0], SBPrefixForString(kEvil1Url2));
1289
1290 urls[0] = GURL("http://www.randomevil.com");
1291 EXPECT_FALSE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1292
1293 // Should match with query args stripped.
1294 urls[0] = GURL(std::string("http://") + kEvil1Url2 + "?blah");
1295 EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1296 ASSERT_EQ(prefix_hits.size(), 1U);
1297 EXPECT_EQ(prefix_hits[0], SBPrefixForString(kEvil1Url2));
1298
1299 // Should match with extra path stuff and query args stripped.
1300 urls[0] = GURL(std::string("http://") + kEvil1Url1 + "foo/bar?blah");
1301 EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1302 ASSERT_EQ(prefix_hits.size(), 1U);
1303 EXPECT_EQ(prefix_hits[0], SBPrefixForString(kEvil1Url1));
1304
1305 // First hit in redirect chain is malware.
1306 urls.clear();
1307 urls.push_back(GURL(std::string("http://") + kEvil1Url1));
1308 urls.push_back(GURL("http://www.randomevil.com"));
1309 EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1310 ASSERT_EQ(prefix_hits.size(), 1U);
1311 EXPECT_EQ(prefix_hits[0], SBPrefixForString(kEvil1Url1));
1312
1313 // Middle hit in redirect chain is malware.
1314 urls.clear();
1315 urls.push_back(GURL("http://www.randomevil.com"));
1316 urls.push_back(GURL(std::string("http://") + kEvil1Url1));
1317 urls.push_back(GURL("http://www.randomevil2.com"));
1318 EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1319 ASSERT_EQ(prefix_hits.size(), 1U);
1320 EXPECT_EQ(prefix_hits[0], SBPrefixForString(kEvil1Url1));
1321
1322 // Final hit in redirect chain is malware.
1323 urls.clear();
1324 urls.push_back(GURL("http://www.randomevil.com"));
1325 urls.push_back(GURL(std::string("http://") + kEvil1Url1));
1326 EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1327 ASSERT_EQ(prefix_hits.size(), 1U);
1328 EXPECT_EQ(prefix_hits[0], SBPrefixForString(kEvil1Url1));
1329
1330 // Multiple hits in redirect chain are in malware list.
1331 urls.clear();
1332 urls.push_back(GURL(std::string("http://") + kEvil1Url1));
1333 urls.push_back(GURL(std::string("https://") + kEvil1Url2));
1334 EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1335 ASSERT_EQ(prefix_hits.size(), 2U);
1336 EXPECT_EQ(prefix_hits[0], SBPrefixForString(kEvil1Url1));
1337 EXPECT_EQ(prefix_hits[1], SBPrefixForString(kEvil1Url2));
1338 database_.reset();
1339 }
1340
1341 // Checks that the whitelists are handled properly.
1342 TEST_F(SafeBrowsingDatabaseTest, Whitelists) {
1343 database_.reset();
1344 // We expect all calls to ContainsCsdWhitelistedUrl in particular to be made
1345 // from the IO thread. In general the whitelist lookups are thread-safe.
1346 content::TestBrowserThreadBundle thread_bundle_;
1347
1348 // If the whitelist is disabled everything should match the whitelist.
1349 database_.reset(new SafeBrowsingDatabaseNew(new SafeBrowsingStoreFile(),
1350 NULL, NULL, NULL, NULL, NULL,
1351 NULL));
1352 database_->Init(database_filename_);
1353 EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1354 GURL(std::string("http://www.phishing.com/"))));
1355 EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1356 GURL(std::string("http://www.phishing.com/"))));
1357 EXPECT_TRUE(database_->ContainsDownloadWhitelistedString("asdf"));
1358
1359 SafeBrowsingStoreFile* browse_store = new SafeBrowsingStoreFile();
1360 SafeBrowsingStoreFile* csd_whitelist_store = new SafeBrowsingStoreFile();
1361 SafeBrowsingStoreFile* download_whitelist_store = new SafeBrowsingStoreFile();
1362 SafeBrowsingStoreFile* extension_blacklist_store =
1363 new SafeBrowsingStoreFile();
1364 database_.reset(new SafeBrowsingDatabaseNew(browse_store, NULL,
1365 csd_whitelist_store,
1366 download_whitelist_store,
1367 extension_blacklist_store,
1368 NULL, NULL));
1369 database_->Init(database_filename_);
1370
1371 const char kGood1Host[] = "www.good1.com/";
1372 const char kGood1Url1[] = "www.good1.com/a/b.html";
1373 const char kGood1Url2[] = "www.good1.com/b/";
1374
1375 const char kGood2Host[] = "www.good2.com/";
1376 const char kGood2Url1[] = "www.good2.com/c"; // Should match '/c/bla'.
1377
1378 // good3.com/a/b/c/d/e/f/g/ should match because it's a whitelist.
1379 const char kGood3Host[] = "good3.com/";
1380 const char kGood3Url1[] = "good3.com/";
1381
1382 const char kGoodString[] = "good_string";
1383
1384 SBChunkList download_chunks, csd_chunks;
1385 SBChunk chunk;
1386 // Add two simple chunks to the csd whitelist.
1387 InsertAddChunkHost2FullHashes(&chunk, 1, kGood1Host,
1388 kGood1Url1, kGood1Url2);
1389 csd_chunks.push_back(chunk);
1390
1391 chunk.hosts.clear();
1392 InsertAddChunkHostFullHashes(&chunk, 2, kGood2Host, kGood2Url1);
1393 csd_chunks.push_back(chunk);
1394
1395 chunk.hosts.clear();
1396 InsertAddChunkHostFullHashes(&chunk, 2, kGood2Host, kGood2Url1);
1397 download_chunks.push_back(chunk);
1398
1399 chunk.hosts.clear();
1400 InsertAddChunkHostFullHashes(&chunk, 3, kGoodString, kGoodString);
1401 download_chunks.push_back(chunk);
1402
1403 chunk.hosts.clear();
1404 InsertAddChunkHostFullHashes(&chunk, 4, kGood3Host, kGood3Url1);
1405 download_chunks.push_back(chunk);
1406
1407 std::vector<SBListChunkRanges> lists;
1408 EXPECT_TRUE(database_->UpdateStarted(&lists));
1409 database_->InsertChunks(safe_browsing_util::kCsdWhiteList,
1410 csd_chunks);
1411 database_->InsertChunks(safe_browsing_util::kDownloadWhiteList,
1412 download_chunks);
1413 database_->UpdateFinished(true);
1414
1415 EXPECT_FALSE(database_->ContainsCsdWhitelistedUrl(
1416 GURL(std::string("http://") + kGood1Host)));
1417
1418 EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1419 GURL(std::string("http://") + kGood1Url1)));
1420 EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1421 GURL(std::string("http://") + kGood1Url1 + "?a=b")));
1422
1423 EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1424 GURL(std::string("http://") + kGood1Url2)));
1425 EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1426 GURL(std::string("http://") + kGood1Url2 + "/c.html")));
1427
1428 EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1429 GURL(std::string("https://") + kGood1Url2 + "/c.html")));
1430
1431 EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1432 GURL(std::string("http://") + kGood2Url1 + "/c")));
1433 EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1434 GURL(std::string("http://") + kGood2Url1 + "/c?bla")));
1435 EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1436 GURL(std::string("http://") + kGood2Url1 + "/c/bla")));
1437
1438 EXPECT_FALSE(database_->ContainsCsdWhitelistedUrl(
1439 GURL(std::string("http://www.google.com/"))));
1440
1441 EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1442 GURL(std::string("http://") + kGood2Url1 + "/c")));
1443 EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1444 GURL(std::string("http://") + kGood2Url1 + "/c?bla")));
1445 EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1446 GURL(std::string("http://") + kGood2Url1 + "/c/bla")));
1447
1448 EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1449 GURL(std::string("http://good3.com/a/b/c/d/e/f/g/"))));
1450 EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1451 GURL(std::string("http://a.b.good3.com/"))));
1452
1453 EXPECT_FALSE(database_->ContainsDownloadWhitelistedString("asdf"));
1454 EXPECT_TRUE(database_->ContainsDownloadWhitelistedString(kGoodString));
1455
1456 EXPECT_FALSE(database_->ContainsDownloadWhitelistedUrl(
1457 GURL(std::string("http://www.google.com/"))));
1458
1459 // Test only add the malware IP killswitch
1460 csd_chunks.clear();
1461 chunk.hosts.clear();
1462 InsertAddChunkHostFullHashes(
1463 &chunk, 15, "sb-ssl.google.com/",
1464 "sb-ssl.google.com/safebrowsing/csd/killswitch_malware");
1465 csd_chunks.push_back(chunk);
1466 EXPECT_TRUE(database_->UpdateStarted(&lists));
1467 database_->InsertChunks(safe_browsing_util::kCsdWhiteList, csd_chunks);
1468 database_->UpdateFinished(true);
1469
1470 EXPECT_TRUE(database_->IsMalwareIPMatchKillSwitchOn());
1471
1472 // Test that the kill-switch works as intended.
1473 csd_chunks.clear();
1474 download_chunks.clear();
1475 lists.clear();
1476 chunk.hosts.clear();
1477 InsertAddChunkHostFullHashes(&chunk, 5, "sb-ssl.google.com/",
1478 "sb-ssl.google.com/safebrowsing/csd/killswitch");
1479 csd_chunks.push_back(chunk);
1480 chunk.hosts.clear();
1481 InsertAddChunkHostFullHashes(&chunk, 5, "sb-ssl.google.com/",
1482 "sb-ssl.google.com/safebrowsing/csd/killswitch");
1483 download_chunks.push_back(chunk);
1484
1485 EXPECT_TRUE(database_->UpdateStarted(&lists));
1486 database_->InsertChunks(safe_browsing_util::kCsdWhiteList, csd_chunks);
1487 database_->InsertChunks(safe_browsing_util::kDownloadWhiteList,
1488 download_chunks);
1489 database_->UpdateFinished(true);
1490
1491 EXPECT_TRUE(database_->IsMalwareIPMatchKillSwitchOn());
1492 EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1493 GURL(std::string("https://") + kGood1Url2 + "/c.html")));
1494 EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1495 GURL(std::string("http://www.google.com/"))));
1496 EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1497 GURL(std::string("http://www.phishing_url.com/"))));
1498
1499 EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1500 GURL(std::string("https://") + kGood1Url2 + "/c.html")));
1501 EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1502 GURL(std::string("http://www.google.com/"))));
1503 EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1504 GURL(std::string("http://www.phishing_url.com/"))));
1505
1506 EXPECT_TRUE(database_->ContainsDownloadWhitelistedString("asdf"));
1507 EXPECT_TRUE(database_->ContainsDownloadWhitelistedString(kGoodString));
1508
1509 // Remove the kill-switch and verify that we can recover.
1510 csd_chunks.clear();
1511 download_chunks.clear();
1512 lists.clear();
1513 SBChunk sub_chunk;
1514 InsertSubChunkHostFullHash(&sub_chunk, 1, 5,
1515 "sb-ssl.google.com/",
1516 "sb-ssl.google.com/safebrowsing/csd/killswitch");
1517 csd_chunks.push_back(sub_chunk);
1518
1519 sub_chunk.hosts.clear();
1520 InsertSubChunkHostFullHash(
1521 &sub_chunk, 10, 15, "sb-ssl.google.com/",
1522 "sb-ssl.google.com/safebrowsing/csd/killswitch_malware");
1523 csd_chunks.push_back(sub_chunk);
1524
1525 sub_chunk.hosts.clear();
1526 InsertSubChunkHostFullHash(&sub_chunk, 1, 5,
1527 "sb-ssl.google.com/",
1528 "sb-ssl.google.com/safebrowsing/csd/killswitch");
1529 download_chunks.push_back(sub_chunk);
1530
1531 EXPECT_TRUE(database_->UpdateStarted(&lists));
1532 database_->InsertChunks(safe_browsing_util::kCsdWhiteList, csd_chunks);
1533 database_->InsertChunks(safe_browsing_util::kDownloadWhiteList,
1534 download_chunks);
1535 database_->UpdateFinished(true);
1536
1537 EXPECT_FALSE(database_->IsMalwareIPMatchKillSwitchOn());
1538 EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1539 GURL(std::string("https://") + kGood1Url2 + "/c.html")));
1540 EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1541 GURL(std::string("https://") + kGood2Url1 + "/c/bla")));
1542 EXPECT_FALSE(database_->ContainsCsdWhitelistedUrl(
1543 GURL(std::string("http://www.google.com/"))));
1544 EXPECT_FALSE(database_->ContainsCsdWhitelistedUrl(
1545 GURL(std::string("http://www.phishing_url.com/"))));
1546
1547 EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1548 GURL(std::string("https://") + kGood2Url1 + "/c/bla")));
1549 EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1550 GURL(std::string("https://good3.com/"))));
1551 EXPECT_TRUE(database_->ContainsDownloadWhitelistedString(kGoodString));
1552 EXPECT_FALSE(database_->ContainsDownloadWhitelistedUrl(
1553 GURL(std::string("http://www.google.com/"))));
1554 EXPECT_FALSE(database_->ContainsDownloadWhitelistedUrl(
1555 GURL(std::string("http://www.phishing_url.com/"))));
1556 EXPECT_FALSE(database_->ContainsDownloadWhitelistedString("asdf"));
1557
1558 database_.reset();
1559 }
1560
1561 // Test to make sure we could insert chunk list that
1562 // contains entries for the same host.
1563 TEST_F(SafeBrowsingDatabaseTest, SameHostEntriesOkay) {
1564 SBChunk chunk;
1565
1566 // Add a malware add chunk with two entries of the same host.
1567 InsertAddChunkHostPrefixUrl(&chunk, 1, "www.evil.com/",
1568 "www.evil.com/malware1.html");
1569 InsertAddChunkHostPrefixUrl(&chunk, 1, "www.evil.com/",
1570 "www.evil.com/malware2.html");
1571 SBChunkList chunks;
1572 chunks.push_back(chunk);
1573
1574 // Insert the testing chunks into database.
1575 std::vector<SBListChunkRanges> lists;
1576 EXPECT_TRUE(database_->UpdateStarted(&lists));
1577 database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
1578 database_->UpdateFinished(true);
1579
1580 GetListsInfo(&lists);
1581 EXPECT_EQ(std::string(safe_browsing_util::kMalwareList), lists[0].name);
1582 EXPECT_EQ("1", lists[0].adds);
1583 EXPECT_TRUE(lists[0].subs.empty());
1584
1585 // Add a phishing add chunk with two entries of the same host.
1586 chunk.hosts.clear();
1587 InsertAddChunkHostPrefixUrl(&chunk, 47, "www.evil.com/",
1588 "www.evil.com/phishing1.html");
1589 InsertAddChunkHostPrefixUrl(&chunk, 47, "www.evil.com/",
1590 "www.evil.com/phishing2.html");
1591 chunks.clear();
1592 chunks.push_back(chunk);
1593
1594 EXPECT_TRUE(database_->UpdateStarted(&lists));
1595 database_->InsertChunks(safe_browsing_util::kPhishingList, chunks);
1596 database_->UpdateFinished(true);
1597
1598 GetListsInfo(&lists);
1599 EXPECT_EQ(std::string(safe_browsing_util::kMalwareList), lists[0].name);
1600 EXPECT_EQ("1", lists[0].adds);
1601 EXPECT_EQ(std::string(safe_browsing_util::kPhishingList), lists[1].name);
1602 EXPECT_EQ("47", lists[1].adds);
1603
1604 const Time now = Time::Now();
1605 std::vector<SBPrefix> prefixes;
1606 std::vector<SBFullHashResult> cached_hashes;
1607 std::vector<SBPrefix> prefix_hits;
1608
1609 EXPECT_TRUE(database_->ContainsBrowseUrl(
1610 GURL("http://www.evil.com/malware1.html"),
1611 &prefixes, &cached_hashes, now));
1612 EXPECT_TRUE(database_->ContainsBrowseUrl(
1613 GURL("http://www.evil.com/malware2.html"),
1614 &prefixes, &cached_hashes, now));
1615 EXPECT_TRUE(database_->ContainsBrowseUrl(
1616 GURL("http://www.evil.com/phishing1.html"),
1617 &prefixes, &cached_hashes, now));
1618 EXPECT_TRUE(database_->ContainsBrowseUrl(
1619 GURL("http://www.evil.com/phishing2.html"),
1620 &prefixes, &cached_hashes, now));
1621
1622 // Test removing a single prefix from the add chunk.
1623 // Remove the prefix that added first.
1624 chunk.hosts.clear();
1625 InsertSubChunkHostPrefixUrl(&chunk, 4, 1, "www.evil.com/",
1626 "www.evil.com/malware1.html");
1627 chunks.clear();
1628 chunks.push_back(chunk);
1629 EXPECT_TRUE(database_->UpdateStarted(&lists));
1630 database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
1631 database_->UpdateFinished(true);
1632
1633 // Remove the prefix that added last.
1634 chunk.hosts.clear();
1635 InsertSubChunkHostPrefixUrl(&chunk, 5, 47, "www.evil.com/",
1636 "www.evil.com/phishing2.html");
1637 chunks.clear();
1638 chunks.push_back(chunk);
1639 EXPECT_TRUE(database_->UpdateStarted(&lists));
1640 database_->InsertChunks(safe_browsing_util::kPhishingList, chunks);
1641 database_->UpdateFinished(true);
1642
1643 // Verify that the database contains urls expected.
1644 EXPECT_FALSE(database_->ContainsBrowseUrl(
1645 GURL("http://www.evil.com/malware1.html"),
1646 &prefixes, &cached_hashes, now));
1647 EXPECT_TRUE(database_->ContainsBrowseUrl(
1648 GURL("http://www.evil.com/malware2.html"),
1649 &prefixes, &cached_hashes, now));
1650 EXPECT_TRUE(database_->ContainsBrowseUrl(
1651 GURL("http://www.evil.com/phishing1.html"),
1652 &prefixes, &cached_hashes, now));
1653 EXPECT_FALSE(database_->ContainsBrowseUrl(
1654 GURL("http://www.evil.com/phishing2.html"),
1655 &prefixes, &cached_hashes, now));
1656 }
1657
1658 // Test that an empty update doesn't actually update the database.
1659 // This isn't a functionality requirement, but it is a useful
1660 // optimization.
1661 TEST_F(SafeBrowsingDatabaseTest, EmptyUpdate) {
1662 SBChunkList chunks;
1663 SBChunk chunk;
1664
1665 base::FilePath filename = database_->BrowseDBFilename(database_filename_);
1666
1667 // Prime the database.
1668 std::vector<SBListChunkRanges> lists;
1669 EXPECT_TRUE(database_->UpdateStarted(&lists));
1670
1671 InsertAddChunkHostPrefixUrl(&chunk, 1, "www.evil.com/",
1672 "www.evil.com/malware.html");
1673 chunks.clear();
1674 chunks.push_back(chunk);
1675 database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
1676 database_->UpdateFinished(true);
1677
1678 // Get an older time to reset the lastmod time for detecting whether
1679 // the file has been updated.
1680 base::File::Info before_info, after_info;
1681 ASSERT_TRUE(base::GetFileInfo(filename, &before_info));
1682 const base::Time old_last_modified =
1683 before_info.last_modified - base::TimeDelta::FromSeconds(10);
1684
1685 // Inserting another chunk updates the database file. The sleep is
1686 // needed because otherwise the entire test can finish w/in the
1687 // resolution of the lastmod time.
1688 ASSERT_TRUE(base::TouchFile(filename, old_last_modified, old_last_modified));
1689 ASSERT_TRUE(base::GetFileInfo(filename, &before_info));
1690 EXPECT_TRUE(database_->UpdateStarted(&lists));
1691 chunk.hosts.clear();
1692 InsertAddChunkHostPrefixUrl(&chunk, 2, "www.foo.com/",
1693 "www.foo.com/malware.html");
1694 chunks.clear();
1695 chunks.push_back(chunk);
1696 database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
1697 database_->UpdateFinished(true);
1698 ASSERT_TRUE(base::GetFileInfo(filename, &after_info));
1699 EXPECT_LT(before_info.last_modified, after_info.last_modified);
1700
1701 // Deleting a chunk updates the database file.
1702 ASSERT_TRUE(base::TouchFile(filename, old_last_modified, old_last_modified));
1703 ASSERT_TRUE(base::GetFileInfo(filename, &before_info));
1704 EXPECT_TRUE(database_->UpdateStarted(&lists));
1705 AddDelChunk(safe_browsing_util::kMalwareList, chunk.chunk_number);
1706 database_->UpdateFinished(true);
1707 ASSERT_TRUE(base::GetFileInfo(filename, &after_info));
1708 EXPECT_LT(before_info.last_modified, after_info.last_modified);
1709
1710 // Simply calling |UpdateStarted()| then |UpdateFinished()| does not
1711 // update the database file.
1712 ASSERT_TRUE(base::TouchFile(filename, old_last_modified, old_last_modified));
1713 ASSERT_TRUE(base::GetFileInfo(filename, &before_info));
1714 EXPECT_TRUE(database_->UpdateStarted(&lists));
1715 database_->UpdateFinished(true);
1716 ASSERT_TRUE(base::GetFileInfo(filename, &after_info));
1717 EXPECT_EQ(before_info.last_modified, after_info.last_modified);
1718 }
1719
1720 // Test that a filter file is written out during update and read back
1721 // in during setup.
1722 TEST_F(SafeBrowsingDatabaseTest, FilterFile) {
1723 // Create a database with trivial example data and write it out.
1724 {
1725 SBChunkList chunks;
1726 SBChunk chunk;
1727
1728 // Prime the database.
1729 std::vector<SBListChunkRanges> lists;
1730 EXPECT_TRUE(database_->UpdateStarted(&lists));
1731
1732 InsertAddChunkHostPrefixUrl(&chunk, 1, "www.evil.com/",
1733 "www.evil.com/malware.html");
1734 chunks.clear();
1735 chunks.push_back(chunk);
1736 database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
1737 database_->UpdateFinished(true);
1738 }
1739
1740 // Find the malware url in the database, don't find a good url.
1741 const Time now = Time::Now();
1742 std::vector<SBFullHashResult> cached_hashes;
1743 std::vector<SBPrefix> prefix_hits;
1744 EXPECT_TRUE(database_->ContainsBrowseUrl(
1745 GURL("http://www.evil.com/malware.html"),
1746 &prefix_hits, &cached_hashes, now));
1747 EXPECT_FALSE(database_->ContainsBrowseUrl(
1748 GURL("http://www.good.com/goodware.html"),
1749 &prefix_hits, &cached_hashes, now));
1750
1751 base::FilePath filter_file = database_->PrefixSetForFilename(
1752 database_->BrowseDBFilename(database_filename_));
1753
1754 // After re-creating the database, it should have a filter read from
1755 // a file, so it should find the same results.
1756 ASSERT_TRUE(base::PathExists(filter_file));
1757 database_.reset(new SafeBrowsingDatabaseNew);
1758 database_->Init(database_filename_);
1759 EXPECT_TRUE(database_->ContainsBrowseUrl(
1760 GURL("http://www.evil.com/malware.html"),
1761 &prefix_hits, &cached_hashes, now));
1762 EXPECT_FALSE(database_->ContainsBrowseUrl(
1763 GURL("http://www.good.com/goodware.html"),
1764 &prefix_hits, &cached_hashes, now));
1765
1766 // If there is no filter file, the database cannot find malware urls.
1767 base::DeleteFile(filter_file, false);
1768 ASSERT_FALSE(base::PathExists(filter_file));
1769 database_.reset(new SafeBrowsingDatabaseNew);
1770 database_->Init(database_filename_);
1771 EXPECT_FALSE(database_->ContainsBrowseUrl(
1772 GURL("http://www.evil.com/malware.html"),
1773 &prefix_hits, &cached_hashes, now));
1774 EXPECT_FALSE(database_->ContainsBrowseUrl(
1775 GURL("http://www.good.com/goodware.html"),
1776 &prefix_hits, &cached_hashes, now));
1777 }
1778
1779 TEST_F(SafeBrowsingDatabaseTest, MalwareIpBlacklist) {
1780 database_.reset();
1781 SafeBrowsingStoreFile* browse_store = new SafeBrowsingStoreFile();
1782 SafeBrowsingStoreFile* ip_blacklist_store = new SafeBrowsingStoreFile();
1783 database_.reset(new SafeBrowsingDatabaseNew(browse_store,
1784 NULL,
1785 NULL,
1786 NULL,
1787 NULL,
1788 NULL,
1789 ip_blacklist_store));
1790 database_->Init(database_filename_);
1791 std::vector<SBListChunkRanges> lists;
1792 EXPECT_TRUE(database_->UpdateStarted(&lists));
1793
1794 // IPv4 prefix match for ::ffff:192.168.1.0/120.
1795 SBChunkList chunks;
1796 SBChunk chunk;
1797 InsertAddChunkFullHash(&chunk, 1, "::ffff:192.168.1.0", 120);
1798 chunks.push_back(chunk);
1799 database_->InsertChunks(safe_browsing_util::kIPBlacklist, chunks);
1800
1801 // IPv4 exact match for ::ffff:192.1.1.1.
1802 chunks.clear();
1803 chunk.hosts.clear();
1804 InsertAddChunkFullHash(&chunk, 2, "::ffff:192.1.1.1", 128);
1805 chunks.push_back(chunk);
1806 database_->InsertChunks(safe_browsing_util::kIPBlacklist, chunks);
1807
1808 // IPv6 exact match for: fe80::31a:a0ff:fe10:786e/128.
1809 chunks.clear();
1810 chunk.hosts.clear();
1811 InsertAddChunkFullHash(&chunk, 3, "fe80::31a:a0ff:fe10:786e", 128);
1812 chunks.push_back(chunk);
1813 database_->InsertChunks(safe_browsing_util::kIPBlacklist, chunks);
1814
1815 // IPv6 prefix match for: 2620:0:1000:3103::/64.
1816 chunks.clear();
1817 chunk.hosts.clear();
1818 InsertAddChunkFullHash(&chunk, 4, "2620:0:1000:3103::", 64);
1819 chunks.push_back(chunk);
1820 database_->InsertChunks(safe_browsing_util::kIPBlacklist, chunks);
1821
1822 // IPv4 prefix match for ::ffff:192.1.122.0/119.
1823 chunks.clear();
1824 chunk.hosts.clear();
1825 InsertAddChunkFullHash(&chunk, 5, "::ffff:192.1.122.0", 119);
1826 chunks.push_back(chunk);
1827 database_->InsertChunks(safe_browsing_util::kIPBlacklist, chunks);
1828
1829 // IPv4 prefix match for ::ffff:192.1.128.0/113.
1830 chunks.clear();
1831 chunk.hosts.clear();
1832 InsertAddChunkFullHash(&chunk, 6, "::ffff:192.1.128.0", 113);
1833 chunks.push_back(chunk);
1834 database_->InsertChunks(safe_browsing_util::kIPBlacklist, chunks);
1835
1836 database_->UpdateFinished(true);
1837
1838 EXPECT_FALSE(database_->ContainsMalwareIP("192.168.0.255"));
1839 EXPECT_TRUE(database_->ContainsMalwareIP("192.168.1.0"));
1840 EXPECT_TRUE(database_->ContainsMalwareIP("192.168.1.255"));
1841 EXPECT_TRUE(database_->ContainsMalwareIP("192.168.1.10"));
1842 EXPECT_TRUE(database_->ContainsMalwareIP("::ffff:192.168.1.2"));
1843 EXPECT_FALSE(database_->ContainsMalwareIP("192.168.2.0"));
1844
1845 EXPECT_FALSE(database_->ContainsMalwareIP("192.1.1.0"));
1846 EXPECT_TRUE(database_->ContainsMalwareIP("192.1.1.1"));
1847 EXPECT_FALSE(database_->ContainsMalwareIP("192.1.1.2"));
1848
1849 EXPECT_FALSE(database_->ContainsMalwareIP(
1850 "2620:0:1000:3102:ffff:ffff:ffff:ffff"));
1851 EXPECT_TRUE(database_->ContainsMalwareIP("2620:0:1000:3103::"));
1852 EXPECT_TRUE(database_->ContainsMalwareIP(
1853 "2620:0:1000:3103:ffff:ffff:ffff:ffff"));
1854 EXPECT_FALSE(database_->ContainsMalwareIP("2620:0:1000:3104::"));
1855
1856 EXPECT_FALSE(database_->ContainsMalwareIP("fe80::21a:a0ff:fe10:786d"));
1857 EXPECT_TRUE(database_->ContainsMalwareIP("fe80::31a:a0ff:fe10:786e"));
1858 EXPECT_FALSE(database_->ContainsMalwareIP("fe80::21a:a0ff:fe10:786f"));
1859
1860 EXPECT_FALSE(database_->ContainsMalwareIP("192.1.121.255"));
1861 EXPECT_TRUE(database_->ContainsMalwareIP("192.1.122.0"));
1862 EXPECT_TRUE(database_->ContainsMalwareIP("::ffff:192.1.122.1"));
1863 EXPECT_TRUE(database_->ContainsMalwareIP("192.1.122.255"));
1864 EXPECT_TRUE(database_->ContainsMalwareIP("192.1.123.0"));
1865 EXPECT_TRUE(database_->ContainsMalwareIP("192.1.123.255"));
1866 EXPECT_FALSE(database_->ContainsMalwareIP("192.1.124.0"));
1867
1868 EXPECT_FALSE(database_->ContainsMalwareIP("192.1.127.255"));
1869 EXPECT_TRUE(database_->ContainsMalwareIP("192.1.128.0"));
1870 EXPECT_TRUE(database_->ContainsMalwareIP("::ffff:192.1.128.1"));
1871 EXPECT_TRUE(database_->ContainsMalwareIP("192.1.128.255"));
1872 EXPECT_TRUE(database_->ContainsMalwareIP("192.1.255.0"));
1873 EXPECT_TRUE(database_->ContainsMalwareIP("192.1.255.255"));
1874 EXPECT_FALSE(database_->ContainsMalwareIP("192.2.0.0"));
1875 }
1876
1877 TEST_F(SafeBrowsingDatabaseTest, ContainsBrowseURL) {
1878 std::vector<SBListChunkRanges> lists;
1879 EXPECT_TRUE(database_->UpdateStarted(&lists));
1880
1881 // Add a host-level hit.
1882 {
1883 SBChunkList chunks;
1884 SBChunk chunk;
1885 InsertAddChunkHostPrefix(&chunk, 1, "www.evil.com/");
1886 chunks.push_back(chunk);
1887 database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
1888 }
1889
1890 // Add a specific fullhash.
1891 static const char kWhateverMalware[] = "www.whatever.com/malware.html";
1892 {
1893 SBChunkList chunks;
1894 SBChunk chunk;
1895 InsertAddChunkHostFullHashes(&chunk, 2, "www.whatever.com/",
1896 kWhateverMalware);
1897 chunks.push_back(chunk);
1898 database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
1899 }
1900
1901 // Add a fullhash which has a prefix collision for a known url.
1902 static const char kExampleFine[] = "www.example.com/fine.html";
1903 static const char kExampleCollision[] =
1904 "www.example.com/3123364814/malware.htm";
1905 ASSERT_EQ(SBPrefixForString(kExampleFine),
1906 SBPrefixForString(kExampleCollision));
1907 {
1908 SBChunkList chunks;
1909 SBChunk chunk;
1910 InsertAddChunkHostFullHashes(&chunk, 3, "www.example.com/",
1911 kExampleCollision);
1912 chunks.push_back(chunk);
1913 database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
1914 }
1915
1916 database_->UpdateFinished(true);
1917
1918 const Time now = Time::Now();
1919 std::vector<SBFullHashResult> cached_hashes;
1920 std::vector<SBPrefix> prefix_hits;
1921
1922 // Anything will hit the host prefix.
1923 EXPECT_TRUE(database_->ContainsBrowseUrl(
1924 GURL("http://www.evil.com/malware.html"),
1925 &prefix_hits, &cached_hashes, now));
1926 ASSERT_EQ(1U, prefix_hits.size());
1927 EXPECT_EQ(SBPrefixForString("www.evil.com/"), prefix_hits[0]);
1928 EXPECT_TRUE(cached_hashes.empty());
1929
1930 // Hit the specific URL prefix.
1931 EXPECT_TRUE(database_->ContainsBrowseUrl(
1932 GURL(std::string("http://") + kWhateverMalware),
1933 &prefix_hits, &cached_hashes, now));
1934 ASSERT_EQ(1U, prefix_hits.size());
1935 EXPECT_EQ(SBPrefixForString(kWhateverMalware), prefix_hits[0]);
1936 EXPECT_TRUE(cached_hashes.empty());
1937
1938 // Other URLs at that host are fine.
1939 EXPECT_FALSE(database_->ContainsBrowseUrl(
1940 GURL("http://www.whatever.com/fine.html"),
1941 &prefix_hits, &cached_hashes, now));
1942 EXPECT_TRUE(prefix_hits.empty());
1943 EXPECT_TRUE(cached_hashes.empty());
1944
1945 // Hit the specific URL full hash.
1946 EXPECT_TRUE(database_->ContainsBrowseUrl(
1947 GURL(std::string("http://") + kExampleCollision),
1948 &prefix_hits, &cached_hashes, now));
1949 ASSERT_EQ(1U, prefix_hits.size());
1950 EXPECT_EQ(SBPrefixForString(kExampleCollision), prefix_hits[0]);
1951 EXPECT_TRUE(cached_hashes.empty());
1952
1953 // This prefix collides, but no full hash match.
1954 EXPECT_FALSE(database_->ContainsBrowseUrl(
1955 GURL(std::string("http://") + kExampleFine),
1956 &prefix_hits, &cached_hashes, now));
1957 }