QUIC - cleanup changes to sync chromium tree with internal source.
[chromium-blink-merge.git] / chrome / browser / safe_browsing / local_database_manager.cc
blobb61499c6e0b5f744001e386b402500d865861575
1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "chrome/browser/safe_browsing/local_database_manager.h"
7 #include <algorithm>
9 #include "base/bind.h"
10 #include "base/bind_helpers.h"
11 #include "base/callback.h"
12 #include "base/command_line.h"
13 #include "base/debug/leak_tracker.h"
14 #include "base/location.h"
15 #include "base/metrics/histogram_macros.h"
16 #include "base/prefs/pref_service.h"
17 #include "base/single_thread_task_runner.h"
18 #include "base/stl_util.h"
19 #include "base/strings/string_util.h"
20 #include "base/thread_task_runner_handle.h"
21 #include "chrome/browser/browser_process.h"
22 #include "chrome/browser/chrome_notification_types.h"
23 #include "chrome/browser/prerender/prerender_field_trial.h"
24 #include "chrome/browser/profiles/profile_manager.h"
25 #include "chrome/browser/safe_browsing/client_side_detection_service.h"
26 #include "chrome/browser/safe_browsing/download_protection_service.h"
27 #include "chrome/browser/safe_browsing/malware_details.h"
28 #include "chrome/browser/safe_browsing/protocol_manager.h"
29 #include "chrome/browser/safe_browsing/safe_browsing_database.h"
30 #include "chrome/browser/safe_browsing/safe_browsing_service.h"
31 #include "chrome/browser/safe_browsing/ui_manager.h"
32 #include "chrome/common/chrome_constants.h"
33 #include "chrome/common/chrome_paths.h"
34 #include "chrome/common/chrome_switches.h"
35 #include "chrome/common/pref_names.h"
36 #include "content/public/browser/browser_thread.h"
37 #include "content/public/browser/notification_service.h"
38 #include "url/url_constants.h"
40 using content::BrowserThread;
42 namespace {
44 // Timeout for match checks, e.g. download URLs, hashes.
45 const int kCheckTimeoutMs = 10000;
47 // Records disposition information about the check. |hit| should be
48 // |true| if there were any prefix hits in |full_hashes|.
49 void RecordGetHashCheckStatus(
50 bool hit,
51 safe_browsing_util::ListType check_type,
52 const std::vector<SBFullHashResult>& full_hashes) {
53 SafeBrowsingProtocolManager::ResultType result;
54 if (full_hashes.empty()) {
55 result = SafeBrowsingProtocolManager::GET_HASH_FULL_HASH_EMPTY;
56 } else if (hit) {
57 result = SafeBrowsingProtocolManager::GET_HASH_FULL_HASH_HIT;
58 } else {
59 result = SafeBrowsingProtocolManager::GET_HASH_FULL_HASH_MISS;
61 bool is_download = check_type == safe_browsing_util::BINURL;
62 SafeBrowsingProtocolManager::RecordGetHashResult(is_download, result);
65 bool IsExpectedThreat(
66 const SBThreatType threat_type,
67 const std::vector<SBThreatType>& expected_threats) {
68 return expected_threats.end() != std::find(expected_threats.begin(),
69 expected_threats.end(),
70 threat_type);
73 // Return the severest list id from the results in |full_hashes| which matches
74 // |hash|, or INVALID if none match.
75 safe_browsing_util::ListType GetHashSeverestThreatListType(
76 const SBFullHash& hash,
77 const std::vector<SBFullHashResult>& full_hashes,
78 size_t* index) {
79 safe_browsing_util::ListType pending_threat = safe_browsing_util::INVALID;
80 for (size_t i = 0; i < full_hashes.size(); ++i) {
81 if (SBFullHashEqual(hash, full_hashes[i].hash)) {
82 const safe_browsing_util::ListType threat =
83 static_cast<safe_browsing_util::ListType>(full_hashes[i].list_id);
84 switch (threat) {
85 case safe_browsing_util::INVALID:
86 // |full_hashes| should never contain INVALID as a |list_id|.
87 NOTREACHED();
88 break;
89 case safe_browsing_util::MALWARE: // Falls through.
90 case safe_browsing_util::PHISH: // Falls through.
91 case safe_browsing_util::BINURL: // Falls through.
92 case safe_browsing_util::CSDWHITELIST: // Falls through.
93 case safe_browsing_util::DOWNLOADWHITELIST: // Falls through.
94 case safe_browsing_util::INCLUSIONWHITELIST: // Falls through.
95 case safe_browsing_util::EXTENSIONBLACKLIST: // Falls through.
96 case safe_browsing_util::IPBLACKLIST:
97 if (index)
98 *index = i;
99 return threat;
100 case safe_browsing_util::UNWANTEDURL:
101 // UNWANTEDURL is considered less severe than other threats, keep
102 // looking.
103 pending_threat = threat;
104 if (index)
105 *index = i;
106 break;
110 return pending_threat;
113 // Given a URL, compare all the possible host + path full hashes to the set of
114 // provided full hashes. Returns the list id of the severest matching result
115 // from |full_hashes|, or INVALID if none match.
116 safe_browsing_util::ListType GetUrlSeverestThreatListType(
117 const GURL& url,
118 const std::vector<SBFullHashResult>& full_hashes,
119 size_t* index) {
120 if (full_hashes.empty())
121 return safe_browsing_util::INVALID;
123 std::vector<std::string> patterns;
124 safe_browsing_util::GeneratePatternsToCheck(url, &patterns);
126 safe_browsing_util::ListType pending_threat = safe_browsing_util::INVALID;
127 for (size_t i = 0; i < patterns.size(); ++i) {
128 safe_browsing_util::ListType threat = GetHashSeverestThreatListType(
129 SBFullHashForString(patterns[i]), full_hashes, index);
130 switch (threat) {
131 case safe_browsing_util::INVALID:
132 // Ignore patterns with no matching threat.
133 break;
134 case safe_browsing_util::MALWARE: // Falls through.
135 case safe_browsing_util::PHISH: // Falls through.
136 case safe_browsing_util::BINURL: // Falls through.
137 case safe_browsing_util::CSDWHITELIST: // Falls through.
138 case safe_browsing_util::DOWNLOADWHITELIST: // Falls through.
139 case safe_browsing_util::INCLUSIONWHITELIST: // Falls through.
140 case safe_browsing_util::EXTENSIONBLACKLIST: // Falls through.
141 case safe_browsing_util::IPBLACKLIST:
142 return threat;
143 case safe_browsing_util::UNWANTEDURL:
144 // UNWANTEDURL is considered less severe than other threats, keep
145 // looking.
146 pending_threat = threat;
147 break;
150 return pending_threat;
153 SBThreatType GetThreatTypeFromListType(safe_browsing_util::ListType list_type) {
154 switch (list_type) {
155 case safe_browsing_util::PHISH:
156 return SB_THREAT_TYPE_URL_PHISHING;
157 case safe_browsing_util::MALWARE:
158 return SB_THREAT_TYPE_URL_MALWARE;
159 case safe_browsing_util::UNWANTEDURL:
160 return SB_THREAT_TYPE_URL_UNWANTED;
161 case safe_browsing_util::BINURL:
162 return SB_THREAT_TYPE_BINARY_MALWARE_URL;
163 case safe_browsing_util::EXTENSIONBLACKLIST:
164 return SB_THREAT_TYPE_EXTENSION;
165 default:
166 DVLOG(1) << "Unknown safe browsing list id " << list_type;
167 return SB_THREAT_TYPE_SAFE;
171 } // namespace
173 // static
174 SBThreatType LocalSafeBrowsingDatabaseManager::GetHashSeverestThreatType(
175 const SBFullHash& hash,
176 const std::vector<SBFullHashResult>& full_hashes) {
177 return GetThreatTypeFromListType(
178 GetHashSeverestThreatListType(hash, full_hashes, NULL));
181 // static
182 SBThreatType LocalSafeBrowsingDatabaseManager::GetUrlSeverestThreatType(
183 const GURL& url,
184 const std::vector<SBFullHashResult>& full_hashes,
185 size_t* index) {
186 return GetThreatTypeFromListType(
187 GetUrlSeverestThreatListType(url, full_hashes, index));
190 LocalSafeBrowsingDatabaseManager::SafeBrowsingCheck::SafeBrowsingCheck(
191 const std::vector<GURL>& urls,
192 const std::vector<SBFullHash>& full_hashes,
193 Client* client,
194 safe_browsing_util::ListType check_type,
195 const std::vector<SBThreatType>& expected_threats)
196 : urls(urls),
197 url_results(urls.size(), SB_THREAT_TYPE_SAFE),
198 url_metadata(urls.size()),
199 full_hashes(full_hashes),
200 full_hash_results(full_hashes.size(), SB_THREAT_TYPE_SAFE),
201 client(client),
202 need_get_hash(false),
203 check_type(check_type),
204 expected_threats(expected_threats) {
205 DCHECK_EQ(urls.empty(), !full_hashes.empty())
206 << "Exactly one of urls and full_hashes must be set";
209 LocalSafeBrowsingDatabaseManager::SafeBrowsingCheck::~SafeBrowsingCheck() {
212 void LocalSafeBrowsingDatabaseManager::SafeBrowsingCheck::
213 OnSafeBrowsingResult() {
214 DCHECK_CURRENTLY_ON(BrowserThread::IO);
216 DCHECK(client);
217 DCHECK_EQ(urls.size(), url_results.size());
218 DCHECK_EQ(full_hashes.size(), full_hash_results.size());
219 if (!urls.empty()) {
220 DCHECK(full_hashes.empty());
221 switch (check_type) {
222 case safe_browsing_util::MALWARE:
223 case safe_browsing_util::PHISH:
224 case safe_browsing_util::UNWANTEDURL:
225 DCHECK_EQ(1u, urls.size());
226 client->OnCheckBrowseUrlResult(urls[0], url_results[0],
227 url_metadata[0]);
228 break;
229 case safe_browsing_util::BINURL:
230 DCHECK_EQ(urls.size(), url_results.size());
231 client->OnCheckDownloadUrlResult(
232 urls, *std::max_element(url_results.begin(), url_results.end()));
233 break;
234 default:
235 NOTREACHED();
237 } else if (!full_hashes.empty()) {
238 switch (check_type) {
239 case safe_browsing_util::EXTENSIONBLACKLIST: {
240 std::set<std::string> unsafe_extension_ids;
241 for (size_t i = 0; i < full_hashes.size(); ++i) {
242 std::string extension_id =
243 safe_browsing_util::SBFullHashToString(full_hashes[i]);
244 if (full_hash_results[i] == SB_THREAT_TYPE_EXTENSION)
245 unsafe_extension_ids.insert(extension_id);
247 client->OnCheckExtensionsResult(unsafe_extension_ids);
248 break;
250 default:
251 NOTREACHED();
253 } else {
254 NOTREACHED();
258 LocalSafeBrowsingDatabaseManager::LocalSafeBrowsingDatabaseManager(
259 const scoped_refptr<SafeBrowsingService>& service)
260 : sb_service_(service),
261 database_(NULL),
262 enabled_(false),
263 enable_download_protection_(false),
264 enable_csd_whitelist_(false),
265 enable_download_whitelist_(false),
266 enable_extension_blacklist_(false),
267 enable_ip_blacklist_(false),
268 enable_unwanted_software_blacklist_(true),
269 update_in_progress_(false),
270 database_update_in_progress_(false),
271 closing_database_(false),
272 check_timeout_(base::TimeDelta::FromMilliseconds(kCheckTimeoutMs)) {
273 DCHECK_CURRENTLY_ON(BrowserThread::UI);
274 DCHECK(sb_service_.get() != NULL);
276 // Android only supports a subset of FULL_SAFE_BROWSING.
277 // TODO(shess): This shouldn't be OS-driven <http://crbug.com/394379>
278 #if !defined(OS_ANDROID)
279 base::CommandLine* cmdline = base::CommandLine::ForCurrentProcess();
280 enable_download_protection_ =
281 !cmdline->HasSwitch(switches::kSbDisableDownloadProtection);
283 // We only download the csd-whitelist if client-side phishing detection is
284 // enabled.
285 enable_csd_whitelist_ =
286 !cmdline->HasSwitch(switches::kDisableClientSidePhishingDetection);
288 // TODO(noelutz): remove this boolean variable since it should always be true
289 // if SafeBrowsing is enabled. Unfortunately, we have no test data for this
290 // list right now. This means that we need to be able to disable this list
291 // for the SafeBrowsing test to pass.
292 enable_download_whitelist_ = enable_csd_whitelist_;
294 // TODO(kalman): there really shouldn't be a flag for this.
295 enable_extension_blacklist_ =
296 !cmdline->HasSwitch(switches::kSbDisableExtensionBlacklist);
298 // The client-side IP blacklist feature is tightly integrated with client-side
299 // phishing protection for now.
300 enable_ip_blacklist_ = enable_csd_whitelist_;
301 #endif
304 LocalSafeBrowsingDatabaseManager::~LocalSafeBrowsingDatabaseManager() {
305 // The DCHECK is disabled due to crbug.com/438754.
306 // DCHECK_CURRENTLY_ON(BrowserThread::UI);
308 // We should have already been shut down. If we're still enabled, then the
309 // database isn't going to be closed properly, which could lead to corruption.
310 DCHECK(!enabled_);
313 bool LocalSafeBrowsingDatabaseManager::CanCheckUrl(const GURL& url) const {
314 return url.SchemeIs(url::kFtpScheme) ||
315 url.SchemeIs(url::kHttpScheme) ||
316 url.SchemeIs(url::kHttpsScheme);
319 bool LocalSafeBrowsingDatabaseManager::CheckDownloadUrl(
320 const std::vector<GURL>& url_chain,
321 Client* client) {
322 DCHECK_CURRENTLY_ON(BrowserThread::IO);
323 if (!enabled_ || !enable_download_protection_)
324 return true;
326 // We need to check the database for url prefix, and later may fetch the url
327 // from the safebrowsing backends. These need to be asynchronous.
328 SafeBrowsingCheck* check =
329 new SafeBrowsingCheck(url_chain,
330 std::vector<SBFullHash>(),
331 client,
332 safe_browsing_util::BINURL,
333 std::vector<SBThreatType>(1,
334 SB_THREAT_TYPE_BINARY_MALWARE_URL));
335 std::vector<SBPrefix> prefixes;
336 SafeBrowsingDatabase::GetDownloadUrlPrefixes(url_chain, &prefixes);
337 StartSafeBrowsingCheck(
338 check,
339 base::Bind(&LocalSafeBrowsingDatabaseManager::CheckDownloadUrlOnSBThread,
340 this, prefixes));
341 return false;
344 bool LocalSafeBrowsingDatabaseManager::CheckExtensionIDs(
345 const std::set<std::string>& extension_ids,
346 Client* client) {
347 DCHECK_CURRENTLY_ON(BrowserThread::IO);
349 if (!enabled_ || !enable_extension_blacklist_)
350 return true;
352 std::vector<SBFullHash> extension_id_hashes;
353 std::transform(extension_ids.begin(), extension_ids.end(),
354 std::back_inserter(extension_id_hashes),
355 safe_browsing_util::StringToSBFullHash);
356 std::vector<SBPrefix> prefixes;
357 for (const SBFullHash& hash : extension_id_hashes)
358 prefixes.push_back(hash.prefix);
360 SafeBrowsingCheck* check = new SafeBrowsingCheck(
361 std::vector<GURL>(),
362 extension_id_hashes,
363 client,
364 safe_browsing_util::EXTENSIONBLACKLIST,
365 std::vector<SBThreatType>(1, SB_THREAT_TYPE_EXTENSION));
366 StartSafeBrowsingCheck(
367 check,
368 base::Bind(&LocalSafeBrowsingDatabaseManager::CheckExtensionIDsOnSBThread,
369 this, prefixes));
370 return false;
373 bool LocalSafeBrowsingDatabaseManager::MatchMalwareIP(
374 const std::string& ip_address) {
375 DCHECK_CURRENTLY_ON(BrowserThread::IO);
376 if (!enabled_ || !enable_ip_blacklist_ || !MakeDatabaseAvailable()) {
377 return false; // Fail open.
379 return database_->ContainsMalwareIP(ip_address);
382 bool LocalSafeBrowsingDatabaseManager::MatchCsdWhitelistUrl(const GURL& url) {
383 DCHECK_CURRENTLY_ON(BrowserThread::IO);
384 if (!enabled_ || !enable_csd_whitelist_ || !MakeDatabaseAvailable()) {
385 // There is something funky going on here -- for example, perhaps the user
386 // has not restarted since enabling metrics reporting, so we haven't
387 // enabled the csd whitelist yet. Just to be safe we return true in this
388 // case.
389 return true;
391 return database_->ContainsCsdWhitelistedUrl(url);
394 bool LocalSafeBrowsingDatabaseManager::MatchDownloadWhitelistUrl(
395 const GURL& url) {
396 DCHECK_CURRENTLY_ON(BrowserThread::IO);
397 if (!enabled_ || !enable_download_whitelist_ || !MakeDatabaseAvailable()) {
398 return true;
400 return database_->ContainsDownloadWhitelistedUrl(url);
403 bool LocalSafeBrowsingDatabaseManager::MatchDownloadWhitelistString(
404 const std::string& str) {
405 DCHECK_CURRENTLY_ON(BrowserThread::IO);
406 if (!enabled_ || !enable_download_whitelist_ || !MakeDatabaseAvailable()) {
407 return true;
409 return database_->ContainsDownloadWhitelistedString(str);
412 bool LocalSafeBrowsingDatabaseManager::MatchInclusionWhitelistUrl(
413 const GURL& url) {
414 DCHECK_CURRENTLY_ON(BrowserThread::IO);
415 if (!enabled_ || !MakeDatabaseAvailable())
416 return true;
417 return database_->ContainsInclusionWhitelistedUrl(url);
420 bool LocalSafeBrowsingDatabaseManager::IsMalwareKillSwitchOn() {
421 DCHECK_CURRENTLY_ON(BrowserThread::IO);
422 if (!enabled_ || !MakeDatabaseAvailable()) {
423 return true;
425 return database_->IsMalwareIPMatchKillSwitchOn();
428 bool LocalSafeBrowsingDatabaseManager::IsCsdWhitelistKillSwitchOn() {
429 DCHECK_CURRENTLY_ON(BrowserThread::IO);
430 if (!enabled_ || !MakeDatabaseAvailable()) {
431 return true;
433 return database_->IsCsdWhitelistKillSwitchOn();
436 bool LocalSafeBrowsingDatabaseManager::CheckBrowseUrl(const GURL& url,
437 Client* client) {
438 DCHECK_CURRENTLY_ON(BrowserThread::IO);
439 if (!enabled_)
440 return true;
442 if (!CanCheckUrl(url))
443 return true;
445 std::vector<SBThreatType> expected_threats;
446 expected_threats.push_back(SB_THREAT_TYPE_URL_MALWARE);
447 expected_threats.push_back(SB_THREAT_TYPE_URL_PHISHING);
448 expected_threats.push_back(SB_THREAT_TYPE_URL_UNWANTED);
450 const base::TimeTicks start = base::TimeTicks::Now();
451 if (!MakeDatabaseAvailable()) {
452 QueuedCheck queued_check(safe_browsing_util::MALWARE, // or PHISH
453 client,
454 url,
455 expected_threats,
456 start);
457 queued_checks_.push_back(queued_check);
458 return false;
461 // Cache hits should, in general, be the same for both (ignoring potential
462 // cache evictions in the second call for entries that were just about to be
463 // evicted in the first call).
464 // TODO(gab): Refactor SafeBrowsingDatabase to avoid depending on this here.
465 std::vector<SBFullHashResult> cache_hits;
467 std::vector<SBPrefix> browse_prefix_hits;
468 bool browse_prefix_match = database_->ContainsBrowseUrl(
469 url, &browse_prefix_hits, &cache_hits);
471 std::vector<SBPrefix> unwanted_prefix_hits;
472 std::vector<SBFullHashResult> unused_cache_hits;
473 bool unwanted_prefix_match = database_->ContainsUnwantedSoftwareUrl(
474 url, &unwanted_prefix_hits, &unused_cache_hits);
476 // Merge the two pre-sorted prefix hits lists.
477 // TODO(gab): Refactor SafeBrowsingDatabase for it to return this merged list
478 // by default rather than building it here.
479 std::vector<SBPrefix> prefix_hits(browse_prefix_hits.size() +
480 unwanted_prefix_hits.size());
481 std::merge(browse_prefix_hits.begin(),
482 browse_prefix_hits.end(),
483 unwanted_prefix_hits.begin(),
484 unwanted_prefix_hits.end(),
485 prefix_hits.begin());
486 prefix_hits.erase(std::unique(prefix_hits.begin(), prefix_hits.end()),
487 prefix_hits.end());
489 UMA_HISTOGRAM_TIMES("SB2.FilterCheck", base::TimeTicks::Now() - start);
491 if (!browse_prefix_match && !unwanted_prefix_match)
492 return true; // URL is okay.
494 // Needs to be asynchronous, since we could be in the constructor of a
495 // ResourceDispatcherHost event handler which can't pause there.
496 // This check will ping the Safe Browsing servers and get all lists which it
497 // matches. These lists will then be filtered against the |expected_threats|
498 // and the result callback for MALWARE (which is the same as for PHISH and
499 // UNWANTEDURL) will eventually be invoked with the final decision.
500 SafeBrowsingCheck* check = new SafeBrowsingCheck(std::vector<GURL>(1, url),
501 std::vector<SBFullHash>(),
502 client,
503 safe_browsing_util::MALWARE,
504 expected_threats);
505 check->need_get_hash = cache_hits.empty();
506 check->prefix_hits.swap(prefix_hits);
507 check->cache_hits.swap(cache_hits);
508 checks_.insert(check);
510 BrowserThread::PostTask(
511 BrowserThread::IO, FROM_HERE,
512 base::Bind(&LocalSafeBrowsingDatabaseManager::OnCheckDone, this, check));
514 return false;
517 void LocalSafeBrowsingDatabaseManager::CancelCheck(Client* client) {
518 DCHECK_CURRENTLY_ON(BrowserThread::IO);
519 for (CurrentChecks::iterator i = checks_.begin(); i != checks_.end(); ++i) {
520 // We can't delete matching checks here because the db thread has a copy of
521 // the pointer. Instead, we simply NULL out the client, and when the db
522 // thread calls us back, we'll clean up the check.
523 if ((*i)->client == client)
524 (*i)->client = NULL;
527 // Scan the queued clients store. Clients may be here if they requested a URL
528 // check before the database has finished loading.
529 for (std::deque<QueuedCheck>::iterator it(queued_checks_.begin());
530 it != queued_checks_.end(); ) {
531 // In this case it's safe to delete matches entirely since nothing has a
532 // pointer to them.
533 if (it->client == client)
534 it = queued_checks_.erase(it);
535 else
536 ++it;
540 void LocalSafeBrowsingDatabaseManager::HandleGetHashResults(
541 SafeBrowsingCheck* check,
542 const std::vector<SBFullHashResult>& full_hashes,
543 const base::TimeDelta& cache_lifetime) {
544 DCHECK_CURRENTLY_ON(BrowserThread::IO);
546 if (!enabled_)
547 return;
549 // If the service has been shut down, |check| should have been deleted.
550 DCHECK(checks_.find(check) != checks_.end());
552 // |start| is set before calling |GetFullHash()|, which should be
553 // the only path which gets to here.
554 DCHECK(!check->start.is_null());
555 UMA_HISTOGRAM_LONG_TIMES("SB2.Network",
556 base::TimeTicks::Now() - check->start);
558 std::vector<SBPrefix> prefixes = check->prefix_hits;
559 OnHandleGetHashResults(check, full_hashes); // 'check' is deleted here.
561 // Cache the GetHash results.
562 if (cache_lifetime != base::TimeDelta() && MakeDatabaseAvailable())
563 database_->CacheHashResults(prefixes, full_hashes, cache_lifetime);
566 void LocalSafeBrowsingDatabaseManager::GetChunks(GetChunksCallback callback) {
567 DCHECK_CURRENTLY_ON(BrowserThread::IO);
568 DCHECK(enabled_);
569 DCHECK(!callback.is_null());
570 safe_browsing_task_runner_->PostTask(
571 FROM_HERE,
572 base::Bind(&LocalSafeBrowsingDatabaseManager::GetAllChunksFromDatabase,
573 this, callback));
576 void LocalSafeBrowsingDatabaseManager::AddChunks(
577 const std::string& list,
578 scoped_ptr<ScopedVector<SBChunkData> > chunks,
579 AddChunksCallback callback) {
580 DCHECK_CURRENTLY_ON(BrowserThread::IO);
581 DCHECK(enabled_);
582 DCHECK(!callback.is_null());
583 safe_browsing_task_runner_->PostTask(
584 FROM_HERE,
585 base::Bind(&LocalSafeBrowsingDatabaseManager::AddDatabaseChunks, this,
586 list, base::Passed(&chunks), callback));
589 void LocalSafeBrowsingDatabaseManager::DeleteChunks(
590 scoped_ptr<std::vector<SBChunkDelete> > chunk_deletes) {
591 DCHECK_CURRENTLY_ON(BrowserThread::IO);
592 DCHECK(enabled_);
593 safe_browsing_task_runner_->PostTask(
594 FROM_HERE,
595 base::Bind(&LocalSafeBrowsingDatabaseManager::DeleteDatabaseChunks, this,
596 base::Passed(&chunk_deletes)));
599 void LocalSafeBrowsingDatabaseManager::UpdateStarted() {
600 DCHECK_CURRENTLY_ON(BrowserThread::IO);
601 DCHECK(enabled_);
602 DCHECK(!update_in_progress_);
603 update_in_progress_ = true;
606 void LocalSafeBrowsingDatabaseManager::UpdateFinished(bool update_succeeded) {
607 DCHECK_CURRENTLY_ON(BrowserThread::IO);
608 DCHECK(enabled_);
609 if (update_in_progress_) {
610 update_in_progress_ = false;
611 safe_browsing_task_runner_->PostTask(
612 FROM_HERE,
613 base::Bind(&LocalSafeBrowsingDatabaseManager::DatabaseUpdateFinished,
614 this, update_succeeded));
618 void LocalSafeBrowsingDatabaseManager::ResetDatabase() {
619 DCHECK_CURRENTLY_ON(BrowserThread::IO);
620 DCHECK(enabled_);
621 safe_browsing_task_runner_->PostTask(
622 FROM_HERE,
623 base::Bind(&LocalSafeBrowsingDatabaseManager::OnResetDatabase, this));
626 void LocalSafeBrowsingDatabaseManager::StartOnIOThread() {
627 DCHECK_CURRENTLY_ON(BrowserThread::IO);
628 if (enabled_)
629 return;
631 // Only get a new task runner if there isn't one already. If the service has
632 // previously been started and stopped, a task runner could already exist.
633 if (!safe_browsing_task_runner_) {
634 base::SequencedWorkerPool* pool = BrowserThread::GetBlockingPool();
635 safe_browsing_task_runner_ =
636 pool->GetSequencedTaskRunnerWithShutdownBehavior(
637 pool->GetSequenceToken(),
638 base::SequencedWorkerPool::SKIP_ON_SHUTDOWN);
641 enabled_ = true;
643 MakeDatabaseAvailable();
646 void LocalSafeBrowsingDatabaseManager::StopOnIOThread(bool shutdown) {
647 DCHECK_CURRENTLY_ON(BrowserThread::IO);
649 DoStopOnIOThread();
650 if (shutdown) {
651 sb_service_ = NULL;
655 void LocalSafeBrowsingDatabaseManager::NotifyDatabaseUpdateFinished(
656 bool update_succeeded) {
657 DCHECK_CURRENTLY_ON(BrowserThread::UI);
658 content::NotificationService::current()->Notify(
659 chrome::NOTIFICATION_SAFE_BROWSING_UPDATE_COMPLETE,
660 content::Source<SafeBrowsingDatabaseManager>(this),
661 content::Details<bool>(&update_succeeded));
664 LocalSafeBrowsingDatabaseManager::QueuedCheck::QueuedCheck(
665 const safe_browsing_util::ListType check_type,
666 Client* client,
667 const GURL& url,
668 const std::vector<SBThreatType>& expected_threats,
669 const base::TimeTicks& start)
670 : check_type(check_type),
671 client(client),
672 url(url),
673 expected_threats(expected_threats),
674 start(start) {
677 LocalSafeBrowsingDatabaseManager::QueuedCheck::~QueuedCheck() {
680 void LocalSafeBrowsingDatabaseManager::DoStopOnIOThread() {
681 DCHECK_CURRENTLY_ON(BrowserThread::IO);
683 if (!enabled_)
684 return;
686 enabled_ = false;
688 // Delete queued checks, calling back any clients with 'SB_THREAT_TYPE_SAFE'.
689 while (!queued_checks_.empty()) {
690 QueuedCheck queued = queued_checks_.front();
691 if (queued.client) {
692 SafeBrowsingCheck sb_check(std::vector<GURL>(1, queued.url),
693 std::vector<SBFullHash>(),
694 queued.client,
695 queued.check_type,
696 queued.expected_threats);
697 sb_check.OnSafeBrowsingResult();
699 queued_checks_.pop_front();
702 // Close the database. Cases to avoid:
703 // * If |closing_database_| is true, continuing will queue up a second
704 // request, |closing_database_| will be reset after handling the first
705 // request, and if any functions on the db thread recreate the database, we
706 // could start using it on the IO thread and then have the second request
707 // handler delete it out from under us.
708 // * If |database_| is NULL, then either no creation request is in flight, in
709 // which case we don't need to do anything, or one is in flight, in which
710 // case the database will be recreated before our deletion request is
711 // handled, and could be used on the IO thread in that time period, leading
712 // to the same problem as above.
713 // Checking DatabaseAvailable() avoids both of these.
714 if (DatabaseAvailable()) {
715 closing_database_ = true;
716 safe_browsing_task_runner_->PostTask(
717 FROM_HERE,
718 base::Bind(&LocalSafeBrowsingDatabaseManager::OnCloseDatabase, this));
721 // Delete pending checks, calling back any clients with 'SB_THREAT_TYPE_SAFE'.
722 // We have to do this after the db thread returns because methods on it can
723 // have copies of these pointers, so deleting them might lead to accessing
724 // garbage.
725 for (CurrentChecks::iterator it = checks_.begin();
726 it != checks_.end(); ++it) {
727 SafeBrowsingCheck* check = *it;
728 if (check->client)
729 check->OnSafeBrowsingResult();
731 STLDeleteElements(&checks_);
733 gethash_requests_.clear();
736 bool LocalSafeBrowsingDatabaseManager::DatabaseAvailable() const {
737 base::AutoLock lock(database_lock_);
738 return !closing_database_ && (database_ != NULL);
741 bool LocalSafeBrowsingDatabaseManager::MakeDatabaseAvailable() {
742 DCHECK_CURRENTLY_ON(BrowserThread::IO);
743 DCHECK(enabled_);
744 if (DatabaseAvailable())
745 return true;
746 safe_browsing_task_runner_->PostTask(
747 FROM_HERE, base::Bind(base::IgnoreResult(
748 &LocalSafeBrowsingDatabaseManager::GetDatabase),
749 this));
750 return false;
753 SafeBrowsingDatabase* LocalSafeBrowsingDatabaseManager::GetDatabase() {
754 DCHECK(safe_browsing_task_runner_->RunsTasksOnCurrentThread());
756 if (database_)
757 return database_;
759 const base::TimeTicks before = base::TimeTicks::Now();
760 SafeBrowsingDatabase* database = SafeBrowsingDatabase::Create(
761 safe_browsing_task_runner_, enable_download_protection_,
762 enable_csd_whitelist_, enable_download_whitelist_,
763 enable_extension_blacklist_, enable_ip_blacklist_,
764 enable_unwanted_software_blacklist_);
766 database->Init(SafeBrowsingService::GetBaseFilename());
768 // Acquiring the lock here guarantees correct ordering between the writes to
769 // the new database object above, and the setting of |database_| below.
770 base::AutoLock lock(database_lock_);
771 database_ = database;
774 BrowserThread::PostTask(
775 BrowserThread::IO, FROM_HERE,
776 base::Bind(&LocalSafeBrowsingDatabaseManager::DatabaseLoadComplete,
777 this));
779 UMA_HISTOGRAM_TIMES("SB2.DatabaseOpen", base::TimeTicks::Now() - before);
780 return database_;
783 void LocalSafeBrowsingDatabaseManager::OnCheckDone(SafeBrowsingCheck* check) {
784 DCHECK_CURRENTLY_ON(BrowserThread::IO);
786 if (!enabled_)
787 return;
789 // If the service has been shut down, |check| should have been deleted.
790 DCHECK(checks_.find(check) != checks_.end());
792 if (check->client && check->need_get_hash) {
793 // We have a partial match so we need to query Google for the full hash.
794 // Clean up will happen in HandleGetHashResults.
796 // See if we have a GetHash request already in progress for this particular
797 // prefix. If so, we just append ourselves to the list of interested parties
798 // when the results arrive. We only do this for checks involving one prefix,
799 // since that is the common case (multiple prefixes will issue the request
800 // as normal).
801 if (check->prefix_hits.size() == 1) {
802 SBPrefix prefix = check->prefix_hits[0];
803 GetHashRequests::iterator it = gethash_requests_.find(prefix);
804 if (it != gethash_requests_.end()) {
805 // There's already a request in progress.
806 it->second.push_back(check);
807 return;
810 // No request in progress, so we're the first for this prefix.
811 GetHashRequestors requestors;
812 requestors.push_back(check);
813 gethash_requests_[prefix] = requestors;
816 // Reset the start time so that we can measure the network time without the
817 // database time.
818 check->start = base::TimeTicks::Now();
820 BrowserThread::PostTask(
821 BrowserThread::UI, FROM_HERE,
822 base::Bind(&LocalSafeBrowsingDatabaseManager::OnRequestFullHash, this,
823 check));
824 } else {
825 // We may have cached results for previous GetHash queries. Since
826 // this data comes from cache, don't histogram hits.
827 HandleOneCheck(check, check->cache_hits);
831 void LocalSafeBrowsingDatabaseManager::OnRequestFullHash(
832 SafeBrowsingCheck* check) {
833 DCHECK_CURRENTLY_ON(BrowserThread::UI);
834 check->is_extended_reporting = GetExtendedReporting();
835 BrowserThread::PostTask(
836 BrowserThread::IO, FROM_HERE,
837 base::Bind(&LocalSafeBrowsingDatabaseManager::RequestFullHash, this,
838 check));
841 bool LocalSafeBrowsingDatabaseManager::GetExtendedReporting() {
842 DCHECK_CURRENTLY_ON(BrowserThread::UI);
844 // Determine if the last used profile is opted into extended reporting.
845 // Note: It is possible that the last used profile is not the one triggers
846 // the hash request, but not very likely.
847 bool is_extended_reporting = false;
848 ProfileManager* profile_manager = g_browser_process->profile_manager();
849 if (profile_manager) {
850 Profile* profile = profile_manager->GetLastUsedProfile();
851 is_extended_reporting = profile &&
852 profile->GetPrefs()->GetBoolean(
853 prefs::kSafeBrowsingExtendedReportingEnabled);
855 return is_extended_reporting;
858 void LocalSafeBrowsingDatabaseManager::RequestFullHash(
859 SafeBrowsingCheck* check) {
860 DCHECK_CURRENTLY_ON(BrowserThread::IO);
861 bool is_download = check->check_type == safe_browsing_util::BINURL;
862 sb_service_->protocol_manager()->GetFullHash(
863 check->prefix_hits,
864 base::Bind(&LocalSafeBrowsingDatabaseManager::HandleGetHashResults,
865 base::Unretained(this), check),
866 is_download, check->is_extended_reporting);
869 void LocalSafeBrowsingDatabaseManager::GetAllChunksFromDatabase(
870 GetChunksCallback callback) {
871 DCHECK(safe_browsing_task_runner_->RunsTasksOnCurrentThread());
873 bool database_error = true;
874 std::vector<SBListChunkRanges> lists;
875 DCHECK(!database_update_in_progress_);
876 database_update_in_progress_ = true;
877 GetDatabase(); // This guarantees that |database_| is non-NULL.
878 if (database_->UpdateStarted(&lists)) {
879 database_error = false;
880 } else {
881 database_->UpdateFinished(false);
884 BrowserThread::PostTask(
885 BrowserThread::UI, FROM_HERE,
886 base::Bind(
887 &LocalSafeBrowsingDatabaseManager::BeforeGetAllChunksFromDatabase,
888 this, lists, database_error, callback));
891 void LocalSafeBrowsingDatabaseManager::BeforeGetAllChunksFromDatabase(
892 const std::vector<SBListChunkRanges>& lists,
893 bool database_error,
894 GetChunksCallback callback) {
895 DCHECK_CURRENTLY_ON(BrowserThread::UI);
897 bool is_extended_reporting = GetExtendedReporting();
899 BrowserThread::PostTask(
900 BrowserThread::IO, FROM_HERE,
901 base::Bind(&LocalSafeBrowsingDatabaseManager::OnGetAllChunksFromDatabase,
902 this, lists, database_error, is_extended_reporting, callback));
905 void LocalSafeBrowsingDatabaseManager::OnGetAllChunksFromDatabase(
906 const std::vector<SBListChunkRanges>& lists,
907 bool database_error,
908 bool is_extended_reporting,
909 GetChunksCallback callback) {
910 DCHECK_CURRENTLY_ON(BrowserThread::IO);
911 if (enabled_)
912 callback.Run(lists, database_error, is_extended_reporting);
915 void LocalSafeBrowsingDatabaseManager::OnAddChunksComplete(
916 AddChunksCallback callback) {
917 DCHECK_CURRENTLY_ON(BrowserThread::IO);
918 if (enabled_)
919 callback.Run();
922 void LocalSafeBrowsingDatabaseManager::DatabaseLoadComplete() {
923 DCHECK_CURRENTLY_ON(BrowserThread::IO);
924 if (!enabled_)
925 return;
927 LOCAL_HISTOGRAM_COUNTS("SB.QueueDepth", queued_checks_.size());
928 if (queued_checks_.empty())
929 return;
931 // If the database isn't already available, calling CheckUrl() in the loop
932 // below will add the check back to the queue, and we'll infinite-loop.
933 DCHECK(DatabaseAvailable());
934 while (!queued_checks_.empty()) {
935 QueuedCheck check = queued_checks_.front();
936 DCHECK(!check.start.is_null());
937 LOCAL_HISTOGRAM_TIMES("SB.QueueDelay",
938 base::TimeTicks::Now() - check.start);
939 // If CheckUrl() determines the URL is safe immediately, it doesn't call the
940 // client's handler function (because normally it's being directly called by
941 // the client). Since we're not the client, we have to convey this result.
942 if (check.client && CheckBrowseUrl(check.url, check.client)) {
943 SafeBrowsingCheck sb_check(std::vector<GURL>(1, check.url),
944 std::vector<SBFullHash>(),
945 check.client,
946 check.check_type,
947 check.expected_threats);
948 sb_check.OnSafeBrowsingResult();
950 queued_checks_.pop_front();
954 void LocalSafeBrowsingDatabaseManager::AddDatabaseChunks(
955 const std::string& list_name,
956 scoped_ptr<ScopedVector<SBChunkData> > chunks,
957 AddChunksCallback callback) {
958 DCHECK(safe_browsing_task_runner_->RunsTasksOnCurrentThread());
959 if (chunks)
960 GetDatabase()->InsertChunks(list_name, chunks->get());
961 BrowserThread::PostTask(
962 BrowserThread::IO, FROM_HERE,
963 base::Bind(&LocalSafeBrowsingDatabaseManager::OnAddChunksComplete, this,
964 callback));
967 void LocalSafeBrowsingDatabaseManager::DeleteDatabaseChunks(
968 scoped_ptr<std::vector<SBChunkDelete> > chunk_deletes) {
969 DCHECK(safe_browsing_task_runner_->RunsTasksOnCurrentThread());
970 if (chunk_deletes)
971 GetDatabase()->DeleteChunks(*chunk_deletes);
974 void LocalSafeBrowsingDatabaseManager::DatabaseUpdateFinished(
975 bool update_succeeded) {
976 DCHECK(safe_browsing_task_runner_->RunsTasksOnCurrentThread());
977 GetDatabase()->UpdateFinished(update_succeeded);
978 DCHECK(database_update_in_progress_);
979 database_update_in_progress_ = false;
980 BrowserThread::PostTask(
981 BrowserThread::UI, FROM_HERE,
982 base::Bind(
983 &LocalSafeBrowsingDatabaseManager::NotifyDatabaseUpdateFinished, this,
984 update_succeeded));
987 void LocalSafeBrowsingDatabaseManager::OnCloseDatabase() {
988 DCHECK(safe_browsing_task_runner_->RunsTasksOnCurrentThread());
989 DCHECK(closing_database_);
991 // Because |closing_database_| is true, nothing on the IO thread will be
992 // accessing the database, so it's safe to delete and then NULL the pointer.
993 delete database_;
994 database_ = NULL;
996 // Acquiring the lock here guarantees correct ordering between the resetting
997 // of |database_| above and of |closing_database_| below, which ensures there
998 // won't be a window during which the IO thread falsely believes the database
999 // is available.
1000 base::AutoLock lock(database_lock_);
1001 closing_database_ = false;
1004 void LocalSafeBrowsingDatabaseManager::OnResetDatabase() {
1005 DCHECK(safe_browsing_task_runner_->RunsTasksOnCurrentThread());
1007 GetDatabase()->ResetDatabase();
1010 void LocalSafeBrowsingDatabaseManager::OnHandleGetHashResults(
1011 SafeBrowsingCheck* check,
1012 const std::vector<SBFullHashResult>& full_hashes) {
1013 DCHECK_CURRENTLY_ON(BrowserThread::IO);
1014 safe_browsing_util::ListType check_type = check->check_type;
1015 SBPrefix prefix = check->prefix_hits[0];
1016 GetHashRequests::iterator it = gethash_requests_.find(prefix);
1017 if (check->prefix_hits.size() > 1 || it == gethash_requests_.end()) {
1018 const bool hit = HandleOneCheck(check, full_hashes);
1019 RecordGetHashCheckStatus(hit, check_type, full_hashes);
1020 return;
1023 // Call back all interested parties, noting if any has a hit.
1024 GetHashRequestors& requestors = it->second;
1025 bool hit = false;
1026 for (GetHashRequestors::iterator r = requestors.begin();
1027 r != requestors.end(); ++r) {
1028 if (HandleOneCheck(*r, full_hashes))
1029 hit = true;
1031 RecordGetHashCheckStatus(hit, check_type, full_hashes);
1033 gethash_requests_.erase(it);
1036 bool LocalSafeBrowsingDatabaseManager::HandleOneCheck(
1037 SafeBrowsingCheck* check,
1038 const std::vector<SBFullHashResult>& full_hashes) {
1039 DCHECK_CURRENTLY_ON(BrowserThread::IO);
1040 DCHECK(check);
1042 bool is_threat = false;
1044 // TODO(shess): GetHashSeverestThreadListType() contains a loop,
1045 // GetUrlSeverestThreatListType() a loop around that loop. Having another
1046 // loop out here concerns me. It is likely that SAFE is an expected outcome,
1047 // which means all of those loops run to completion. Refactoring this to
1048 // generate a set of sorted items to compare in sequence would probably
1049 // improve things.
1051 // Additionally, the set of patterns generated from the urls is very similar
1052 // to the patterns generated in ContainsBrowseUrl() and other database checks,
1053 // which are called from this code. Refactoring that across the checks could
1054 // interact well with batching the checks here.
1056 // TODO(gab): Fix the fact that Get(Url|Hash)SeverestThreatType() may return a
1057 // threat for which IsExpectedThreat() returns false even if |full_hashes|
1058 // actually contains an expected threat.
1060 for (size_t i = 0; i < check->urls.size(); ++i) {
1061 size_t threat_index;
1062 SBThreatType threat =
1063 GetUrlSeverestThreatType(check->urls[i], full_hashes, &threat_index);
1064 if (threat != SB_THREAT_TYPE_SAFE &&
1065 IsExpectedThreat(threat, check->expected_threats)) {
1066 check->url_results[i] = threat;
1067 check->url_metadata[i] = full_hashes[threat_index].metadata;
1068 is_threat = true;
1072 for (size_t i = 0; i < check->full_hashes.size(); ++i) {
1073 SBThreatType threat =
1074 GetHashSeverestThreatType(check->full_hashes[i], full_hashes);
1075 if (threat != SB_THREAT_TYPE_SAFE &&
1076 IsExpectedThreat(threat, check->expected_threats)) {
1077 check->full_hash_results[i] = threat;
1078 is_threat = true;
1082 SafeBrowsingCheckDone(check);
1083 return is_threat;
1086 void LocalSafeBrowsingDatabaseManager::OnAsyncCheckDone(
1087 SafeBrowsingCheck* check,
1088 const std::vector<SBPrefix>& prefix_hits) {
1089 DCHECK_CURRENTLY_ON(BrowserThread::IO);
1090 DCHECK(enable_download_protection_);
1092 check->prefix_hits = prefix_hits;
1093 if (check->prefix_hits.empty()) {
1094 SafeBrowsingCheckDone(check);
1095 } else {
1096 check->need_get_hash = true;
1097 OnCheckDone(check);
1101 std::vector<SBPrefix>
1102 LocalSafeBrowsingDatabaseManager::CheckDownloadUrlOnSBThread(
1103 const std::vector<SBPrefix>& prefixes) {
1104 DCHECK(safe_browsing_task_runner_->RunsTasksOnCurrentThread());
1105 DCHECK(enable_download_protection_);
1107 std::vector<SBPrefix> prefix_hits;
1108 const bool result =
1109 database_->ContainsDownloadUrlPrefixes(prefixes, &prefix_hits);
1110 DCHECK_EQ(result, !prefix_hits.empty());
1111 return prefix_hits;
1114 std::vector<SBPrefix>
1115 LocalSafeBrowsingDatabaseManager::CheckExtensionIDsOnSBThread(
1116 const std::vector<SBPrefix>& prefixes) {
1117 DCHECK(safe_browsing_task_runner_->RunsTasksOnCurrentThread());
1119 std::vector<SBPrefix> prefix_hits;
1120 const bool result =
1121 database_->ContainsExtensionPrefixes(prefixes, &prefix_hits);
1122 DCHECK_EQ(result, !prefix_hits.empty());
1123 return prefix_hits;
1126 void LocalSafeBrowsingDatabaseManager::TimeoutCallback(
1127 SafeBrowsingCheck* check) {
1128 DCHECK_CURRENTLY_ON(BrowserThread::IO);
1129 DCHECK(check);
1131 if (!enabled_)
1132 return;
1134 DCHECK(checks_.find(check) != checks_.end());
1135 if (check->client) {
1136 check->OnSafeBrowsingResult();
1137 check->client = NULL;
1141 void LocalSafeBrowsingDatabaseManager::SafeBrowsingCheckDone(
1142 SafeBrowsingCheck* check) {
1143 DCHECK_CURRENTLY_ON(BrowserThread::IO);
1144 DCHECK(check);
1146 if (!enabled_)
1147 return;
1149 DVLOG(1) << "SafeBrowsingCheckDone";
1150 DCHECK(checks_.find(check) != checks_.end());
1151 if (check->client)
1152 check->OnSafeBrowsingResult();
1153 checks_.erase(check);
1154 delete check;
1157 void LocalSafeBrowsingDatabaseManager::StartSafeBrowsingCheck(
1158 SafeBrowsingCheck* check,
1159 const base::Callback<std::vector<SBPrefix>(void)>& task) {
1160 DCHECK_CURRENTLY_ON(BrowserThread::IO);
1161 check->weak_ptr_factory_.reset(
1162 new base::WeakPtrFactory<LocalSafeBrowsingDatabaseManager>(this));
1163 checks_.insert(check);
1165 base::PostTaskAndReplyWithResult(
1166 safe_browsing_task_runner_.get(), FROM_HERE, task,
1167 base::Bind(&LocalSafeBrowsingDatabaseManager::OnAsyncCheckDone,
1168 check->weak_ptr_factory_->GetWeakPtr(), check));
1169 base::ThreadTaskRunnerHandle::Get()->PostDelayedTask(
1170 FROM_HERE, base::Bind(&LocalSafeBrowsingDatabaseManager::TimeoutCallback,
1171 check->weak_ptr_factory_->GetWeakPtr(), check),
1172 check_timeout_);
1175 bool LocalSafeBrowsingDatabaseManager::download_protection_enabled() const {
1176 return enable_download_protection_;