Add domain request detection to incident reporting service.
[chromium-blink-merge.git] / chrome / browser / safe_browsing / database_manager.cc
blob27624196dd62bea57838577f79ff21da92e489a8
1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "chrome/browser/safe_browsing/database_manager.h"
7 #include <algorithm>
9 #include "base/bind.h"
10 #include "base/bind_helpers.h"
11 #include "base/callback.h"
12 #include "base/command_line.h"
13 #include "base/debug/leak_tracker.h"
14 #include "base/metrics/histogram_macros.h"
15 #include "base/profiler/scoped_tracker.h"
16 #include "base/stl_util.h"
17 #include "base/strings/string_util.h"
18 #include "chrome/browser/browser_process.h"
19 #include "chrome/browser/chrome_notification_types.h"
20 #include "chrome/browser/prerender/prerender_field_trial.h"
21 #include "chrome/browser/safe_browsing/client_side_detection_service.h"
22 #include "chrome/browser/safe_browsing/download_protection_service.h"
23 #include "chrome/browser/safe_browsing/malware_details.h"
24 #include "chrome/browser/safe_browsing/protocol_manager.h"
25 #include "chrome/browser/safe_browsing/safe_browsing_database.h"
26 #include "chrome/browser/safe_browsing/safe_browsing_service.h"
27 #include "chrome/browser/safe_browsing/ui_manager.h"
28 #include "chrome/common/chrome_constants.h"
29 #include "chrome/common/chrome_paths.h"
30 #include "chrome/common/chrome_switches.h"
31 #include "components/startup_metric_utils/startup_metric_utils.h"
32 #include "content/public/browser/browser_thread.h"
33 #include "content/public/browser/notification_service.h"
34 #include "url/url_constants.h"
36 using content::BrowserThread;
38 namespace {
40 // Timeout for match checks, e.g. download URLs, hashes.
41 const int kCheckTimeoutMs = 10000;
43 // Records disposition information about the check. |hit| should be
44 // |true| if there were any prefix hits in |full_hashes|.
45 void RecordGetHashCheckStatus(
46 bool hit,
47 safe_browsing_util::ListType check_type,
48 const std::vector<SBFullHashResult>& full_hashes) {
49 SafeBrowsingProtocolManager::ResultType result;
50 if (full_hashes.empty()) {
51 result = SafeBrowsingProtocolManager::GET_HASH_FULL_HASH_EMPTY;
52 } else if (hit) {
53 result = SafeBrowsingProtocolManager::GET_HASH_FULL_HASH_HIT;
54 } else {
55 result = SafeBrowsingProtocolManager::GET_HASH_FULL_HASH_MISS;
57 bool is_download = check_type == safe_browsing_util::BINURL;
58 SafeBrowsingProtocolManager::RecordGetHashResult(is_download, result);
61 bool IsExpectedThreat(
62 const SBThreatType threat_type,
63 const std::vector<SBThreatType>& expected_threats) {
64 return expected_threats.end() != std::find(expected_threats.begin(),
65 expected_threats.end(),
66 threat_type);
69 // Return the severest list id from the results in |full_hashes| which matches
70 // |hash|, or INVALID if none match.
71 safe_browsing_util::ListType GetHashSeverestThreatListType(
72 const SBFullHash& hash,
73 const std::vector<SBFullHashResult>& full_hashes,
74 size_t* index) {
75 safe_browsing_util::ListType pending_threat = safe_browsing_util::INVALID;
76 for (size_t i = 0; i < full_hashes.size(); ++i) {
77 if (SBFullHashEqual(hash, full_hashes[i].hash)) {
78 const safe_browsing_util::ListType threat =
79 static_cast<safe_browsing_util::ListType>(full_hashes[i].list_id);
80 switch (threat) {
81 case safe_browsing_util::INVALID:
82 // |full_hashes| should never contain INVALID as a |list_id|.
83 NOTREACHED();
84 break;
85 case safe_browsing_util::MALWARE: // Falls through.
86 case safe_browsing_util::PHISH: // Falls through.
87 case safe_browsing_util::BINURL: // Falls through.
88 case safe_browsing_util::CSDWHITELIST: // Falls through.
89 case safe_browsing_util::DOWNLOADWHITELIST: // Falls through.
90 case safe_browsing_util::INCLUSIONWHITELIST: // Falls through.
91 case safe_browsing_util::EXTENSIONBLACKLIST: // Falls through.
92 case safe_browsing_util::SIDEEFFECTFREEWHITELIST: // Falls through.
93 case safe_browsing_util::IPBLACKLIST:
94 if (index)
95 *index = i;
96 return threat;
97 case safe_browsing_util::UNWANTEDURL:
98 // UNWANTEDURL is considered less severe than other threats, keep
99 // looking.
100 pending_threat = threat;
101 if (index)
102 *index = i;
103 break;
107 return pending_threat;
110 // Given a URL, compare all the possible host + path full hashes to the set of
111 // provided full hashes. Returns the list id of the severest matching result
112 // from |full_hashes|, or INVALID if none match.
113 safe_browsing_util::ListType GetUrlSeverestThreatListType(
114 const GURL& url,
115 const std::vector<SBFullHashResult>& full_hashes,
116 size_t* index) {
117 if (full_hashes.empty())
118 return safe_browsing_util::INVALID;
120 std::vector<std::string> patterns;
121 safe_browsing_util::GeneratePatternsToCheck(url, &patterns);
123 safe_browsing_util::ListType pending_threat = safe_browsing_util::INVALID;
124 for (size_t i = 0; i < patterns.size(); ++i) {
125 safe_browsing_util::ListType threat = GetHashSeverestThreatListType(
126 SBFullHashForString(patterns[i]), full_hashes, index);
127 switch (threat) {
128 case safe_browsing_util::INVALID:
129 // Ignore patterns with no matching threat.
130 break;
131 case safe_browsing_util::MALWARE: // Falls through.
132 case safe_browsing_util::PHISH: // Falls through.
133 case safe_browsing_util::BINURL: // Falls through.
134 case safe_browsing_util::CSDWHITELIST: // Falls through.
135 case safe_browsing_util::DOWNLOADWHITELIST: // Falls through.
136 case safe_browsing_util::INCLUSIONWHITELIST: // Falls through.
137 case safe_browsing_util::EXTENSIONBLACKLIST: // Falls through.
138 case safe_browsing_util::SIDEEFFECTFREEWHITELIST: // Falls through.
139 case safe_browsing_util::IPBLACKLIST:
140 return threat;
141 case safe_browsing_util::UNWANTEDURL:
142 // UNWANTEDURL is considered less severe than other threats, keep
143 // looking.
144 pending_threat = threat;
145 break;
148 return pending_threat;
151 SBThreatType GetThreatTypeFromListType(safe_browsing_util::ListType list_type) {
152 switch (list_type) {
153 case safe_browsing_util::PHISH:
154 return SB_THREAT_TYPE_URL_PHISHING;
155 case safe_browsing_util::MALWARE:
156 return SB_THREAT_TYPE_URL_MALWARE;
157 case safe_browsing_util::UNWANTEDURL:
158 return SB_THREAT_TYPE_URL_UNWANTED;
159 case safe_browsing_util::BINURL:
160 return SB_THREAT_TYPE_BINARY_MALWARE_URL;
161 case safe_browsing_util::EXTENSIONBLACKLIST:
162 return SB_THREAT_TYPE_EXTENSION;
163 default:
164 DVLOG(1) << "Unknown safe browsing list id " << list_type;
165 return SB_THREAT_TYPE_SAFE;
169 } // namespace
171 // static
172 SBThreatType SafeBrowsingDatabaseManager::GetHashSeverestThreatType(
173 const SBFullHash& hash,
174 const std::vector<SBFullHashResult>& full_hashes) {
175 return GetThreatTypeFromListType(
176 GetHashSeverestThreatListType(hash, full_hashes, NULL));
179 // static
180 SBThreatType SafeBrowsingDatabaseManager::GetUrlSeverestThreatType(
181 const GURL& url,
182 const std::vector<SBFullHashResult>& full_hashes,
183 size_t* index) {
184 return GetThreatTypeFromListType(
185 GetUrlSeverestThreatListType(url, full_hashes, index));
188 SafeBrowsingDatabaseManager::SafeBrowsingCheck::SafeBrowsingCheck(
189 const std::vector<GURL>& urls,
190 const std::vector<SBFullHash>& full_hashes,
191 Client* client,
192 safe_browsing_util::ListType check_type,
193 const std::vector<SBThreatType>& expected_threats)
194 : urls(urls),
195 url_results(urls.size(), SB_THREAT_TYPE_SAFE),
196 url_metadata(urls.size()),
197 full_hashes(full_hashes),
198 full_hash_results(full_hashes.size(), SB_THREAT_TYPE_SAFE),
199 client(client),
200 need_get_hash(false),
201 check_type(check_type),
202 expected_threats(expected_threats) {
203 DCHECK_EQ(urls.empty(), !full_hashes.empty())
204 << "Exactly one of urls and full_hashes must be set";
207 SafeBrowsingDatabaseManager::SafeBrowsingCheck::~SafeBrowsingCheck() {}
209 void SafeBrowsingDatabaseManager::Client::OnSafeBrowsingResult(
210 const SafeBrowsingCheck& check) {
211 DCHECK_CURRENTLY_ON(BrowserThread::IO);
213 DCHECK_EQ(check.urls.size(), check.url_results.size());
214 DCHECK_EQ(check.full_hashes.size(), check.full_hash_results.size());
215 if (!check.urls.empty()) {
216 DCHECK(check.full_hashes.empty());
217 switch (check.check_type) {
218 case safe_browsing_util::MALWARE:
219 case safe_browsing_util::PHISH:
220 case safe_browsing_util::UNWANTEDURL:
221 DCHECK_EQ(1u, check.urls.size());
222 OnCheckBrowseUrlResult(
223 check.urls[0], check.url_results[0], check.url_metadata[0]);
224 break;
225 case safe_browsing_util::BINURL:
226 DCHECK_EQ(check.urls.size(), check.url_results.size());
227 OnCheckDownloadUrlResult(
228 check.urls,
229 *std::max_element(check.url_results.begin(),
230 check.url_results.end()));
231 break;
232 default:
233 NOTREACHED();
235 } else if (!check.full_hashes.empty()) {
236 switch (check.check_type) {
237 case safe_browsing_util::EXTENSIONBLACKLIST: {
238 std::set<std::string> unsafe_extension_ids;
239 for (size_t i = 0; i < check.full_hashes.size(); ++i) {
240 std::string extension_id =
241 safe_browsing_util::SBFullHashToString(check.full_hashes[i]);
242 if (check.full_hash_results[i] == SB_THREAT_TYPE_EXTENSION)
243 unsafe_extension_ids.insert(extension_id);
245 OnCheckExtensionsResult(unsafe_extension_ids);
246 break;
248 default:
249 NOTREACHED();
251 } else {
252 NOTREACHED();
256 SafeBrowsingDatabaseManager::SafeBrowsingDatabaseManager(
257 const scoped_refptr<SafeBrowsingService>& service)
258 : sb_service_(service),
259 database_(NULL),
260 enabled_(false),
261 enable_download_protection_(false),
262 enable_csd_whitelist_(false),
263 enable_download_whitelist_(false),
264 enable_extension_blacklist_(false),
265 enable_side_effect_free_whitelist_(false),
266 enable_ip_blacklist_(false),
267 enable_unwanted_software_blacklist_(false),
268 update_in_progress_(false),
269 database_update_in_progress_(false),
270 closing_database_(false),
271 check_timeout_(base::TimeDelta::FromMilliseconds(kCheckTimeoutMs)) {
272 DCHECK_CURRENTLY_ON(BrowserThread::UI);
273 DCHECK(sb_service_.get() != NULL);
275 // Android only supports a subset of FULL_SAFE_BROWSING.
276 // TODO(shess): This shouldn't be OS-driven <http://crbug.com/394379>
277 #if !defined(OS_ANDROID)
278 base::CommandLine* cmdline = base::CommandLine::ForCurrentProcess();
279 enable_download_protection_ =
280 !cmdline->HasSwitch(switches::kSbDisableDownloadProtection);
282 // We only download the csd-whitelist if client-side phishing detection is
283 // enabled.
284 enable_csd_whitelist_ =
285 !cmdline->HasSwitch(switches::kDisableClientSidePhishingDetection);
287 // TODO(noelutz): remove this boolean variable since it should always be true
288 // if SafeBrowsing is enabled. Unfortunately, we have no test data for this
289 // list right now. This means that we need to be able to disable this list
290 // for the SafeBrowsing test to pass.
291 enable_download_whitelist_ = enable_csd_whitelist_;
293 // TODO(kalman): there really shouldn't be a flag for this.
294 enable_extension_blacklist_ =
295 !cmdline->HasSwitch(switches::kSbDisableExtensionBlacklist);
297 enable_side_effect_free_whitelist_ =
298 prerender::IsSideEffectFreeWhitelistEnabled() &&
299 !cmdline->HasSwitch(switches::kSbDisableSideEffectFreeWhitelist);
301 // The client-side IP blacklist feature is tightly integrated with client-side
302 // phishing protection for now.
303 enable_ip_blacklist_ = enable_csd_whitelist_;
305 // The UwS blacklist feature is controlled by a flag for M40.
306 enable_unwanted_software_blacklist_ =
307 safe_browsing_util::GetUnwantedTrialGroup() > safe_browsing_util::UWS_OFF;
309 enum SideEffectFreeWhitelistStatus {
310 SIDE_EFFECT_FREE_WHITELIST_ENABLED,
311 SIDE_EFFECT_FREE_WHITELIST_DISABLED,
312 SIDE_EFFECT_FREE_WHITELIST_STATUS_MAX
315 SideEffectFreeWhitelistStatus side_effect_free_whitelist_status =
316 enable_side_effect_free_whitelist_ ? SIDE_EFFECT_FREE_WHITELIST_ENABLED :
317 SIDE_EFFECT_FREE_WHITELIST_DISABLED;
319 UMA_HISTOGRAM_ENUMERATION("SB2.SideEffectFreeWhitelistStatus",
320 side_effect_free_whitelist_status,
321 SIDE_EFFECT_FREE_WHITELIST_STATUS_MAX);
322 #endif
325 SafeBrowsingDatabaseManager::~SafeBrowsingDatabaseManager() {
326 // The DCHECK is disabled due to crbug.com/438754.
327 // DCHECK_CURRENTLY_ON(BrowserThread::UI);
329 // We should have already been shut down. If we're still enabled, then the
330 // database isn't going to be closed properly, which could lead to corruption.
331 DCHECK(!enabled_);
334 bool SafeBrowsingDatabaseManager::CanCheckUrl(const GURL& url) const {
335 return url.SchemeIs(url::kFtpScheme) ||
336 url.SchemeIs(url::kHttpScheme) ||
337 url.SchemeIs(url::kHttpsScheme);
340 bool SafeBrowsingDatabaseManager::CheckDownloadUrl(
341 const std::vector<GURL>& url_chain,
342 Client* client) {
343 DCHECK_CURRENTLY_ON(BrowserThread::IO);
344 if (!enabled_ || !enable_download_protection_)
345 return true;
347 // We need to check the database for url prefix, and later may fetch the url
348 // from the safebrowsing backends. These need to be asynchronous.
349 SafeBrowsingCheck* check =
350 new SafeBrowsingCheck(url_chain,
351 std::vector<SBFullHash>(),
352 client,
353 safe_browsing_util::BINURL,
354 std::vector<SBThreatType>(1,
355 SB_THREAT_TYPE_BINARY_MALWARE_URL));
356 std::vector<SBPrefix> prefixes;
357 SafeBrowsingDatabase::GetDownloadUrlPrefixes(url_chain, &prefixes);
358 StartSafeBrowsingCheck(
359 check,
360 base::Bind(&SafeBrowsingDatabaseManager::CheckDownloadUrlOnSBThread, this,
361 prefixes));
362 return false;
365 bool SafeBrowsingDatabaseManager::CheckExtensionIDs(
366 const std::set<std::string>& extension_ids,
367 Client* client) {
368 DCHECK_CURRENTLY_ON(BrowserThread::IO);
370 if (!enabled_ || !enable_extension_blacklist_)
371 return true;
373 std::vector<SBFullHash> extension_id_hashes;
374 std::transform(extension_ids.begin(), extension_ids.end(),
375 std::back_inserter(extension_id_hashes),
376 safe_browsing_util::StringToSBFullHash);
377 std::vector<SBPrefix> prefixes;
378 for (const SBFullHash& hash : extension_id_hashes)
379 prefixes.push_back(hash.prefix);
381 SafeBrowsingCheck* check = new SafeBrowsingCheck(
382 std::vector<GURL>(),
383 extension_id_hashes,
384 client,
385 safe_browsing_util::EXTENSIONBLACKLIST,
386 std::vector<SBThreatType>(1, SB_THREAT_TYPE_EXTENSION));
387 StartSafeBrowsingCheck(
388 check,
389 base::Bind(&SafeBrowsingDatabaseManager::CheckExtensionIDsOnSBThread,
390 this, prefixes));
391 return false;
394 bool SafeBrowsingDatabaseManager::CheckSideEffectFreeWhitelistUrl(
395 const GURL& url) {
396 if (!enabled_)
397 return false;
399 if (!CanCheckUrl(url))
400 return false;
402 return database_->ContainsSideEffectFreeWhitelistUrl(url);
405 bool SafeBrowsingDatabaseManager::MatchMalwareIP(
406 const std::string& ip_address) {
407 DCHECK_CURRENTLY_ON(BrowserThread::IO);
408 if (!enabled_ || !enable_ip_blacklist_ || !MakeDatabaseAvailable()) {
409 return false; // Fail open.
411 return database_->ContainsMalwareIP(ip_address);
414 bool SafeBrowsingDatabaseManager::MatchCsdWhitelistUrl(const GURL& url) {
415 DCHECK_CURRENTLY_ON(BrowserThread::IO);
416 if (!enabled_ || !enable_csd_whitelist_ || !MakeDatabaseAvailable()) {
417 // There is something funky going on here -- for example, perhaps the user
418 // has not restarted since enabling metrics reporting, so we haven't
419 // enabled the csd whitelist yet. Just to be safe we return true in this
420 // case.
421 return true;
423 return database_->ContainsCsdWhitelistedUrl(url);
426 bool SafeBrowsingDatabaseManager::MatchDownloadWhitelistUrl(const GURL& url) {
427 DCHECK_CURRENTLY_ON(BrowserThread::IO);
428 if (!enabled_ || !enable_download_whitelist_ || !MakeDatabaseAvailable()) {
429 return true;
431 return database_->ContainsDownloadWhitelistedUrl(url);
434 bool SafeBrowsingDatabaseManager::MatchDownloadWhitelistString(
435 const std::string& str) {
436 DCHECK_CURRENTLY_ON(BrowserThread::IO);
437 if (!enabled_ || !enable_download_whitelist_ || !MakeDatabaseAvailable()) {
438 return true;
440 return database_->ContainsDownloadWhitelistedString(str);
443 bool SafeBrowsingDatabaseManager::MatchInclusionWhitelistUrl(const GURL& url) {
444 DCHECK_CURRENTLY_ON(BrowserThread::IO);
445 if (!enabled_ || !MakeDatabaseAvailable())
446 return true;
447 return database_->ContainsInclusionWhitelistedUrl(url);
450 bool SafeBrowsingDatabaseManager::IsMalwareKillSwitchOn() {
451 DCHECK_CURRENTLY_ON(BrowserThread::IO);
452 if (!enabled_ || !MakeDatabaseAvailable()) {
453 return true;
455 return database_->IsMalwareIPMatchKillSwitchOn();
458 bool SafeBrowsingDatabaseManager::IsCsdWhitelistKillSwitchOn() {
459 DCHECK_CURRENTLY_ON(BrowserThread::IO);
460 if (!enabled_ || !MakeDatabaseAvailable()) {
461 return true;
463 return database_->IsCsdWhitelistKillSwitchOn();
466 bool SafeBrowsingDatabaseManager::CheckBrowseUrl(const GURL& url,
467 Client* client) {
468 DCHECK_CURRENTLY_ON(BrowserThread::IO);
469 if (!enabled_)
470 return true;
472 if (!CanCheckUrl(url))
473 return true;
475 std::vector<SBThreatType> expected_threats;
476 expected_threats.push_back(SB_THREAT_TYPE_URL_MALWARE);
477 expected_threats.push_back(SB_THREAT_TYPE_URL_PHISHING);
478 expected_threats.push_back(SB_THREAT_TYPE_URL_UNWANTED);
480 const base::TimeTicks start = base::TimeTicks::Now();
481 if (!MakeDatabaseAvailable()) {
482 QueuedCheck queued_check(safe_browsing_util::MALWARE, // or PHISH
483 client,
484 url,
485 expected_threats,
486 start);
487 queued_checks_.push_back(queued_check);
488 return false;
491 // Cache hits should, in general, be the same for both (ignoring potential
492 // cache evictions in the second call for entries that were just about to be
493 // evicted in the first call).
494 // TODO(gab): Refactor SafeBrowsingDatabase to avoid depending on this here.
495 std::vector<SBFullHashResult> cache_hits;
497 std::vector<SBPrefix> browse_prefix_hits;
498 bool browse_prefix_match = database_->ContainsBrowseUrl(
499 url, &browse_prefix_hits, &cache_hits);
501 std::vector<SBPrefix> unwanted_prefix_hits;
502 std::vector<SBFullHashResult> unused_cache_hits;
503 bool unwanted_prefix_match = database_->ContainsUnwantedSoftwareUrl(
504 url, &unwanted_prefix_hits, &unused_cache_hits);
506 // Merge the two pre-sorted prefix hits lists.
507 // TODO(gab): Refactor SafeBrowsingDatabase for it to return this merged list
508 // by default rather than building it here.
509 std::vector<SBPrefix> prefix_hits(browse_prefix_hits.size() +
510 unwanted_prefix_hits.size());
511 std::merge(browse_prefix_hits.begin(),
512 browse_prefix_hits.end(),
513 unwanted_prefix_hits.begin(),
514 unwanted_prefix_hits.end(),
515 prefix_hits.begin());
516 prefix_hits.erase(std::unique(prefix_hits.begin(), prefix_hits.end()),
517 prefix_hits.end());
519 UMA_HISTOGRAM_TIMES("SB2.FilterCheck", base::TimeTicks::Now() - start);
521 if (!browse_prefix_match && !unwanted_prefix_match)
522 return true; // URL is okay.
524 // Needs to be asynchronous, since we could be in the constructor of a
525 // ResourceDispatcherHost event handler which can't pause there.
526 // This check will ping the Safe Browsing servers and get all lists which it
527 // matches. These lists will then be filtered against the |expected_threats|
528 // and the result callback for MALWARE (which is the same as for PHISH and
529 // UNWANTEDURL) will eventually be invoked with the final decision.
530 SafeBrowsingCheck* check = new SafeBrowsingCheck(std::vector<GURL>(1, url),
531 std::vector<SBFullHash>(),
532 client,
533 safe_browsing_util::MALWARE,
534 expected_threats);
535 check->need_get_hash = cache_hits.empty();
536 check->prefix_hits.swap(prefix_hits);
537 check->cache_hits.swap(cache_hits);
538 checks_.insert(check);
540 BrowserThread::PostTask(
541 BrowserThread::IO, FROM_HERE,
542 base::Bind(&SafeBrowsingDatabaseManager::OnCheckDone, this, check));
544 return false;
547 void SafeBrowsingDatabaseManager::CancelCheck(Client* client) {
548 DCHECK_CURRENTLY_ON(BrowserThread::IO);
549 for (CurrentChecks::iterator i = checks_.begin(); i != checks_.end(); ++i) {
550 // We can't delete matching checks here because the db thread has a copy of
551 // the pointer. Instead, we simply NULL out the client, and when the db
552 // thread calls us back, we'll clean up the check.
553 if ((*i)->client == client)
554 (*i)->client = NULL;
557 // Scan the queued clients store. Clients may be here if they requested a URL
558 // check before the database has finished loading.
559 for (std::deque<QueuedCheck>::iterator it(queued_checks_.begin());
560 it != queued_checks_.end(); ) {
561 // In this case it's safe to delete matches entirely since nothing has a
562 // pointer to them.
563 if (it->client == client)
564 it = queued_checks_.erase(it);
565 else
566 ++it;
570 void SafeBrowsingDatabaseManager::HandleGetHashResults(
571 SafeBrowsingCheck* check,
572 const std::vector<SBFullHashResult>& full_hashes,
573 const base::TimeDelta& cache_lifetime) {
574 DCHECK_CURRENTLY_ON(BrowserThread::IO);
576 if (!enabled_)
577 return;
579 // If the service has been shut down, |check| should have been deleted.
580 DCHECK(checks_.find(check) != checks_.end());
582 // |start| is set before calling |GetFullHash()|, which should be
583 // the only path which gets to here.
584 DCHECK(!check->start.is_null());
585 UMA_HISTOGRAM_LONG_TIMES("SB2.Network",
586 base::TimeTicks::Now() - check->start);
588 std::vector<SBPrefix> prefixes = check->prefix_hits;
589 OnHandleGetHashResults(check, full_hashes); // 'check' is deleted here.
591 // Cache the GetHash results.
592 if (cache_lifetime != base::TimeDelta() && MakeDatabaseAvailable())
593 database_->CacheHashResults(prefixes, full_hashes, cache_lifetime);
596 void SafeBrowsingDatabaseManager::GetChunks(GetChunksCallback callback) {
597 DCHECK_CURRENTLY_ON(BrowserThread::IO);
598 DCHECK(enabled_);
599 DCHECK(!callback.is_null());
600 safe_browsing_task_runner_->PostTask(
601 FROM_HERE,
602 base::Bind(&SafeBrowsingDatabaseManager::GetAllChunksFromDatabase, this,
603 callback));
606 void SafeBrowsingDatabaseManager::AddChunks(
607 const std::string& list,
608 scoped_ptr<ScopedVector<SBChunkData> > chunks,
609 AddChunksCallback callback) {
610 DCHECK_CURRENTLY_ON(BrowserThread::IO);
611 DCHECK(enabled_);
612 DCHECK(!callback.is_null());
613 safe_browsing_task_runner_->PostTask(
614 FROM_HERE, base::Bind(&SafeBrowsingDatabaseManager::AddDatabaseChunks,
615 this, list, base::Passed(&chunks), callback));
618 void SafeBrowsingDatabaseManager::DeleteChunks(
619 scoped_ptr<std::vector<SBChunkDelete> > chunk_deletes) {
620 DCHECK_CURRENTLY_ON(BrowserThread::IO);
621 DCHECK(enabled_);
622 safe_browsing_task_runner_->PostTask(
623 FROM_HERE, base::Bind(&SafeBrowsingDatabaseManager::DeleteDatabaseChunks,
624 this, base::Passed(&chunk_deletes)));
627 void SafeBrowsingDatabaseManager::UpdateStarted() {
628 DCHECK_CURRENTLY_ON(BrowserThread::IO);
629 DCHECK(enabled_);
630 DCHECK(!update_in_progress_);
631 update_in_progress_ = true;
634 void SafeBrowsingDatabaseManager::UpdateFinished(bool update_succeeded) {
635 DCHECK_CURRENTLY_ON(BrowserThread::IO);
636 DCHECK(enabled_);
637 if (update_in_progress_) {
638 update_in_progress_ = false;
639 safe_browsing_task_runner_->PostTask(
640 FROM_HERE,
641 base::Bind(&SafeBrowsingDatabaseManager::DatabaseUpdateFinished, this,
642 update_succeeded));
646 void SafeBrowsingDatabaseManager::ResetDatabase() {
647 DCHECK_CURRENTLY_ON(BrowserThread::IO);
648 DCHECK(enabled_);
649 safe_browsing_task_runner_->PostTask(
650 FROM_HERE,
651 base::Bind(&SafeBrowsingDatabaseManager::OnResetDatabase, this));
654 void SafeBrowsingDatabaseManager::StartOnIOThread() {
655 // TODO(pkasting): Remove ScopedTracker below once crbug.com/455469 is fixed.
656 tracked_objects::ScopedTracker tracking_profile1(
657 FROM_HERE_WITH_EXPLICIT_FUNCTION(
658 "455469 SafeBrowsingDatabaseManager::StartOnIOThread1"));
659 DCHECK_CURRENTLY_ON(BrowserThread::IO);
660 if (enabled_)
661 return;
663 // Only get a new task runner if there isn't one already. If the service has
664 // previously been started and stopped, a task runner could already exist.
665 if (!safe_browsing_task_runner_) {
666 // TODO(pkasting): Remove ScopedTracker below once crbug.com/455469 is
667 // fixed.
668 tracked_objects::ScopedTracker tracking_profile2(
669 FROM_HERE_WITH_EXPLICIT_FUNCTION(
670 "455469 SafeBrowsingDatabaseManager::StartOnIOThread2"));
671 base::SequencedWorkerPool* pool = BrowserThread::GetBlockingPool();
672 safe_browsing_task_runner_ =
673 pool->GetSequencedTaskRunnerWithShutdownBehavior(
674 pool->GetSequenceToken(),
675 base::SequencedWorkerPool::SKIP_ON_SHUTDOWN);
678 enabled_ = true;
680 MakeDatabaseAvailable();
683 void SafeBrowsingDatabaseManager::StopOnIOThread(bool shutdown) {
684 DCHECK_CURRENTLY_ON(BrowserThread::IO);
686 DoStopOnIOThread();
687 if (shutdown) {
688 sb_service_ = NULL;
692 void SafeBrowsingDatabaseManager::NotifyDatabaseUpdateFinished(
693 bool update_succeeded) {
694 DCHECK_CURRENTLY_ON(BrowserThread::UI);
695 content::NotificationService::current()->Notify(
696 chrome::NOTIFICATION_SAFE_BROWSING_UPDATE_COMPLETE,
697 content::Source<SafeBrowsingDatabaseManager>(this),
698 content::Details<bool>(&update_succeeded));
701 SafeBrowsingDatabaseManager::QueuedCheck::QueuedCheck(
702 const safe_browsing_util::ListType check_type,
703 Client* client,
704 const GURL& url,
705 const std::vector<SBThreatType>& expected_threats,
706 const base::TimeTicks& start)
707 : check_type(check_type),
708 client(client),
709 url(url),
710 expected_threats(expected_threats),
711 start(start) {
714 SafeBrowsingDatabaseManager::QueuedCheck::~QueuedCheck() {
717 void SafeBrowsingDatabaseManager::DoStopOnIOThread() {
718 DCHECK_CURRENTLY_ON(BrowserThread::IO);
720 if (!enabled_)
721 return;
723 enabled_ = false;
725 // Delete queued checks, calling back any clients with 'SB_THREAT_TYPE_SAFE'.
726 while (!queued_checks_.empty()) {
727 QueuedCheck queued = queued_checks_.front();
728 if (queued.client) {
729 SafeBrowsingCheck sb_check(std::vector<GURL>(1, queued.url),
730 std::vector<SBFullHash>(),
731 queued.client,
732 queued.check_type,
733 queued.expected_threats);
734 queued.client->OnSafeBrowsingResult(sb_check);
736 queued_checks_.pop_front();
739 // Close the database. Cases to avoid:
740 // * If |closing_database_| is true, continuing will queue up a second
741 // request, |closing_database_| will be reset after handling the first
742 // request, and if any functions on the db thread recreate the database, we
743 // could start using it on the IO thread and then have the second request
744 // handler delete it out from under us.
745 // * If |database_| is NULL, then either no creation request is in flight, in
746 // which case we don't need to do anything, or one is in flight, in which
747 // case the database will be recreated before our deletion request is
748 // handled, and could be used on the IO thread in that time period, leading
749 // to the same problem as above.
750 // Checking DatabaseAvailable() avoids both of these.
751 if (DatabaseAvailable()) {
752 closing_database_ = true;
753 safe_browsing_task_runner_->PostTask(
754 FROM_HERE,
755 base::Bind(&SafeBrowsingDatabaseManager::OnCloseDatabase, this));
758 // Delete pending checks, calling back any clients with 'SB_THREAT_TYPE_SAFE'.
759 // We have to do this after the db thread returns because methods on it can
760 // have copies of these pointers, so deleting them might lead to accessing
761 // garbage.
762 for (CurrentChecks::iterator it = checks_.begin();
763 it != checks_.end(); ++it) {
764 SafeBrowsingCheck* check = *it;
765 if (check->client)
766 check->client->OnSafeBrowsingResult(*check);
768 STLDeleteElements(&checks_);
770 gethash_requests_.clear();
773 bool SafeBrowsingDatabaseManager::DatabaseAvailable() const {
774 base::AutoLock lock(database_lock_);
775 return !closing_database_ && (database_ != NULL);
778 bool SafeBrowsingDatabaseManager::MakeDatabaseAvailable() {
779 DCHECK_CURRENTLY_ON(BrowserThread::IO);
780 DCHECK(enabled_);
781 if (DatabaseAvailable())
782 return true;
783 safe_browsing_task_runner_->PostTask(
784 FROM_HERE,
785 base::Bind(base::IgnoreResult(&SafeBrowsingDatabaseManager::GetDatabase),
786 this));
787 return false;
790 SafeBrowsingDatabase* SafeBrowsingDatabaseManager::GetDatabase() {
791 DCHECK(safe_browsing_task_runner_->RunsTasksOnCurrentThread());
793 if (database_)
794 return database_;
795 startup_metric_utils::ScopedSlowStartupUMA
796 scoped_timer("Startup.SlowStartupSafeBrowsingGetDatabase");
797 const base::TimeTicks before = base::TimeTicks::Now();
799 SafeBrowsingDatabase* database = SafeBrowsingDatabase::Create(
800 safe_browsing_task_runner_, enable_download_protection_,
801 enable_csd_whitelist_, enable_download_whitelist_,
802 enable_extension_blacklist_, enable_side_effect_free_whitelist_,
803 enable_ip_blacklist_, enable_unwanted_software_blacklist_);
805 database->Init(SafeBrowsingService::GetBaseFilename());
807 // Acquiring the lock here guarantees correct ordering between the writes to
808 // the new database object above, and the setting of |database_| below.
809 base::AutoLock lock(database_lock_);
810 database_ = database;
813 BrowserThread::PostTask(
814 BrowserThread::IO, FROM_HERE,
815 base::Bind(&SafeBrowsingDatabaseManager::DatabaseLoadComplete, this));
817 UMA_HISTOGRAM_TIMES("SB2.DatabaseOpen", base::TimeTicks::Now() - before);
818 return database_;
821 void SafeBrowsingDatabaseManager::OnCheckDone(SafeBrowsingCheck* check) {
822 DCHECK_CURRENTLY_ON(BrowserThread::IO);
824 if (!enabled_)
825 return;
827 // If the service has been shut down, |check| should have been deleted.
828 DCHECK(checks_.find(check) != checks_.end());
830 if (check->client && check->need_get_hash) {
831 // We have a partial match so we need to query Google for the full hash.
832 // Clean up will happen in HandleGetHashResults.
834 // See if we have a GetHash request already in progress for this particular
835 // prefix. If so, we just append ourselves to the list of interested parties
836 // when the results arrive. We only do this for checks involving one prefix,
837 // since that is the common case (multiple prefixes will issue the request
838 // as normal).
839 if (check->prefix_hits.size() == 1) {
840 SBPrefix prefix = check->prefix_hits[0];
841 GetHashRequests::iterator it = gethash_requests_.find(prefix);
842 if (it != gethash_requests_.end()) {
843 // There's already a request in progress.
844 it->second.push_back(check);
845 return;
848 // No request in progress, so we're the first for this prefix.
849 GetHashRequestors requestors;
850 requestors.push_back(check);
851 gethash_requests_[prefix] = requestors;
854 // Reset the start time so that we can measure the network time without the
855 // database time.
856 check->start = base::TimeTicks::Now();
857 // Note: If |this| is deleted or stopped, the protocol_manager will
858 // be destroyed as well - hence it's OK to do unretained in this case.
859 bool is_download = check->check_type == safe_browsing_util::BINURL;
860 sb_service_->protocol_manager()->GetFullHash(
861 check->prefix_hits,
862 base::Bind(&SafeBrowsingDatabaseManager::HandleGetHashResults,
863 base::Unretained(this),
864 check),
865 is_download);
866 } else {
867 // We may have cached results for previous GetHash queries. Since
868 // this data comes from cache, don't histogram hits.
869 HandleOneCheck(check, check->cache_hits);
873 void SafeBrowsingDatabaseManager::GetAllChunksFromDatabase(
874 GetChunksCallback callback) {
875 DCHECK(safe_browsing_task_runner_->RunsTasksOnCurrentThread());
877 bool database_error = true;
878 std::vector<SBListChunkRanges> lists;
879 DCHECK(!database_update_in_progress_);
880 database_update_in_progress_ = true;
881 GetDatabase(); // This guarantees that |database_| is non-NULL.
882 if (database_->UpdateStarted(&lists)) {
883 database_error = false;
884 } else {
885 database_->UpdateFinished(false);
888 BrowserThread::PostTask(
889 BrowserThread::IO, FROM_HERE,
890 base::Bind(&SafeBrowsingDatabaseManager::OnGetAllChunksFromDatabase,
891 this, lists, database_error, callback));
894 void SafeBrowsingDatabaseManager::OnGetAllChunksFromDatabase(
895 const std::vector<SBListChunkRanges>& lists, bool database_error,
896 GetChunksCallback callback) {
897 DCHECK_CURRENTLY_ON(BrowserThread::IO);
898 if (enabled_)
899 callback.Run(lists, database_error);
902 void SafeBrowsingDatabaseManager::OnAddChunksComplete(
903 AddChunksCallback callback) {
904 DCHECK_CURRENTLY_ON(BrowserThread::IO);
905 if (enabled_)
906 callback.Run();
909 void SafeBrowsingDatabaseManager::DatabaseLoadComplete() {
910 DCHECK_CURRENTLY_ON(BrowserThread::IO);
911 if (!enabled_)
912 return;
914 LOCAL_HISTOGRAM_COUNTS("SB.QueueDepth", queued_checks_.size());
915 if (queued_checks_.empty())
916 return;
918 // If the database isn't already available, calling CheckUrl() in the loop
919 // below will add the check back to the queue, and we'll infinite-loop.
920 DCHECK(DatabaseAvailable());
921 while (!queued_checks_.empty()) {
922 QueuedCheck check = queued_checks_.front();
923 DCHECK(!check.start.is_null());
924 LOCAL_HISTOGRAM_TIMES("SB.QueueDelay",
925 base::TimeTicks::Now() - check.start);
926 // If CheckUrl() determines the URL is safe immediately, it doesn't call the
927 // client's handler function (because normally it's being directly called by
928 // the client). Since we're not the client, we have to convey this result.
929 if (check.client && CheckBrowseUrl(check.url, check.client)) {
930 SafeBrowsingCheck sb_check(std::vector<GURL>(1, check.url),
931 std::vector<SBFullHash>(),
932 check.client,
933 check.check_type,
934 check.expected_threats);
935 check.client->OnSafeBrowsingResult(sb_check);
937 queued_checks_.pop_front();
941 void SafeBrowsingDatabaseManager::AddDatabaseChunks(
942 const std::string& list_name,
943 scoped_ptr<ScopedVector<SBChunkData> > chunks,
944 AddChunksCallback callback) {
945 DCHECK(safe_browsing_task_runner_->RunsTasksOnCurrentThread());
946 if (chunks)
947 GetDatabase()->InsertChunks(list_name, chunks->get());
948 BrowserThread::PostTask(
949 BrowserThread::IO, FROM_HERE,
950 base::Bind(&SafeBrowsingDatabaseManager::OnAddChunksComplete, this,
951 callback));
954 void SafeBrowsingDatabaseManager::DeleteDatabaseChunks(
955 scoped_ptr<std::vector<SBChunkDelete> > chunk_deletes) {
956 DCHECK(safe_browsing_task_runner_->RunsTasksOnCurrentThread());
957 if (chunk_deletes)
958 GetDatabase()->DeleteChunks(*chunk_deletes);
961 void SafeBrowsingDatabaseManager::DatabaseUpdateFinished(
962 bool update_succeeded) {
963 DCHECK(safe_browsing_task_runner_->RunsTasksOnCurrentThread());
964 GetDatabase()->UpdateFinished(update_succeeded);
965 DCHECK(database_update_in_progress_);
966 database_update_in_progress_ = false;
967 BrowserThread::PostTask(
968 BrowserThread::UI, FROM_HERE,
969 base::Bind(&SafeBrowsingDatabaseManager::NotifyDatabaseUpdateFinished,
970 this, update_succeeded));
973 void SafeBrowsingDatabaseManager::OnCloseDatabase() {
974 DCHECK(safe_browsing_task_runner_->RunsTasksOnCurrentThread());
975 DCHECK(closing_database_);
977 // Because |closing_database_| is true, nothing on the IO thread will be
978 // accessing the database, so it's safe to delete and then NULL the pointer.
979 delete database_;
980 database_ = NULL;
982 // Acquiring the lock here guarantees correct ordering between the resetting
983 // of |database_| above and of |closing_database_| below, which ensures there
984 // won't be a window during which the IO thread falsely believes the database
985 // is available.
986 base::AutoLock lock(database_lock_);
987 closing_database_ = false;
990 void SafeBrowsingDatabaseManager::OnResetDatabase() {
991 DCHECK(safe_browsing_task_runner_->RunsTasksOnCurrentThread());
993 GetDatabase()->ResetDatabase();
996 void SafeBrowsingDatabaseManager::OnHandleGetHashResults(
997 SafeBrowsingCheck* check,
998 const std::vector<SBFullHashResult>& full_hashes) {
999 DCHECK_CURRENTLY_ON(BrowserThread::IO);
1000 safe_browsing_util::ListType check_type = check->check_type;
1001 SBPrefix prefix = check->prefix_hits[0];
1002 GetHashRequests::iterator it = gethash_requests_.find(prefix);
1003 if (check->prefix_hits.size() > 1 || it == gethash_requests_.end()) {
1004 const bool hit = HandleOneCheck(check, full_hashes);
1005 RecordGetHashCheckStatus(hit, check_type, full_hashes);
1006 return;
1009 // Call back all interested parties, noting if any has a hit.
1010 GetHashRequestors& requestors = it->second;
1011 bool hit = false;
1012 for (GetHashRequestors::iterator r = requestors.begin();
1013 r != requestors.end(); ++r) {
1014 if (HandleOneCheck(*r, full_hashes))
1015 hit = true;
1017 RecordGetHashCheckStatus(hit, check_type, full_hashes);
1019 gethash_requests_.erase(it);
1022 bool SafeBrowsingDatabaseManager::HandleOneCheck(
1023 SafeBrowsingCheck* check,
1024 const std::vector<SBFullHashResult>& full_hashes) {
1025 DCHECK_CURRENTLY_ON(BrowserThread::IO);
1026 DCHECK(check);
1028 bool is_threat = false;
1030 // TODO(shess): GetHashSeverestThreadListType() contains a loop,
1031 // GetUrlSeverestThreatListType() a loop around that loop. Having another
1032 // loop out here concerns me. It is likely that SAFE is an expected outcome,
1033 // which means all of those loops run to completion. Refactoring this to
1034 // generate a set of sorted items to compare in sequence would probably
1035 // improve things.
1037 // Additionally, the set of patterns generated from the urls is very similar
1038 // to the patterns generated in ContainsBrowseUrl() and other database checks,
1039 // which are called from this code. Refactoring that across the checks could
1040 // interact well with batching the checks here.
1042 // TODO(gab): Fix the fact that Get(Url|Hash)SeverestThreatType() may return a
1043 // threat for which IsExpectedThreat() returns false even if |full_hashes|
1044 // actually contains an expected threat.
1046 for (size_t i = 0; i < check->urls.size(); ++i) {
1047 size_t threat_index;
1048 SBThreatType threat =
1049 GetUrlSeverestThreatType(check->urls[i], full_hashes, &threat_index);
1050 if (threat != SB_THREAT_TYPE_SAFE &&
1051 IsExpectedThreat(threat, check->expected_threats)) {
1052 check->url_results[i] = threat;
1053 check->url_metadata[i] = full_hashes[threat_index].metadata;
1054 is_threat = true;
1058 for (size_t i = 0; i < check->full_hashes.size(); ++i) {
1059 SBThreatType threat =
1060 GetHashSeverestThreatType(check->full_hashes[i], full_hashes);
1061 if (threat != SB_THREAT_TYPE_SAFE &&
1062 IsExpectedThreat(threat, check->expected_threats)) {
1063 check->full_hash_results[i] = threat;
1064 is_threat = true;
1068 SafeBrowsingCheckDone(check);
1069 return is_threat;
1072 void SafeBrowsingDatabaseManager::OnAsyncCheckDone(
1073 SafeBrowsingCheck* check,
1074 const std::vector<SBPrefix>& prefix_hits) {
1075 DCHECK_CURRENTLY_ON(BrowserThread::IO);
1076 DCHECK(enable_download_protection_);
1078 check->prefix_hits = prefix_hits;
1079 if (check->prefix_hits.empty()) {
1080 SafeBrowsingCheckDone(check);
1081 } else {
1082 check->need_get_hash = true;
1083 OnCheckDone(check);
1087 std::vector<SBPrefix> SafeBrowsingDatabaseManager::CheckDownloadUrlOnSBThread(
1088 const std::vector<SBPrefix>& prefixes) {
1089 DCHECK(safe_browsing_task_runner_->RunsTasksOnCurrentThread());
1090 DCHECK(enable_download_protection_);
1092 std::vector<SBPrefix> prefix_hits;
1093 const bool result =
1094 database_->ContainsDownloadUrlPrefixes(prefixes, &prefix_hits);
1095 DCHECK_EQ(result, !prefix_hits.empty());
1096 return prefix_hits;
1099 std::vector<SBPrefix> SafeBrowsingDatabaseManager::CheckExtensionIDsOnSBThread(
1100 const std::vector<SBPrefix>& prefixes) {
1101 DCHECK(safe_browsing_task_runner_->RunsTasksOnCurrentThread());
1103 std::vector<SBPrefix> prefix_hits;
1104 const bool result =
1105 database_->ContainsExtensionPrefixes(prefixes, &prefix_hits);
1106 DCHECK_EQ(result, !prefix_hits.empty());
1107 return prefix_hits;
1110 void SafeBrowsingDatabaseManager::TimeoutCallback(SafeBrowsingCheck* check) {
1111 DCHECK_CURRENTLY_ON(BrowserThread::IO);
1112 DCHECK(check);
1114 if (!enabled_)
1115 return;
1117 DCHECK(checks_.find(check) != checks_.end());
1118 if (check->client) {
1119 check->client->OnSafeBrowsingResult(*check);
1120 check->client = NULL;
1124 void SafeBrowsingDatabaseManager::SafeBrowsingCheckDone(
1125 SafeBrowsingCheck* check) {
1126 DCHECK_CURRENTLY_ON(BrowserThread::IO);
1127 DCHECK(check);
1129 if (!enabled_)
1130 return;
1132 DVLOG(1) << "SafeBrowsingCheckDone";
1133 DCHECK(checks_.find(check) != checks_.end());
1134 if (check->client)
1135 check->client->OnSafeBrowsingResult(*check);
1136 checks_.erase(check);
1137 delete check;
1140 void SafeBrowsingDatabaseManager::StartSafeBrowsingCheck(
1141 SafeBrowsingCheck* check,
1142 const base::Callback<std::vector<SBPrefix>(void)>& task) {
1143 DCHECK_CURRENTLY_ON(BrowserThread::IO);
1144 check->weak_ptr_factory_.reset(
1145 new base::WeakPtrFactory<SafeBrowsingDatabaseManager>(this));
1146 checks_.insert(check);
1148 base::PostTaskAndReplyWithResult(
1149 safe_browsing_task_runner_.get(), FROM_HERE, task,
1150 base::Bind(&SafeBrowsingDatabaseManager::OnAsyncCheckDone,
1151 check->weak_ptr_factory_->GetWeakPtr(), check));
1152 base::MessageLoop::current()->PostDelayedTask(FROM_HERE,
1153 base::Bind(&SafeBrowsingDatabaseManager::TimeoutCallback,
1154 check->weak_ptr_factory_->GetWeakPtr(), check),
1155 check_timeout_);