1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "chrome/browser/safe_browsing/database_manager.h"
10 #include "base/bind_helpers.h"
11 #include "base/callback.h"
12 #include "base/command_line.h"
13 #include "base/debug/leak_tracker.h"
14 #include "base/stl_util.h"
15 #include "base/strings/string_util.h"
16 #include "base/threading/thread.h"
17 #include "base/threading/thread_restrictions.h"
18 #include "chrome/browser/browser_process.h"
19 #include "chrome/browser/chrome_notification_types.h"
20 #include "chrome/browser/prerender/prerender_field_trial.h"
21 #include "chrome/browser/safe_browsing/client_side_detection_service.h"
22 #include "chrome/browser/safe_browsing/download_protection_service.h"
23 #include "chrome/browser/safe_browsing/malware_details.h"
24 #include "chrome/browser/safe_browsing/protocol_manager.h"
25 #include "chrome/browser/safe_browsing/safe_browsing_database.h"
26 #include "chrome/browser/safe_browsing/safe_browsing_service.h"
27 #include "chrome/browser/safe_browsing/ui_manager.h"
28 #include "chrome/common/chrome_constants.h"
29 #include "chrome/common/chrome_paths.h"
30 #include "chrome/common/chrome_switches.h"
31 #include "components/metrics/metrics_service.h"
32 #include "components/startup_metric_utils/startup_metric_utils.h"
33 #include "content/public/browser/browser_thread.h"
34 #include "content/public/browser/notification_service.h"
35 #include "url/url_constants.h"
37 using content::BrowserThread
;
41 // Timeout for match checks, e.g. download URLs, hashes.
42 const int kCheckTimeoutMs
= 10000;
44 // Records disposition information about the check. |hit| should be
45 // |true| if there were any prefix hits in |full_hashes|.
46 void RecordGetHashCheckStatus(
48 safe_browsing_util::ListType check_type
,
49 const std::vector
<SBFullHashResult
>& full_hashes
) {
50 SafeBrowsingProtocolManager::ResultType result
;
51 if (full_hashes
.empty()) {
52 result
= SafeBrowsingProtocolManager::GET_HASH_FULL_HASH_EMPTY
;
54 result
= SafeBrowsingProtocolManager::GET_HASH_FULL_HASH_HIT
;
56 result
= SafeBrowsingProtocolManager::GET_HASH_FULL_HASH_MISS
;
58 bool is_download
= check_type
== safe_browsing_util::BINURL
;
59 SafeBrowsingProtocolManager::RecordGetHashResult(is_download
, result
);
62 bool IsExpectedThreat(
63 const SBThreatType threat_type
,
64 const std::vector
<SBThreatType
>& expected_threats
) {
65 return expected_threats
.end() != std::find(expected_threats
.begin(),
66 expected_threats
.end(),
70 // Return the severest list id from the results in |full_hashes| which matches
71 // |hash|, or INVALID if none match.
72 safe_browsing_util::ListType
GetHashSeverestThreatListType(
73 const SBFullHash
& hash
,
74 const std::vector
<SBFullHashResult
>& full_hashes
,
76 safe_browsing_util::ListType pending_threat
= safe_browsing_util::INVALID
;
77 for (size_t i
= 0; i
< full_hashes
.size(); ++i
) {
78 if (SBFullHashEqual(hash
, full_hashes
[i
].hash
)) {
79 const safe_browsing_util::ListType threat
=
80 static_cast<safe_browsing_util::ListType
>(full_hashes
[i
].list_id
);
82 case safe_browsing_util::INVALID
:
83 // |full_hashes| should never contain INVALID as a |list_id|.
86 case safe_browsing_util::MALWARE
: // Falls through.
87 case safe_browsing_util::PHISH
: // Falls through.
88 case safe_browsing_util::BINURL
: // Falls through.
89 case safe_browsing_util::CSDWHITELIST
: // Falls through.
90 case safe_browsing_util::DOWNLOADWHITELIST
: // Falls through.
91 case safe_browsing_util::EXTENSIONBLACKLIST
: // Falls through.
92 case safe_browsing_util::SIDEEFFECTFREEWHITELIST
: // Falls through.
93 case safe_browsing_util::IPBLACKLIST
:
97 case safe_browsing_util::UNWANTEDURL
:
98 // UNWANTEDURL is considered less severe than other threats, keep
100 pending_threat
= threat
;
107 return pending_threat
;
110 // Given a URL, compare all the possible host + path full hashes to the set of
111 // provided full hashes. Returns the list id of the severest matching result
112 // from |full_hashes|, or INVALID if none match.
113 safe_browsing_util::ListType
GetUrlSeverestThreatListType(
115 const std::vector
<SBFullHashResult
>& full_hashes
,
117 if (full_hashes
.empty())
118 return safe_browsing_util::INVALID
;
120 std::vector
<std::string
> patterns
;
121 safe_browsing_util::GeneratePatternsToCheck(url
, &patterns
);
123 safe_browsing_util::ListType pending_threat
= safe_browsing_util::INVALID
;
124 for (size_t i
= 0; i
< patterns
.size(); ++i
) {
125 safe_browsing_util::ListType threat
= GetHashSeverestThreatListType(
126 SBFullHashForString(patterns
[i
]), full_hashes
, index
);
128 case safe_browsing_util::INVALID
:
129 // Ignore patterns with no matching threat.
131 case safe_browsing_util::MALWARE
: // Falls through.
132 case safe_browsing_util::PHISH
: // Falls through.
133 case safe_browsing_util::BINURL
: // Falls through.
134 case safe_browsing_util::CSDWHITELIST
: // Falls through.
135 case safe_browsing_util::DOWNLOADWHITELIST
: // Falls through.
136 case safe_browsing_util::EXTENSIONBLACKLIST
: // Falls through.
137 case safe_browsing_util::SIDEEFFECTFREEWHITELIST
: // Falls through.
138 case safe_browsing_util::IPBLACKLIST
:
140 case safe_browsing_util::UNWANTEDURL
:
141 // UNWANTEDURL is considered less severe than other threats, keep
143 pending_threat
= threat
;
147 return pending_threat
;
150 SBThreatType
GetThreatTypeFromListType(safe_browsing_util::ListType list_type
) {
152 case safe_browsing_util::PHISH
:
153 return SB_THREAT_TYPE_URL_PHISHING
;
154 case safe_browsing_util::MALWARE
:
155 return SB_THREAT_TYPE_URL_MALWARE
;
156 case safe_browsing_util::UNWANTEDURL
:
157 return SB_THREAT_TYPE_URL_UNWANTED
;
158 case safe_browsing_util::BINURL
:
159 return SB_THREAT_TYPE_BINARY_MALWARE_URL
;
160 case safe_browsing_util::EXTENSIONBLACKLIST
:
161 return SB_THREAT_TYPE_EXTENSION
;
163 DVLOG(1) << "Unknown safe browsing list id " << list_type
;
164 return SB_THREAT_TYPE_SAFE
;
171 SBThreatType
SafeBrowsingDatabaseManager::GetHashSeverestThreatType(
172 const SBFullHash
& hash
,
173 const std::vector
<SBFullHashResult
>& full_hashes
) {
174 return GetThreatTypeFromListType(
175 GetHashSeverestThreatListType(hash
, full_hashes
, NULL
));
179 SBThreatType
SafeBrowsingDatabaseManager::GetUrlSeverestThreatType(
181 const std::vector
<SBFullHashResult
>& full_hashes
,
183 return GetThreatTypeFromListType(
184 GetUrlSeverestThreatListType(url
, full_hashes
, index
));
187 SafeBrowsingDatabaseManager::SafeBrowsingCheck::SafeBrowsingCheck(
188 const std::vector
<GURL
>& urls
,
189 const std::vector
<SBFullHash
>& full_hashes
,
191 safe_browsing_util::ListType check_type
,
192 const std::vector
<SBThreatType
>& expected_threats
)
194 url_results(urls
.size(), SB_THREAT_TYPE_SAFE
),
195 url_metadata(urls
.size()),
196 full_hashes(full_hashes
),
197 full_hash_results(full_hashes
.size(), SB_THREAT_TYPE_SAFE
),
199 need_get_hash(false),
200 check_type(check_type
),
201 expected_threats(expected_threats
) {
202 DCHECK_EQ(urls
.empty(), !full_hashes
.empty())
203 << "Exactly one of urls and full_hashes must be set";
206 SafeBrowsingDatabaseManager::SafeBrowsingCheck::~SafeBrowsingCheck() {}
208 void SafeBrowsingDatabaseManager::Client::OnSafeBrowsingResult(
209 const SafeBrowsingCheck
& check
) {
210 DCHECK_EQ(check
.urls
.size(), check
.url_results
.size());
211 DCHECK_EQ(check
.full_hashes
.size(), check
.full_hash_results
.size());
212 if (!check
.urls
.empty()) {
213 DCHECK(check
.full_hashes
.empty());
214 switch (check
.check_type
) {
215 case safe_browsing_util::MALWARE
:
216 case safe_browsing_util::PHISH
:
217 case safe_browsing_util::UNWANTEDURL
:
218 DCHECK_EQ(1u, check
.urls
.size());
219 OnCheckBrowseUrlResult(
220 check
.urls
[0], check
.url_results
[0], check
.url_metadata
[0]);
222 case safe_browsing_util::BINURL
:
223 DCHECK_EQ(check
.urls
.size(), check
.url_results
.size());
224 OnCheckDownloadUrlResult(
226 *std::max_element(check
.url_results
.begin(),
227 check
.url_results
.end()));
232 } else if (!check
.full_hashes
.empty()) {
233 switch (check
.check_type
) {
234 case safe_browsing_util::EXTENSIONBLACKLIST
: {
235 std::set
<std::string
> unsafe_extension_ids
;
236 for (size_t i
= 0; i
< check
.full_hashes
.size(); ++i
) {
237 std::string extension_id
=
238 safe_browsing_util::SBFullHashToString(check
.full_hashes
[i
]);
239 if (check
.full_hash_results
[i
] == SB_THREAT_TYPE_EXTENSION
)
240 unsafe_extension_ids
.insert(extension_id
);
242 OnCheckExtensionsResult(unsafe_extension_ids
);
253 SafeBrowsingDatabaseManager::SafeBrowsingDatabaseManager(
254 const scoped_refptr
<SafeBrowsingService
>& service
)
255 : sb_service_(service
),
258 enable_download_protection_(false),
259 enable_csd_whitelist_(false),
260 enable_download_whitelist_(false),
261 enable_extension_blacklist_(false),
262 enable_side_effect_free_whitelist_(false),
263 enable_ip_blacklist_(false),
264 enable_unwanted_software_blacklist_(false),
265 update_in_progress_(false),
266 database_update_in_progress_(false),
267 closing_database_(false),
268 check_timeout_(base::TimeDelta::FromMilliseconds(kCheckTimeoutMs
)) {
269 DCHECK(sb_service_
.get() != NULL
);
271 // Android only supports a subset of FULL_SAFE_BROWSING.
272 // TODO(shess): This shouldn't be OS-driven <http://crbug.com/394379>
273 #if !defined(OS_ANDROID)
274 CommandLine
* cmdline
= CommandLine::ForCurrentProcess();
275 enable_download_protection_
=
276 !cmdline
->HasSwitch(switches::kSbDisableDownloadProtection
);
278 // We only download the csd-whitelist if client-side phishing detection is
280 enable_csd_whitelist_
=
281 !cmdline
->HasSwitch(switches::kDisableClientSidePhishingDetection
);
283 // TODO(noelutz): remove this boolean variable since it should always be true
284 // if SafeBrowsing is enabled. Unfortunately, we have no test data for this
285 // list right now. This means that we need to be able to disable this list
286 // for the SafeBrowsing test to pass.
287 enable_download_whitelist_
= enable_csd_whitelist_
;
289 // TODO(kalman): there really shouldn't be a flag for this.
290 enable_extension_blacklist_
=
291 !cmdline
->HasSwitch(switches::kSbDisableExtensionBlacklist
);
293 enable_side_effect_free_whitelist_
=
294 prerender::IsSideEffectFreeWhitelistEnabled() &&
295 !cmdline
->HasSwitch(switches::kSbDisableSideEffectFreeWhitelist
);
297 // The client-side IP blacklist feature is tightly integrated with client-side
298 // phishing protection for now.
299 enable_ip_blacklist_
= enable_csd_whitelist_
;
301 // TODO(gab): Gate this on the same experiment that will soon control the UwS
303 enable_unwanted_software_blacklist_
= true;
305 enum SideEffectFreeWhitelistStatus
{
306 SIDE_EFFECT_FREE_WHITELIST_ENABLED
,
307 SIDE_EFFECT_FREE_WHITELIST_DISABLED
,
308 SIDE_EFFECT_FREE_WHITELIST_STATUS_MAX
311 SideEffectFreeWhitelistStatus side_effect_free_whitelist_status
=
312 enable_side_effect_free_whitelist_
? SIDE_EFFECT_FREE_WHITELIST_ENABLED
:
313 SIDE_EFFECT_FREE_WHITELIST_DISABLED
;
315 UMA_HISTOGRAM_ENUMERATION("SB2.SideEffectFreeWhitelistStatus",
316 side_effect_free_whitelist_status
,
317 SIDE_EFFECT_FREE_WHITELIST_STATUS_MAX
);
321 SafeBrowsingDatabaseManager::~SafeBrowsingDatabaseManager() {
322 // We should have already been shut down. If we're still enabled, then the
323 // database isn't going to be closed properly, which could lead to corruption.
327 bool SafeBrowsingDatabaseManager::CanCheckUrl(const GURL
& url
) const {
328 return url
.SchemeIs(url::kFtpScheme
) ||
329 url
.SchemeIs(url::kHttpScheme
) ||
330 url
.SchemeIs(url::kHttpsScheme
);
333 bool SafeBrowsingDatabaseManager::CheckDownloadUrl(
334 const std::vector
<GURL
>& url_chain
,
336 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO
));
337 if (!enabled_
|| !enable_download_protection_
)
340 // We need to check the database for url prefix, and later may fetch the url
341 // from the safebrowsing backends. These need to be asynchronous.
342 SafeBrowsingCheck
* check
=
343 new SafeBrowsingCheck(url_chain
,
344 std::vector
<SBFullHash
>(),
346 safe_browsing_util::BINURL
,
347 std::vector
<SBThreatType
>(1,
348 SB_THREAT_TYPE_BINARY_MALWARE_URL
));
349 StartSafeBrowsingCheck(
351 base::Bind(&SafeBrowsingDatabaseManager::CheckDownloadUrlOnSBThread
, this,
356 bool SafeBrowsingDatabaseManager::CheckExtensionIDs(
357 const std::set
<std::string
>& extension_ids
,
359 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO
));
361 if (!enabled_
|| !enable_extension_blacklist_
)
364 std::vector
<SBFullHash
> extension_id_hashes
;
365 std::transform(extension_ids
.begin(), extension_ids
.end(),
366 std::back_inserter(extension_id_hashes
),
367 safe_browsing_util::StringToSBFullHash
);
369 SafeBrowsingCheck
* check
= new SafeBrowsingCheck(
373 safe_browsing_util::EXTENSIONBLACKLIST
,
374 std::vector
<SBThreatType
>(1, SB_THREAT_TYPE_EXTENSION
));
376 StartSafeBrowsingCheck(
378 base::Bind(&SafeBrowsingDatabaseManager::CheckExtensionIDsOnSBThread
,
384 bool SafeBrowsingDatabaseManager::CheckSideEffectFreeWhitelistUrl(
389 if (!CanCheckUrl(url
))
392 return database_
->ContainsSideEffectFreeWhitelistUrl(url
);
395 bool SafeBrowsingDatabaseManager::MatchMalwareIP(
396 const std::string
& ip_address
) {
397 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO
));
398 if (!enabled_
|| !enable_ip_blacklist_
|| !MakeDatabaseAvailable()) {
399 return false; // Fail open.
401 return database_
->ContainsMalwareIP(ip_address
);
404 bool SafeBrowsingDatabaseManager::MatchCsdWhitelistUrl(const GURL
& url
) {
405 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO
));
406 if (!enabled_
|| !enable_csd_whitelist_
|| !MakeDatabaseAvailable()) {
407 // There is something funky going on here -- for example, perhaps the user
408 // has not restarted since enabling metrics reporting, so we haven't
409 // enabled the csd whitelist yet. Just to be safe we return true in this
413 return database_
->ContainsCsdWhitelistedUrl(url
);
416 bool SafeBrowsingDatabaseManager::MatchDownloadWhitelistUrl(const GURL
& url
) {
417 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO
));
418 if (!enabled_
|| !enable_download_whitelist_
|| !MakeDatabaseAvailable()) {
421 return database_
->ContainsDownloadWhitelistedUrl(url
);
424 bool SafeBrowsingDatabaseManager::MatchDownloadWhitelistString(
425 const std::string
& str
) {
426 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO
));
427 if (!enabled_
|| !enable_download_whitelist_
|| !MakeDatabaseAvailable()) {
430 return database_
->ContainsDownloadWhitelistedString(str
);
433 bool SafeBrowsingDatabaseManager::IsMalwareKillSwitchOn() {
434 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO
));
435 if (!enabled_
|| !MakeDatabaseAvailable()) {
438 return database_
->IsMalwareIPMatchKillSwitchOn();
441 bool SafeBrowsingDatabaseManager::IsCsdWhitelistKillSwitchOn() {
442 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO
));
443 if (!enabled_
|| !MakeDatabaseAvailable()) {
446 return database_
->IsCsdWhitelistKillSwitchOn();
449 bool SafeBrowsingDatabaseManager::CheckBrowseUrl(const GURL
& url
,
451 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO
));
455 if (!CanCheckUrl(url
))
458 std::vector
<SBThreatType
> expected_threats
;
459 expected_threats
.push_back(SB_THREAT_TYPE_URL_MALWARE
);
460 expected_threats
.push_back(SB_THREAT_TYPE_URL_PHISHING
);
461 expected_threats
.push_back(SB_THREAT_TYPE_URL_UNWANTED
);
463 const base::TimeTicks start
= base::TimeTicks::Now();
464 if (!MakeDatabaseAvailable()) {
465 QueuedCheck
queued_check(safe_browsing_util::MALWARE
, // or PHISH
470 queued_checks_
.push_back(queued_check
);
474 // Cache hits should, in general, be the same for both (ignoring potential
475 // cache evictions in the second call for entries that were just about to be
476 // evicted in the first call).
477 // TODO(gab): Refactor SafeBrowsingDatabase to avoid depending on this here.
478 std::vector
<SBFullHashResult
> cache_hits
;
480 std::vector
<SBPrefix
> browse_prefix_hits
;
481 bool browse_prefix_match
= database_
->ContainsBrowseUrl(
482 url
, &browse_prefix_hits
, &cache_hits
);
484 std::vector
<SBPrefix
> unwanted_prefix_hits
;
485 std::vector
<SBFullHashResult
> unused_cache_hits
;
486 bool unwanted_prefix_match
= database_
->ContainsUnwantedSoftwareUrl(
487 url
, &unwanted_prefix_hits
, &unused_cache_hits
);
489 // Merge the two pre-sorted prefix hits lists.
490 // TODO(gab): Refactor SafeBrowsingDatabase for it to return this merged list
491 // by default rather than building it here.
492 std::vector
<SBPrefix
> prefix_hits(browse_prefix_hits
.size() +
493 unwanted_prefix_hits
.size());
494 std::merge(browse_prefix_hits
.begin(),
495 browse_prefix_hits
.end(),
496 unwanted_prefix_hits
.begin(),
497 unwanted_prefix_hits
.end(),
498 prefix_hits
.begin());
499 prefix_hits
.erase(std::unique(prefix_hits
.begin(), prefix_hits
.end()),
502 UMA_HISTOGRAM_TIMES("SB2.FilterCheck", base::TimeTicks::Now() - start
);
504 if (!browse_prefix_match
&& !unwanted_prefix_match
)
505 return true; // URL is okay.
507 // Needs to be asynchronous, since we could be in the constructor of a
508 // ResourceDispatcherHost event handler which can't pause there.
509 // This check will ping the Safe Browsing servers and get all lists which it
510 // matches. These lists will then be filtered against the |expected_threats|
511 // and the result callback for MALWARE (which is the same as for PHISH and
512 // UNWANTEDURL) will eventually be invoked with the final decision.
513 SafeBrowsingCheck
* check
= new SafeBrowsingCheck(std::vector
<GURL
>(1, url
),
514 std::vector
<SBFullHash
>(),
516 safe_browsing_util::MALWARE
,
518 check
->need_get_hash
= cache_hits
.empty();
519 check
->prefix_hits
.swap(prefix_hits
);
520 check
->cache_hits
.swap(cache_hits
);
521 checks_
.insert(check
);
523 BrowserThread::PostTask(
524 BrowserThread::IO
, FROM_HERE
,
525 base::Bind(&SafeBrowsingDatabaseManager::OnCheckDone
, this, check
));
530 void SafeBrowsingDatabaseManager::CancelCheck(Client
* client
) {
531 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO
));
532 for (CurrentChecks::iterator i
= checks_
.begin(); i
!= checks_
.end(); ++i
) {
533 // We can't delete matching checks here because the db thread has a copy of
534 // the pointer. Instead, we simply NULL out the client, and when the db
535 // thread calls us back, we'll clean up the check.
536 if ((*i
)->client
== client
)
540 // Scan the queued clients store. Clients may be here if they requested a URL
541 // check before the database has finished loading.
542 for (std::deque
<QueuedCheck
>::iterator
it(queued_checks_
.begin());
543 it
!= queued_checks_
.end(); ) {
544 // In this case it's safe to delete matches entirely since nothing has a
546 if (it
->client
== client
)
547 it
= queued_checks_
.erase(it
);
553 void SafeBrowsingDatabaseManager::HandleGetHashResults(
554 SafeBrowsingCheck
* check
,
555 const std::vector
<SBFullHashResult
>& full_hashes
,
556 const base::TimeDelta
& cache_lifetime
) {
557 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO
));
562 // If the service has been shut down, |check| should have been deleted.
563 DCHECK(checks_
.find(check
) != checks_
.end());
565 // |start| is set before calling |GetFullHash()|, which should be
566 // the only path which gets to here.
567 DCHECK(!check
->start
.is_null());
568 UMA_HISTOGRAM_LONG_TIMES("SB2.Network",
569 base::TimeTicks::Now() - check
->start
);
571 std::vector
<SBPrefix
> prefixes
= check
->prefix_hits
;
572 OnHandleGetHashResults(check
, full_hashes
); // 'check' is deleted here.
574 // Cache the GetHash results.
575 if (cache_lifetime
!= base::TimeDelta() && MakeDatabaseAvailable())
576 database_
->CacheHashResults(prefixes
, full_hashes
, cache_lifetime
);
579 void SafeBrowsingDatabaseManager::GetChunks(GetChunksCallback callback
) {
580 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO
));
582 DCHECK(!callback
.is_null());
583 safe_browsing_thread_
->message_loop()->PostTask(FROM_HERE
, base::Bind(
584 &SafeBrowsingDatabaseManager::GetAllChunksFromDatabase
, this, callback
));
587 void SafeBrowsingDatabaseManager::AddChunks(
588 const std::string
& list
,
589 scoped_ptr
<ScopedVector
<SBChunkData
> > chunks
,
590 AddChunksCallback callback
) {
591 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO
));
593 DCHECK(!callback
.is_null());
594 safe_browsing_thread_
->message_loop()->PostTask(FROM_HERE
, base::Bind(
595 &SafeBrowsingDatabaseManager::AddDatabaseChunks
, this, list
,
596 base::Passed(&chunks
), callback
));
599 void SafeBrowsingDatabaseManager::DeleteChunks(
600 scoped_ptr
<std::vector
<SBChunkDelete
> > chunk_deletes
) {
601 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO
));
603 safe_browsing_thread_
->message_loop()->PostTask(FROM_HERE
, base::Bind(
604 &SafeBrowsingDatabaseManager::DeleteDatabaseChunks
, this,
605 base::Passed(&chunk_deletes
)));
608 void SafeBrowsingDatabaseManager::UpdateStarted() {
609 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO
));
611 DCHECK(!update_in_progress_
);
612 update_in_progress_
= true;
615 void SafeBrowsingDatabaseManager::UpdateFinished(bool update_succeeded
) {
616 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO
));
618 if (update_in_progress_
) {
619 update_in_progress_
= false;
620 safe_browsing_thread_
->message_loop()->PostTask(FROM_HERE
,
621 base::Bind(&SafeBrowsingDatabaseManager::DatabaseUpdateFinished
,
622 this, update_succeeded
));
626 void SafeBrowsingDatabaseManager::ResetDatabase() {
627 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO
));
629 safe_browsing_thread_
->message_loop()->PostTask(FROM_HERE
, base::Bind(
630 &SafeBrowsingDatabaseManager::OnResetDatabase
, this));
633 void SafeBrowsingDatabaseManager::StartOnIOThread() {
634 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO
));
638 DCHECK(!safe_browsing_thread_
.get());
639 safe_browsing_thread_
.reset(new base::Thread("Chrome_SafeBrowsingThread"));
640 if (!safe_browsing_thread_
->Start())
644 MakeDatabaseAvailable();
647 void SafeBrowsingDatabaseManager::StopOnIOThread(bool shutdown
) {
648 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO
));
656 void SafeBrowsingDatabaseManager::NotifyDatabaseUpdateFinished(
657 bool update_succeeded
) {
658 content::NotificationService::current()->Notify(
659 chrome::NOTIFICATION_SAFE_BROWSING_UPDATE_COMPLETE
,
660 content::Source
<SafeBrowsingDatabaseManager
>(this),
661 content::Details
<bool>(&update_succeeded
));
664 SafeBrowsingDatabaseManager::QueuedCheck::QueuedCheck(
665 const safe_browsing_util::ListType check_type
,
668 const std::vector
<SBThreatType
>& expected_threats
,
669 const base::TimeTicks
& start
)
670 : check_type(check_type
),
673 expected_threats(expected_threats
),
677 SafeBrowsingDatabaseManager::QueuedCheck::~QueuedCheck() {
680 void SafeBrowsingDatabaseManager::DoStopOnIOThread() {
686 // Delete queued checks, calling back any clients with 'SB_THREAT_TYPE_SAFE'.
687 while (!queued_checks_
.empty()) {
688 QueuedCheck queued
= queued_checks_
.front();
690 SafeBrowsingCheck
sb_check(std::vector
<GURL
>(1, queued
.url
),
691 std::vector
<SBFullHash
>(),
694 queued
.expected_threats
);
695 queued
.client
->OnSafeBrowsingResult(sb_check
);
697 queued_checks_
.pop_front();
700 // Close the database. Cases to avoid:
701 // * If |closing_database_| is true, continuing will queue up a second
702 // request, |closing_database_| will be reset after handling the first
703 // request, and if any functions on the db thread recreate the database, we
704 // could start using it on the IO thread and then have the second request
705 // handler delete it out from under us.
706 // * If |database_| is NULL, then either no creation request is in flight, in
707 // which case we don't need to do anything, or one is in flight, in which
708 // case the database will be recreated before our deletion request is
709 // handled, and could be used on the IO thread in that time period, leading
710 // to the same problem as above.
711 // Checking DatabaseAvailable() avoids both of these.
712 if (DatabaseAvailable()) {
713 closing_database_
= true;
714 safe_browsing_thread_
->message_loop()->PostTask(FROM_HERE
,
715 base::Bind(&SafeBrowsingDatabaseManager::OnCloseDatabase
, this));
718 // Flush the database thread. Any in-progress database check results will be
719 // ignored and cleaned up below.
721 // Note that to avoid leaking the database, we rely on the fact that no new
722 // tasks will be added to the db thread between the call above and this one.
723 // See comments on the declaration of |safe_browsing_thread_|.
725 // A ScopedAllowIO object is required to join the thread when calling Stop.
726 // See http://crbug.com/72696. Note that we call Stop() first to clear out
727 // any remaining tasks before clearing safe_browsing_thread_.
728 base::ThreadRestrictions::ScopedAllowIO allow_io_for_thread_join
;
729 safe_browsing_thread_
->Stop();
730 safe_browsing_thread_
.reset();
733 // Delete pending checks, calling back any clients with 'SB_THREAT_TYPE_SAFE'.
734 // We have to do this after the db thread returns because methods on it can
735 // have copies of these pointers, so deleting them might lead to accessing
737 for (CurrentChecks::iterator it
= checks_
.begin();
738 it
!= checks_
.end(); ++it
) {
739 SafeBrowsingCheck
* check
= *it
;
741 check
->client
->OnSafeBrowsingResult(*check
);
743 STLDeleteElements(&checks_
);
745 gethash_requests_
.clear();
748 bool SafeBrowsingDatabaseManager::DatabaseAvailable() const {
749 base::AutoLock
lock(database_lock_
);
750 return !closing_database_
&& (database_
!= NULL
);
753 bool SafeBrowsingDatabaseManager::MakeDatabaseAvailable() {
754 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO
));
756 if (DatabaseAvailable())
758 safe_browsing_thread_
->message_loop()->PostTask(
760 base::Bind(base::IgnoreResult(&SafeBrowsingDatabaseManager::GetDatabase
),
765 SafeBrowsingDatabase
* SafeBrowsingDatabaseManager::GetDatabase() {
766 DCHECK_EQ(base::MessageLoop::current(),
767 safe_browsing_thread_
->message_loop());
770 startup_metric_utils::ScopedSlowStartupUMA
771 scoped_timer("Startup.SlowStartupSafeBrowsingGetDatabase");
772 const base::TimeTicks before
= base::TimeTicks::Now();
774 SafeBrowsingDatabase
* database
=
775 SafeBrowsingDatabase::Create(enable_download_protection_
,
776 enable_csd_whitelist_
,
777 enable_download_whitelist_
,
778 enable_extension_blacklist_
,
779 enable_side_effect_free_whitelist_
,
780 enable_ip_blacklist_
,
781 enable_unwanted_software_blacklist_
);
783 database
->Init(SafeBrowsingService::GetBaseFilename());
785 // Acquiring the lock here guarantees correct ordering between the writes to
786 // the new database object above, and the setting of |databse_| below.
787 base::AutoLock
lock(database_lock_
);
788 database_
= database
;
791 BrowserThread::PostTask(
792 BrowserThread::IO
, FROM_HERE
,
793 base::Bind(&SafeBrowsingDatabaseManager::DatabaseLoadComplete
, this));
795 UMA_HISTOGRAM_TIMES("SB2.DatabaseOpen", base::TimeTicks::Now() - before
);
799 void SafeBrowsingDatabaseManager::OnCheckDone(SafeBrowsingCheck
* check
) {
800 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO
));
805 // If the service has been shut down, |check| should have been deleted.
806 DCHECK(checks_
.find(check
) != checks_
.end());
808 if (check
->client
&& check
->need_get_hash
) {
809 // We have a partial match so we need to query Google for the full hash.
810 // Clean up will happen in HandleGetHashResults.
812 // See if we have a GetHash request already in progress for this particular
813 // prefix. If so, we just append ourselves to the list of interested parties
814 // when the results arrive. We only do this for checks involving one prefix,
815 // since that is the common case (multiple prefixes will issue the request
817 if (check
->prefix_hits
.size() == 1) {
818 SBPrefix prefix
= check
->prefix_hits
[0];
819 GetHashRequests::iterator it
= gethash_requests_
.find(prefix
);
820 if (it
!= gethash_requests_
.end()) {
821 // There's already a request in progress.
822 it
->second
.push_back(check
);
826 // No request in progress, so we're the first for this prefix.
827 GetHashRequestors requestors
;
828 requestors
.push_back(check
);
829 gethash_requests_
[prefix
] = requestors
;
832 // Reset the start time so that we can measure the network time without the
834 check
->start
= base::TimeTicks::Now();
835 // Note: If |this| is deleted or stopped, the protocol_manager will
836 // be destroyed as well - hence it's OK to do unretained in this case.
837 bool is_download
= check
->check_type
== safe_browsing_util::BINURL
;
838 sb_service_
->protocol_manager()->GetFullHash(
840 base::Bind(&SafeBrowsingDatabaseManager::HandleGetHashResults
,
841 base::Unretained(this),
845 // We may have cached results for previous GetHash queries. Since
846 // this data comes from cache, don't histogram hits.
847 HandleOneCheck(check
, check
->cache_hits
);
851 void SafeBrowsingDatabaseManager::GetAllChunksFromDatabase(
852 GetChunksCallback callback
) {
853 DCHECK_EQ(base::MessageLoop::current(),
854 safe_browsing_thread_
->message_loop());
856 bool database_error
= true;
857 std::vector
<SBListChunkRanges
> lists
;
858 DCHECK(!database_update_in_progress_
);
859 database_update_in_progress_
= true;
860 GetDatabase(); // This guarantees that |database_| is non-NULL.
861 if (database_
->UpdateStarted(&lists
)) {
862 database_error
= false;
864 database_
->UpdateFinished(false);
867 BrowserThread::PostTask(
868 BrowserThread::IO
, FROM_HERE
,
869 base::Bind(&SafeBrowsingDatabaseManager::OnGetAllChunksFromDatabase
,
870 this, lists
, database_error
, callback
));
873 void SafeBrowsingDatabaseManager::OnGetAllChunksFromDatabase(
874 const std::vector
<SBListChunkRanges
>& lists
, bool database_error
,
875 GetChunksCallback callback
) {
876 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO
));
878 callback
.Run(lists
, database_error
);
881 void SafeBrowsingDatabaseManager::OnAddChunksComplete(
882 AddChunksCallback callback
) {
883 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO
));
888 void SafeBrowsingDatabaseManager::DatabaseLoadComplete() {
889 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO
));
893 LOCAL_HISTOGRAM_COUNTS("SB.QueueDepth", queued_checks_
.size());
894 if (queued_checks_
.empty())
897 // If the database isn't already available, calling CheckUrl() in the loop
898 // below will add the check back to the queue, and we'll infinite-loop.
899 DCHECK(DatabaseAvailable());
900 while (!queued_checks_
.empty()) {
901 QueuedCheck check
= queued_checks_
.front();
902 DCHECK(!check
.start
.is_null());
903 LOCAL_HISTOGRAM_TIMES("SB.QueueDelay",
904 base::TimeTicks::Now() - check
.start
);
905 // If CheckUrl() determines the URL is safe immediately, it doesn't call the
906 // client's handler function (because normally it's being directly called by
907 // the client). Since we're not the client, we have to convey this result.
908 if (check
.client
&& CheckBrowseUrl(check
.url
, check
.client
)) {
909 SafeBrowsingCheck
sb_check(std::vector
<GURL
>(1, check
.url
),
910 std::vector
<SBFullHash
>(),
913 check
.expected_threats
);
914 check
.client
->OnSafeBrowsingResult(sb_check
);
916 queued_checks_
.pop_front();
920 void SafeBrowsingDatabaseManager::AddDatabaseChunks(
921 const std::string
& list_name
,
922 scoped_ptr
<ScopedVector
<SBChunkData
> > chunks
,
923 AddChunksCallback callback
) {
924 DCHECK_EQ(base::MessageLoop::current(),
925 safe_browsing_thread_
->message_loop());
927 GetDatabase()->InsertChunks(list_name
, chunks
->get());
928 BrowserThread::PostTask(
929 BrowserThread::IO
, FROM_HERE
,
930 base::Bind(&SafeBrowsingDatabaseManager::OnAddChunksComplete
, this,
934 void SafeBrowsingDatabaseManager::DeleteDatabaseChunks(
935 scoped_ptr
<std::vector
<SBChunkDelete
> > chunk_deletes
) {
936 DCHECK_EQ(base::MessageLoop::current(),
937 safe_browsing_thread_
->message_loop());
939 GetDatabase()->DeleteChunks(*chunk_deletes
);
942 void SafeBrowsingDatabaseManager::DatabaseUpdateFinished(
943 bool update_succeeded
) {
944 DCHECK_EQ(base::MessageLoop::current(),
945 safe_browsing_thread_
->message_loop());
946 GetDatabase()->UpdateFinished(update_succeeded
);
947 DCHECK(database_update_in_progress_
);
948 database_update_in_progress_
= false;
949 BrowserThread::PostTask(
950 BrowserThread::UI
, FROM_HERE
,
951 base::Bind(&SafeBrowsingDatabaseManager::NotifyDatabaseUpdateFinished
,
952 this, update_succeeded
));
955 void SafeBrowsingDatabaseManager::OnCloseDatabase() {
956 DCHECK_EQ(base::MessageLoop::current(),
957 safe_browsing_thread_
->message_loop());
958 DCHECK(closing_database_
);
960 // Because |closing_database_| is true, nothing on the IO thread will be
961 // accessing the database, so it's safe to delete and then NULL the pointer.
965 // Acquiring the lock here guarantees correct ordering between the resetting
966 // of |database_| above and of |closing_database_| below, which ensures there
967 // won't be a window during which the IO thread falsely believes the database
969 base::AutoLock
lock(database_lock_
);
970 closing_database_
= false;
973 void SafeBrowsingDatabaseManager::OnResetDatabase() {
974 DCHECK_EQ(base::MessageLoop::current(),
975 safe_browsing_thread_
->message_loop());
976 GetDatabase()->ResetDatabase();
979 void SafeBrowsingDatabaseManager::OnHandleGetHashResults(
980 SafeBrowsingCheck
* check
,
981 const std::vector
<SBFullHashResult
>& full_hashes
) {
982 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO
));
983 safe_browsing_util::ListType check_type
= check
->check_type
;
984 SBPrefix prefix
= check
->prefix_hits
[0];
985 GetHashRequests::iterator it
= gethash_requests_
.find(prefix
);
986 if (check
->prefix_hits
.size() > 1 || it
== gethash_requests_
.end()) {
987 const bool hit
= HandleOneCheck(check
, full_hashes
);
988 RecordGetHashCheckStatus(hit
, check_type
, full_hashes
);
992 // Call back all interested parties, noting if any has a hit.
993 GetHashRequestors
& requestors
= it
->second
;
995 for (GetHashRequestors::iterator r
= requestors
.begin();
996 r
!= requestors
.end(); ++r
) {
997 if (HandleOneCheck(*r
, full_hashes
))
1000 RecordGetHashCheckStatus(hit
, check_type
, full_hashes
);
1002 gethash_requests_
.erase(it
);
1005 bool SafeBrowsingDatabaseManager::HandleOneCheck(
1006 SafeBrowsingCheck
* check
,
1007 const std::vector
<SBFullHashResult
>& full_hashes
) {
1008 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO
));
1011 bool is_threat
= false;
1013 // TODO(shess): GetHashSeverestThreadListType() contains a loop,
1014 // GetUrlSeverestThreatListType() a loop around that loop. Having another
1015 // loop out here concerns me. It is likely that SAFE is an expected outcome,
1016 // which means all of those loops run to completion. Refactoring this to
1017 // generate a set of sorted items to compare in sequence would probably
1020 // Additionally, the set of patterns generated from the urls is very similar
1021 // to the patterns generated in ContainsBrowseUrl() and other database checks,
1022 // which are called from this code. Refactoring that across the checks could
1023 // interact well with batching the checks here.
1025 // TODO(gab): Fix the fact that Get(Url|Hash)SeverestThreatType() may return a
1026 // threat for which IsExpectedThreat() returns false even if |full_hashes|
1027 // actually contains an expected threat.
1029 for (size_t i
= 0; i
< check
->urls
.size(); ++i
) {
1030 size_t threat_index
;
1031 SBThreatType threat
=
1032 GetUrlSeverestThreatType(check
->urls
[i
], full_hashes
, &threat_index
);
1033 if (threat
!= SB_THREAT_TYPE_SAFE
&&
1034 IsExpectedThreat(threat
, check
->expected_threats
)) {
1035 check
->url_results
[i
] = threat
;
1036 check
->url_metadata
[i
] = full_hashes
[threat_index
].metadata
;
1041 for (size_t i
= 0; i
< check
->full_hashes
.size(); ++i
) {
1042 SBThreatType threat
=
1043 GetHashSeverestThreatType(check
->full_hashes
[i
], full_hashes
);
1044 if (threat
!= SB_THREAT_TYPE_SAFE
&&
1045 IsExpectedThreat(threat
, check
->expected_threats
)) {
1046 check
->full_hash_results
[i
] = threat
;
1051 SafeBrowsingCheckDone(check
);
1055 void SafeBrowsingDatabaseManager::CheckDownloadUrlOnSBThread(
1056 SafeBrowsingCheck
* check
) {
1057 DCHECK_EQ(base::MessageLoop::current(),
1058 safe_browsing_thread_
->message_loop());
1059 DCHECK(enable_download_protection_
);
1061 std::vector
<SBPrefix
> prefix_hits
;
1063 if (!database_
->ContainsDownloadUrl(check
->urls
, &prefix_hits
)) {
1064 // Good, we don't have hash for this url prefix.
1065 BrowserThread::PostTask(
1066 BrowserThread::IO
, FROM_HERE
,
1067 base::Bind(&SafeBrowsingDatabaseManager::CheckDownloadUrlDone
, this,
1072 check
->need_get_hash
= true;
1073 check
->prefix_hits
.clear();
1074 check
->prefix_hits
= prefix_hits
;
1075 BrowserThread::PostTask(
1076 BrowserThread::IO
, FROM_HERE
,
1077 base::Bind(&SafeBrowsingDatabaseManager::OnCheckDone
, this, check
));
1080 void SafeBrowsingDatabaseManager::CheckExtensionIDsOnSBThread(
1081 SafeBrowsingCheck
* check
) {
1082 DCHECK_EQ(base::MessageLoop::current(),
1083 safe_browsing_thread_
->message_loop());
1085 std::vector
<SBPrefix
> prefixes
;
1086 for (std::vector
<SBFullHash
>::iterator it
= check
->full_hashes
.begin();
1087 it
!= check
->full_hashes
.end(); ++it
) {
1088 prefixes
.push_back((*it
).prefix
);
1090 database_
->ContainsExtensionPrefixes(prefixes
, &check
->prefix_hits
);
1092 if (check
->prefix_hits
.empty()) {
1093 // No matches for any extensions.
1094 BrowserThread::PostTask(
1097 base::Bind(&SafeBrowsingDatabaseManager::SafeBrowsingCheckDone
, this,
1100 // Some prefixes matched, we need to ask Google whether they're legit.
1101 check
->need_get_hash
= true;
1102 BrowserThread::PostTask(
1105 base::Bind(&SafeBrowsingDatabaseManager::OnCheckDone
, this, check
));
1109 void SafeBrowsingDatabaseManager::TimeoutCallback(SafeBrowsingCheck
* check
) {
1110 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO
));
1116 DCHECK(checks_
.find(check
) != checks_
.end());
1117 if (check
->client
) {
1118 check
->client
->OnSafeBrowsingResult(*check
);
1119 check
->client
= NULL
;
1123 void SafeBrowsingDatabaseManager::CheckDownloadUrlDone(
1124 SafeBrowsingCheck
* check
) {
1125 DCHECK(enable_download_protection_
);
1126 SafeBrowsingCheckDone(check
);
1129 void SafeBrowsingDatabaseManager::SafeBrowsingCheckDone(
1130 SafeBrowsingCheck
* check
) {
1131 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO
));
1137 DVLOG(1) << "SafeBrowsingCheckDone";
1138 DCHECK(checks_
.find(check
) != checks_
.end());
1140 check
->client
->OnSafeBrowsingResult(*check
);
1141 checks_
.erase(check
);
1145 void SafeBrowsingDatabaseManager::StartSafeBrowsingCheck(
1146 SafeBrowsingCheck
* check
,
1147 const base::Closure
& task
) {
1148 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO
));
1149 check
->timeout_factory_
.reset(
1150 new base::WeakPtrFactory
<SafeBrowsingDatabaseManager
>(this));
1151 checks_
.insert(check
);
1153 safe_browsing_thread_
->message_loop()->PostTask(FROM_HERE
, task
);
1155 base::MessageLoop::current()->PostDelayedTask(FROM_HERE
,
1156 base::Bind(&SafeBrowsingDatabaseManager::TimeoutCallback
,
1157 check
->timeout_factory_
->GetWeakPtr(), check
),