Roll src/third_party/WebKit 8b42d1d:744641d (svn 186770:186771)
[chromium-blink-merge.git] / chrome / browser / safe_browsing / database_manager.cc
blob583d11af514163cc57490e58e2f6523a1debbf10
1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "chrome/browser/safe_browsing/database_manager.h"
7 #include <algorithm>
9 #include "base/bind.h"
10 #include "base/bind_helpers.h"
11 #include "base/callback.h"
12 #include "base/command_line.h"
13 #include "base/debug/leak_tracker.h"
14 #include "base/stl_util.h"
15 #include "base/strings/string_util.h"
16 #include "base/threading/thread.h"
17 #include "base/threading/thread_restrictions.h"
18 #include "chrome/browser/browser_process.h"
19 #include "chrome/browser/chrome_notification_types.h"
20 #include "chrome/browser/prerender/prerender_field_trial.h"
21 #include "chrome/browser/safe_browsing/client_side_detection_service.h"
22 #include "chrome/browser/safe_browsing/download_protection_service.h"
23 #include "chrome/browser/safe_browsing/malware_details.h"
24 #include "chrome/browser/safe_browsing/protocol_manager.h"
25 #include "chrome/browser/safe_browsing/safe_browsing_database.h"
26 #include "chrome/browser/safe_browsing/safe_browsing_service.h"
27 #include "chrome/browser/safe_browsing/ui_manager.h"
28 #include "chrome/common/chrome_constants.h"
29 #include "chrome/common/chrome_paths.h"
30 #include "chrome/common/chrome_switches.h"
31 #include "components/metrics/metrics_service.h"
32 #include "components/startup_metric_utils/startup_metric_utils.h"
33 #include "content/public/browser/browser_thread.h"
34 #include "content/public/browser/notification_service.h"
35 #include "url/url_constants.h"
37 using content::BrowserThread;
39 namespace {
41 // Timeout for match checks, e.g. download URLs, hashes.
42 const int kCheckTimeoutMs = 10000;
44 // Records disposition information about the check. |hit| should be
45 // |true| if there were any prefix hits in |full_hashes|.
46 void RecordGetHashCheckStatus(
47 bool hit,
48 safe_browsing_util::ListType check_type,
49 const std::vector<SBFullHashResult>& full_hashes) {
50 SafeBrowsingProtocolManager::ResultType result;
51 if (full_hashes.empty()) {
52 result = SafeBrowsingProtocolManager::GET_HASH_FULL_HASH_EMPTY;
53 } else if (hit) {
54 result = SafeBrowsingProtocolManager::GET_HASH_FULL_HASH_HIT;
55 } else {
56 result = SafeBrowsingProtocolManager::GET_HASH_FULL_HASH_MISS;
58 bool is_download = check_type == safe_browsing_util::BINURL;
59 SafeBrowsingProtocolManager::RecordGetHashResult(is_download, result);
62 bool IsExpectedThreat(
63 const SBThreatType threat_type,
64 const std::vector<SBThreatType>& expected_threats) {
65 return expected_threats.end() != std::find(expected_threats.begin(),
66 expected_threats.end(),
67 threat_type);
70 // Return the severest list id from the results in |full_hashes| which matches
71 // |hash|, or INVALID if none match.
72 safe_browsing_util::ListType GetHashSeverestThreatListType(
73 const SBFullHash& hash,
74 const std::vector<SBFullHashResult>& full_hashes,
75 size_t* index) {
76 safe_browsing_util::ListType pending_threat = safe_browsing_util::INVALID;
77 for (size_t i = 0; i < full_hashes.size(); ++i) {
78 if (SBFullHashEqual(hash, full_hashes[i].hash)) {
79 const safe_browsing_util::ListType threat =
80 static_cast<safe_browsing_util::ListType>(full_hashes[i].list_id);
81 switch (threat) {
82 case safe_browsing_util::INVALID:
83 // |full_hashes| should never contain INVALID as a |list_id|.
84 NOTREACHED();
85 break;
86 case safe_browsing_util::MALWARE: // Falls through.
87 case safe_browsing_util::PHISH: // Falls through.
88 case safe_browsing_util::BINURL: // Falls through.
89 case safe_browsing_util::CSDWHITELIST: // Falls through.
90 case safe_browsing_util::DOWNLOADWHITELIST: // Falls through.
91 case safe_browsing_util::EXTENSIONBLACKLIST: // Falls through.
92 case safe_browsing_util::SIDEEFFECTFREEWHITELIST: // Falls through.
93 case safe_browsing_util::IPBLACKLIST:
94 if (index)
95 *index = i;
96 return threat;
97 case safe_browsing_util::UNWANTEDURL:
98 // UNWANTEDURL is considered less severe than other threats, keep
99 // looking.
100 pending_threat = threat;
101 if (index)
102 *index = i;
103 break;
107 return pending_threat;
110 // Given a URL, compare all the possible host + path full hashes to the set of
111 // provided full hashes. Returns the list id of the severest matching result
112 // from |full_hashes|, or INVALID if none match.
113 safe_browsing_util::ListType GetUrlSeverestThreatListType(
114 const GURL& url,
115 const std::vector<SBFullHashResult>& full_hashes,
116 size_t* index) {
117 if (full_hashes.empty())
118 return safe_browsing_util::INVALID;
120 std::vector<std::string> patterns;
121 safe_browsing_util::GeneratePatternsToCheck(url, &patterns);
123 safe_browsing_util::ListType pending_threat = safe_browsing_util::INVALID;
124 for (size_t i = 0; i < patterns.size(); ++i) {
125 safe_browsing_util::ListType threat = GetHashSeverestThreatListType(
126 SBFullHashForString(patterns[i]), full_hashes, index);
127 switch (threat) {
128 case safe_browsing_util::INVALID:
129 // Ignore patterns with no matching threat.
130 break;
131 case safe_browsing_util::MALWARE: // Falls through.
132 case safe_browsing_util::PHISH: // Falls through.
133 case safe_browsing_util::BINURL: // Falls through.
134 case safe_browsing_util::CSDWHITELIST: // Falls through.
135 case safe_browsing_util::DOWNLOADWHITELIST: // Falls through.
136 case safe_browsing_util::EXTENSIONBLACKLIST: // Falls through.
137 case safe_browsing_util::SIDEEFFECTFREEWHITELIST: // Falls through.
138 case safe_browsing_util::IPBLACKLIST:
139 return threat;
140 case safe_browsing_util::UNWANTEDURL:
141 // UNWANTEDURL is considered less severe than other threats, keep
142 // looking.
143 pending_threat = threat;
144 break;
147 return pending_threat;
150 SBThreatType GetThreatTypeFromListType(safe_browsing_util::ListType list_type) {
151 switch (list_type) {
152 case safe_browsing_util::PHISH:
153 return SB_THREAT_TYPE_URL_PHISHING;
154 case safe_browsing_util::MALWARE:
155 return SB_THREAT_TYPE_URL_MALWARE;
156 case safe_browsing_util::UNWANTEDURL:
157 return SB_THREAT_TYPE_URL_UNWANTED;
158 case safe_browsing_util::BINURL:
159 return SB_THREAT_TYPE_BINARY_MALWARE_URL;
160 case safe_browsing_util::EXTENSIONBLACKLIST:
161 return SB_THREAT_TYPE_EXTENSION;
162 default:
163 DVLOG(1) << "Unknown safe browsing list id " << list_type;
164 return SB_THREAT_TYPE_SAFE;
168 } // namespace
170 // static
171 SBThreatType SafeBrowsingDatabaseManager::GetHashSeverestThreatType(
172 const SBFullHash& hash,
173 const std::vector<SBFullHashResult>& full_hashes) {
174 return GetThreatTypeFromListType(
175 GetHashSeverestThreatListType(hash, full_hashes, NULL));
178 // static
179 SBThreatType SafeBrowsingDatabaseManager::GetUrlSeverestThreatType(
180 const GURL& url,
181 const std::vector<SBFullHashResult>& full_hashes,
182 size_t* index) {
183 return GetThreatTypeFromListType(
184 GetUrlSeverestThreatListType(url, full_hashes, index));
187 SafeBrowsingDatabaseManager::SafeBrowsingCheck::SafeBrowsingCheck(
188 const std::vector<GURL>& urls,
189 const std::vector<SBFullHash>& full_hashes,
190 Client* client,
191 safe_browsing_util::ListType check_type,
192 const std::vector<SBThreatType>& expected_threats)
193 : urls(urls),
194 url_results(urls.size(), SB_THREAT_TYPE_SAFE),
195 url_metadata(urls.size()),
196 full_hashes(full_hashes),
197 full_hash_results(full_hashes.size(), SB_THREAT_TYPE_SAFE),
198 client(client),
199 need_get_hash(false),
200 check_type(check_type),
201 expected_threats(expected_threats) {
202 DCHECK_EQ(urls.empty(), !full_hashes.empty())
203 << "Exactly one of urls and full_hashes must be set";
206 SafeBrowsingDatabaseManager::SafeBrowsingCheck::~SafeBrowsingCheck() {}
208 void SafeBrowsingDatabaseManager::Client::OnSafeBrowsingResult(
209 const SafeBrowsingCheck& check) {
210 DCHECK_EQ(check.urls.size(), check.url_results.size());
211 DCHECK_EQ(check.full_hashes.size(), check.full_hash_results.size());
212 if (!check.urls.empty()) {
213 DCHECK(check.full_hashes.empty());
214 switch (check.check_type) {
215 case safe_browsing_util::MALWARE:
216 case safe_browsing_util::PHISH:
217 case safe_browsing_util::UNWANTEDURL:
218 DCHECK_EQ(1u, check.urls.size());
219 OnCheckBrowseUrlResult(
220 check.urls[0], check.url_results[0], check.url_metadata[0]);
221 break;
222 case safe_browsing_util::BINURL:
223 DCHECK_EQ(check.urls.size(), check.url_results.size());
224 OnCheckDownloadUrlResult(
225 check.urls,
226 *std::max_element(check.url_results.begin(),
227 check.url_results.end()));
228 break;
229 default:
230 NOTREACHED();
232 } else if (!check.full_hashes.empty()) {
233 switch (check.check_type) {
234 case safe_browsing_util::EXTENSIONBLACKLIST: {
235 std::set<std::string> unsafe_extension_ids;
236 for (size_t i = 0; i < check.full_hashes.size(); ++i) {
237 std::string extension_id =
238 safe_browsing_util::SBFullHashToString(check.full_hashes[i]);
239 if (check.full_hash_results[i] == SB_THREAT_TYPE_EXTENSION)
240 unsafe_extension_ids.insert(extension_id);
242 OnCheckExtensionsResult(unsafe_extension_ids);
243 break;
245 default:
246 NOTREACHED();
248 } else {
249 NOTREACHED();
253 SafeBrowsingDatabaseManager::SafeBrowsingDatabaseManager(
254 const scoped_refptr<SafeBrowsingService>& service)
255 : sb_service_(service),
256 database_(NULL),
257 enabled_(false),
258 enable_download_protection_(false),
259 enable_csd_whitelist_(false),
260 enable_download_whitelist_(false),
261 enable_extension_blacklist_(false),
262 enable_side_effect_free_whitelist_(false),
263 enable_ip_blacklist_(false),
264 enable_unwanted_software_blacklist_(false),
265 update_in_progress_(false),
266 database_update_in_progress_(false),
267 closing_database_(false),
268 check_timeout_(base::TimeDelta::FromMilliseconds(kCheckTimeoutMs)) {
269 DCHECK(sb_service_.get() != NULL);
271 // Android only supports a subset of FULL_SAFE_BROWSING.
272 // TODO(shess): This shouldn't be OS-driven <http://crbug.com/394379>
273 #if !defined(OS_ANDROID)
274 CommandLine* cmdline = CommandLine::ForCurrentProcess();
275 enable_download_protection_ =
276 !cmdline->HasSwitch(switches::kSbDisableDownloadProtection);
278 // We only download the csd-whitelist if client-side phishing detection is
279 // enabled.
280 enable_csd_whitelist_ =
281 !cmdline->HasSwitch(switches::kDisableClientSidePhishingDetection);
283 // TODO(noelutz): remove this boolean variable since it should always be true
284 // if SafeBrowsing is enabled. Unfortunately, we have no test data for this
285 // list right now. This means that we need to be able to disable this list
286 // for the SafeBrowsing test to pass.
287 enable_download_whitelist_ = enable_csd_whitelist_;
289 // TODO(kalman): there really shouldn't be a flag for this.
290 enable_extension_blacklist_ =
291 !cmdline->HasSwitch(switches::kSbDisableExtensionBlacklist);
293 enable_side_effect_free_whitelist_ =
294 prerender::IsSideEffectFreeWhitelistEnabled() &&
295 !cmdline->HasSwitch(switches::kSbDisableSideEffectFreeWhitelist);
297 // The client-side IP blacklist feature is tightly integrated with client-side
298 // phishing protection for now.
299 enable_ip_blacklist_ = enable_csd_whitelist_;
301 // TODO(gab): Gate this on the same experiment that will soon control the UwS
302 // URL UI.
303 enable_unwanted_software_blacklist_ = true;
305 enum SideEffectFreeWhitelistStatus {
306 SIDE_EFFECT_FREE_WHITELIST_ENABLED,
307 SIDE_EFFECT_FREE_WHITELIST_DISABLED,
308 SIDE_EFFECT_FREE_WHITELIST_STATUS_MAX
311 SideEffectFreeWhitelistStatus side_effect_free_whitelist_status =
312 enable_side_effect_free_whitelist_ ? SIDE_EFFECT_FREE_WHITELIST_ENABLED :
313 SIDE_EFFECT_FREE_WHITELIST_DISABLED;
315 UMA_HISTOGRAM_ENUMERATION("SB2.SideEffectFreeWhitelistStatus",
316 side_effect_free_whitelist_status,
317 SIDE_EFFECT_FREE_WHITELIST_STATUS_MAX);
318 #endif
321 SafeBrowsingDatabaseManager::~SafeBrowsingDatabaseManager() {
322 // We should have already been shut down. If we're still enabled, then the
323 // database isn't going to be closed properly, which could lead to corruption.
324 DCHECK(!enabled_);
327 bool SafeBrowsingDatabaseManager::CanCheckUrl(const GURL& url) const {
328 return url.SchemeIs(url::kFtpScheme) ||
329 url.SchemeIs(url::kHttpScheme) ||
330 url.SchemeIs(url::kHttpsScheme);
333 bool SafeBrowsingDatabaseManager::CheckDownloadUrl(
334 const std::vector<GURL>& url_chain,
335 Client* client) {
336 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
337 if (!enabled_ || !enable_download_protection_)
338 return true;
340 // We need to check the database for url prefix, and later may fetch the url
341 // from the safebrowsing backends. These need to be asynchronous.
342 SafeBrowsingCheck* check =
343 new SafeBrowsingCheck(url_chain,
344 std::vector<SBFullHash>(),
345 client,
346 safe_browsing_util::BINURL,
347 std::vector<SBThreatType>(1,
348 SB_THREAT_TYPE_BINARY_MALWARE_URL));
349 StartSafeBrowsingCheck(
350 check,
351 base::Bind(&SafeBrowsingDatabaseManager::CheckDownloadUrlOnSBThread, this,
352 check));
353 return false;
356 bool SafeBrowsingDatabaseManager::CheckExtensionIDs(
357 const std::set<std::string>& extension_ids,
358 Client* client) {
359 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
361 if (!enabled_ || !enable_extension_blacklist_)
362 return true;
364 std::vector<SBFullHash> extension_id_hashes;
365 std::transform(extension_ids.begin(), extension_ids.end(),
366 std::back_inserter(extension_id_hashes),
367 safe_browsing_util::StringToSBFullHash);
369 SafeBrowsingCheck* check = new SafeBrowsingCheck(
370 std::vector<GURL>(),
371 extension_id_hashes,
372 client,
373 safe_browsing_util::EXTENSIONBLACKLIST,
374 std::vector<SBThreatType>(1, SB_THREAT_TYPE_EXTENSION));
376 StartSafeBrowsingCheck(
377 check,
378 base::Bind(&SafeBrowsingDatabaseManager::CheckExtensionIDsOnSBThread,
379 this,
380 check));
381 return false;
384 bool SafeBrowsingDatabaseManager::CheckSideEffectFreeWhitelistUrl(
385 const GURL& url) {
386 if (!enabled_)
387 return false;
389 if (!CanCheckUrl(url))
390 return false;
392 return database_->ContainsSideEffectFreeWhitelistUrl(url);
395 bool SafeBrowsingDatabaseManager::MatchMalwareIP(
396 const std::string& ip_address) {
397 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
398 if (!enabled_ || !enable_ip_blacklist_ || !MakeDatabaseAvailable()) {
399 return false; // Fail open.
401 return database_->ContainsMalwareIP(ip_address);
404 bool SafeBrowsingDatabaseManager::MatchCsdWhitelistUrl(const GURL& url) {
405 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
406 if (!enabled_ || !enable_csd_whitelist_ || !MakeDatabaseAvailable()) {
407 // There is something funky going on here -- for example, perhaps the user
408 // has not restarted since enabling metrics reporting, so we haven't
409 // enabled the csd whitelist yet. Just to be safe we return true in this
410 // case.
411 return true;
413 return database_->ContainsCsdWhitelistedUrl(url);
416 bool SafeBrowsingDatabaseManager::MatchDownloadWhitelistUrl(const GURL& url) {
417 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
418 if (!enabled_ || !enable_download_whitelist_ || !MakeDatabaseAvailable()) {
419 return true;
421 return database_->ContainsDownloadWhitelistedUrl(url);
424 bool SafeBrowsingDatabaseManager::MatchDownloadWhitelistString(
425 const std::string& str) {
426 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
427 if (!enabled_ || !enable_download_whitelist_ || !MakeDatabaseAvailable()) {
428 return true;
430 return database_->ContainsDownloadWhitelistedString(str);
433 bool SafeBrowsingDatabaseManager::IsMalwareKillSwitchOn() {
434 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
435 if (!enabled_ || !MakeDatabaseAvailable()) {
436 return true;
438 return database_->IsMalwareIPMatchKillSwitchOn();
441 bool SafeBrowsingDatabaseManager::IsCsdWhitelistKillSwitchOn() {
442 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
443 if (!enabled_ || !MakeDatabaseAvailable()) {
444 return true;
446 return database_->IsCsdWhitelistKillSwitchOn();
449 bool SafeBrowsingDatabaseManager::CheckBrowseUrl(const GURL& url,
450 Client* client) {
451 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
452 if (!enabled_)
453 return true;
455 if (!CanCheckUrl(url))
456 return true;
458 std::vector<SBThreatType> expected_threats;
459 expected_threats.push_back(SB_THREAT_TYPE_URL_MALWARE);
460 expected_threats.push_back(SB_THREAT_TYPE_URL_PHISHING);
461 expected_threats.push_back(SB_THREAT_TYPE_URL_UNWANTED);
463 const base::TimeTicks start = base::TimeTicks::Now();
464 if (!MakeDatabaseAvailable()) {
465 QueuedCheck queued_check(safe_browsing_util::MALWARE, // or PHISH
466 client,
467 url,
468 expected_threats,
469 start);
470 queued_checks_.push_back(queued_check);
471 return false;
474 // Cache hits should, in general, be the same for both (ignoring potential
475 // cache evictions in the second call for entries that were just about to be
476 // evicted in the first call).
477 // TODO(gab): Refactor SafeBrowsingDatabase to avoid depending on this here.
478 std::vector<SBFullHashResult> cache_hits;
480 std::vector<SBPrefix> browse_prefix_hits;
481 bool browse_prefix_match = database_->ContainsBrowseUrl(
482 url, &browse_prefix_hits, &cache_hits);
484 std::vector<SBPrefix> unwanted_prefix_hits;
485 std::vector<SBFullHashResult> unused_cache_hits;
486 bool unwanted_prefix_match = database_->ContainsUnwantedSoftwareUrl(
487 url, &unwanted_prefix_hits, &unused_cache_hits);
489 // Merge the two pre-sorted prefix hits lists.
490 // TODO(gab): Refactor SafeBrowsingDatabase for it to return this merged list
491 // by default rather than building it here.
492 std::vector<SBPrefix> prefix_hits(browse_prefix_hits.size() +
493 unwanted_prefix_hits.size());
494 std::merge(browse_prefix_hits.begin(),
495 browse_prefix_hits.end(),
496 unwanted_prefix_hits.begin(),
497 unwanted_prefix_hits.end(),
498 prefix_hits.begin());
499 prefix_hits.erase(std::unique(prefix_hits.begin(), prefix_hits.end()),
500 prefix_hits.end());
502 UMA_HISTOGRAM_TIMES("SB2.FilterCheck", base::TimeTicks::Now() - start);
504 if (!browse_prefix_match && !unwanted_prefix_match)
505 return true; // URL is okay.
507 // Needs to be asynchronous, since we could be in the constructor of a
508 // ResourceDispatcherHost event handler which can't pause there.
509 // This check will ping the Safe Browsing servers and get all lists which it
510 // matches. These lists will then be filtered against the |expected_threats|
511 // and the result callback for MALWARE (which is the same as for PHISH and
512 // UNWANTEDURL) will eventually be invoked with the final decision.
513 SafeBrowsingCheck* check = new SafeBrowsingCheck(std::vector<GURL>(1, url),
514 std::vector<SBFullHash>(),
515 client,
516 safe_browsing_util::MALWARE,
517 expected_threats);
518 check->need_get_hash = cache_hits.empty();
519 check->prefix_hits.swap(prefix_hits);
520 check->cache_hits.swap(cache_hits);
521 checks_.insert(check);
523 BrowserThread::PostTask(
524 BrowserThread::IO, FROM_HERE,
525 base::Bind(&SafeBrowsingDatabaseManager::OnCheckDone, this, check));
527 return false;
530 void SafeBrowsingDatabaseManager::CancelCheck(Client* client) {
531 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
532 for (CurrentChecks::iterator i = checks_.begin(); i != checks_.end(); ++i) {
533 // We can't delete matching checks here because the db thread has a copy of
534 // the pointer. Instead, we simply NULL out the client, and when the db
535 // thread calls us back, we'll clean up the check.
536 if ((*i)->client == client)
537 (*i)->client = NULL;
540 // Scan the queued clients store. Clients may be here if they requested a URL
541 // check before the database has finished loading.
542 for (std::deque<QueuedCheck>::iterator it(queued_checks_.begin());
543 it != queued_checks_.end(); ) {
544 // In this case it's safe to delete matches entirely since nothing has a
545 // pointer to them.
546 if (it->client == client)
547 it = queued_checks_.erase(it);
548 else
549 ++it;
553 void SafeBrowsingDatabaseManager::HandleGetHashResults(
554 SafeBrowsingCheck* check,
555 const std::vector<SBFullHashResult>& full_hashes,
556 const base::TimeDelta& cache_lifetime) {
557 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
559 if (!enabled_)
560 return;
562 // If the service has been shut down, |check| should have been deleted.
563 DCHECK(checks_.find(check) != checks_.end());
565 // |start| is set before calling |GetFullHash()|, which should be
566 // the only path which gets to here.
567 DCHECK(!check->start.is_null());
568 UMA_HISTOGRAM_LONG_TIMES("SB2.Network",
569 base::TimeTicks::Now() - check->start);
571 std::vector<SBPrefix> prefixes = check->prefix_hits;
572 OnHandleGetHashResults(check, full_hashes); // 'check' is deleted here.
574 // Cache the GetHash results.
575 if (cache_lifetime != base::TimeDelta() && MakeDatabaseAvailable())
576 database_->CacheHashResults(prefixes, full_hashes, cache_lifetime);
579 void SafeBrowsingDatabaseManager::GetChunks(GetChunksCallback callback) {
580 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
581 DCHECK(enabled_);
582 DCHECK(!callback.is_null());
583 safe_browsing_thread_->message_loop()->PostTask(FROM_HERE, base::Bind(
584 &SafeBrowsingDatabaseManager::GetAllChunksFromDatabase, this, callback));
587 void SafeBrowsingDatabaseManager::AddChunks(
588 const std::string& list,
589 scoped_ptr<ScopedVector<SBChunkData> > chunks,
590 AddChunksCallback callback) {
591 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
592 DCHECK(enabled_);
593 DCHECK(!callback.is_null());
594 safe_browsing_thread_->message_loop()->PostTask(FROM_HERE, base::Bind(
595 &SafeBrowsingDatabaseManager::AddDatabaseChunks, this, list,
596 base::Passed(&chunks), callback));
599 void SafeBrowsingDatabaseManager::DeleteChunks(
600 scoped_ptr<std::vector<SBChunkDelete> > chunk_deletes) {
601 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
602 DCHECK(enabled_);
603 safe_browsing_thread_->message_loop()->PostTask(FROM_HERE, base::Bind(
604 &SafeBrowsingDatabaseManager::DeleteDatabaseChunks, this,
605 base::Passed(&chunk_deletes)));
608 void SafeBrowsingDatabaseManager::UpdateStarted() {
609 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
610 DCHECK(enabled_);
611 DCHECK(!update_in_progress_);
612 update_in_progress_ = true;
615 void SafeBrowsingDatabaseManager::UpdateFinished(bool update_succeeded) {
616 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
617 DCHECK(enabled_);
618 if (update_in_progress_) {
619 update_in_progress_ = false;
620 safe_browsing_thread_->message_loop()->PostTask(FROM_HERE,
621 base::Bind(&SafeBrowsingDatabaseManager::DatabaseUpdateFinished,
622 this, update_succeeded));
626 void SafeBrowsingDatabaseManager::ResetDatabase() {
627 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
628 DCHECK(enabled_);
629 safe_browsing_thread_->message_loop()->PostTask(FROM_HERE, base::Bind(
630 &SafeBrowsingDatabaseManager::OnResetDatabase, this));
633 void SafeBrowsingDatabaseManager::StartOnIOThread() {
634 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
635 if (enabled_)
636 return;
638 DCHECK(!safe_browsing_thread_.get());
639 safe_browsing_thread_.reset(new base::Thread("Chrome_SafeBrowsingThread"));
640 if (!safe_browsing_thread_->Start())
641 return;
642 enabled_ = true;
644 MakeDatabaseAvailable();
647 void SafeBrowsingDatabaseManager::StopOnIOThread(bool shutdown) {
648 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
650 DoStopOnIOThread();
651 if (shutdown) {
652 sb_service_ = NULL;
656 void SafeBrowsingDatabaseManager::NotifyDatabaseUpdateFinished(
657 bool update_succeeded) {
658 content::NotificationService::current()->Notify(
659 chrome::NOTIFICATION_SAFE_BROWSING_UPDATE_COMPLETE,
660 content::Source<SafeBrowsingDatabaseManager>(this),
661 content::Details<bool>(&update_succeeded));
664 SafeBrowsingDatabaseManager::QueuedCheck::QueuedCheck(
665 const safe_browsing_util::ListType check_type,
666 Client* client,
667 const GURL& url,
668 const std::vector<SBThreatType>& expected_threats,
669 const base::TimeTicks& start)
670 : check_type(check_type),
671 client(client),
672 url(url),
673 expected_threats(expected_threats),
674 start(start) {
677 SafeBrowsingDatabaseManager::QueuedCheck::~QueuedCheck() {
680 void SafeBrowsingDatabaseManager::DoStopOnIOThread() {
681 if (!enabled_)
682 return;
684 enabled_ = false;
686 // Delete queued checks, calling back any clients with 'SB_THREAT_TYPE_SAFE'.
687 while (!queued_checks_.empty()) {
688 QueuedCheck queued = queued_checks_.front();
689 if (queued.client) {
690 SafeBrowsingCheck sb_check(std::vector<GURL>(1, queued.url),
691 std::vector<SBFullHash>(),
692 queued.client,
693 queued.check_type,
694 queued.expected_threats);
695 queued.client->OnSafeBrowsingResult(sb_check);
697 queued_checks_.pop_front();
700 // Close the database. Cases to avoid:
701 // * If |closing_database_| is true, continuing will queue up a second
702 // request, |closing_database_| will be reset after handling the first
703 // request, and if any functions on the db thread recreate the database, we
704 // could start using it on the IO thread and then have the second request
705 // handler delete it out from under us.
706 // * If |database_| is NULL, then either no creation request is in flight, in
707 // which case we don't need to do anything, or one is in flight, in which
708 // case the database will be recreated before our deletion request is
709 // handled, and could be used on the IO thread in that time period, leading
710 // to the same problem as above.
711 // Checking DatabaseAvailable() avoids both of these.
712 if (DatabaseAvailable()) {
713 closing_database_ = true;
714 safe_browsing_thread_->message_loop()->PostTask(FROM_HERE,
715 base::Bind(&SafeBrowsingDatabaseManager::OnCloseDatabase, this));
718 // Flush the database thread. Any in-progress database check results will be
719 // ignored and cleaned up below.
721 // Note that to avoid leaking the database, we rely on the fact that no new
722 // tasks will be added to the db thread between the call above and this one.
723 // See comments on the declaration of |safe_browsing_thread_|.
725 // A ScopedAllowIO object is required to join the thread when calling Stop.
726 // See http://crbug.com/72696. Note that we call Stop() first to clear out
727 // any remaining tasks before clearing safe_browsing_thread_.
728 base::ThreadRestrictions::ScopedAllowIO allow_io_for_thread_join;
729 safe_browsing_thread_->Stop();
730 safe_browsing_thread_.reset();
733 // Delete pending checks, calling back any clients with 'SB_THREAT_TYPE_SAFE'.
734 // We have to do this after the db thread returns because methods on it can
735 // have copies of these pointers, so deleting them might lead to accessing
736 // garbage.
737 for (CurrentChecks::iterator it = checks_.begin();
738 it != checks_.end(); ++it) {
739 SafeBrowsingCheck* check = *it;
740 if (check->client)
741 check->client->OnSafeBrowsingResult(*check);
743 STLDeleteElements(&checks_);
745 gethash_requests_.clear();
748 bool SafeBrowsingDatabaseManager::DatabaseAvailable() const {
749 base::AutoLock lock(database_lock_);
750 return !closing_database_ && (database_ != NULL);
753 bool SafeBrowsingDatabaseManager::MakeDatabaseAvailable() {
754 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
755 DCHECK(enabled_);
756 if (DatabaseAvailable())
757 return true;
758 safe_browsing_thread_->message_loop()->PostTask(
759 FROM_HERE,
760 base::Bind(base::IgnoreResult(&SafeBrowsingDatabaseManager::GetDatabase),
761 this));
762 return false;
765 SafeBrowsingDatabase* SafeBrowsingDatabaseManager::GetDatabase() {
766 DCHECK_EQ(base::MessageLoop::current(),
767 safe_browsing_thread_->message_loop());
768 if (database_)
769 return database_;
770 startup_metric_utils::ScopedSlowStartupUMA
771 scoped_timer("Startup.SlowStartupSafeBrowsingGetDatabase");
772 const base::TimeTicks before = base::TimeTicks::Now();
774 SafeBrowsingDatabase* database =
775 SafeBrowsingDatabase::Create(enable_download_protection_,
776 enable_csd_whitelist_,
777 enable_download_whitelist_,
778 enable_extension_blacklist_,
779 enable_side_effect_free_whitelist_,
780 enable_ip_blacklist_,
781 enable_unwanted_software_blacklist_);
783 database->Init(SafeBrowsingService::GetBaseFilename());
785 // Acquiring the lock here guarantees correct ordering between the writes to
786 // the new database object above, and the setting of |databse_| below.
787 base::AutoLock lock(database_lock_);
788 database_ = database;
791 BrowserThread::PostTask(
792 BrowserThread::IO, FROM_HERE,
793 base::Bind(&SafeBrowsingDatabaseManager::DatabaseLoadComplete, this));
795 UMA_HISTOGRAM_TIMES("SB2.DatabaseOpen", base::TimeTicks::Now() - before);
796 return database_;
799 void SafeBrowsingDatabaseManager::OnCheckDone(SafeBrowsingCheck* check) {
800 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
802 if (!enabled_)
803 return;
805 // If the service has been shut down, |check| should have been deleted.
806 DCHECK(checks_.find(check) != checks_.end());
808 if (check->client && check->need_get_hash) {
809 // We have a partial match so we need to query Google for the full hash.
810 // Clean up will happen in HandleGetHashResults.
812 // See if we have a GetHash request already in progress for this particular
813 // prefix. If so, we just append ourselves to the list of interested parties
814 // when the results arrive. We only do this for checks involving one prefix,
815 // since that is the common case (multiple prefixes will issue the request
816 // as normal).
817 if (check->prefix_hits.size() == 1) {
818 SBPrefix prefix = check->prefix_hits[0];
819 GetHashRequests::iterator it = gethash_requests_.find(prefix);
820 if (it != gethash_requests_.end()) {
821 // There's already a request in progress.
822 it->second.push_back(check);
823 return;
826 // No request in progress, so we're the first for this prefix.
827 GetHashRequestors requestors;
828 requestors.push_back(check);
829 gethash_requests_[prefix] = requestors;
832 // Reset the start time so that we can measure the network time without the
833 // database time.
834 check->start = base::TimeTicks::Now();
835 // Note: If |this| is deleted or stopped, the protocol_manager will
836 // be destroyed as well - hence it's OK to do unretained in this case.
837 bool is_download = check->check_type == safe_browsing_util::BINURL;
838 sb_service_->protocol_manager()->GetFullHash(
839 check->prefix_hits,
840 base::Bind(&SafeBrowsingDatabaseManager::HandleGetHashResults,
841 base::Unretained(this),
842 check),
843 is_download);
844 } else {
845 // We may have cached results for previous GetHash queries. Since
846 // this data comes from cache, don't histogram hits.
847 HandleOneCheck(check, check->cache_hits);
851 void SafeBrowsingDatabaseManager::GetAllChunksFromDatabase(
852 GetChunksCallback callback) {
853 DCHECK_EQ(base::MessageLoop::current(),
854 safe_browsing_thread_->message_loop());
856 bool database_error = true;
857 std::vector<SBListChunkRanges> lists;
858 DCHECK(!database_update_in_progress_);
859 database_update_in_progress_ = true;
860 GetDatabase(); // This guarantees that |database_| is non-NULL.
861 if (database_->UpdateStarted(&lists)) {
862 database_error = false;
863 } else {
864 database_->UpdateFinished(false);
867 BrowserThread::PostTask(
868 BrowserThread::IO, FROM_HERE,
869 base::Bind(&SafeBrowsingDatabaseManager::OnGetAllChunksFromDatabase,
870 this, lists, database_error, callback));
873 void SafeBrowsingDatabaseManager::OnGetAllChunksFromDatabase(
874 const std::vector<SBListChunkRanges>& lists, bool database_error,
875 GetChunksCallback callback) {
876 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
877 if (enabled_)
878 callback.Run(lists, database_error);
881 void SafeBrowsingDatabaseManager::OnAddChunksComplete(
882 AddChunksCallback callback) {
883 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
884 if (enabled_)
885 callback.Run();
888 void SafeBrowsingDatabaseManager::DatabaseLoadComplete() {
889 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
890 if (!enabled_)
891 return;
893 LOCAL_HISTOGRAM_COUNTS("SB.QueueDepth", queued_checks_.size());
894 if (queued_checks_.empty())
895 return;
897 // If the database isn't already available, calling CheckUrl() in the loop
898 // below will add the check back to the queue, and we'll infinite-loop.
899 DCHECK(DatabaseAvailable());
900 while (!queued_checks_.empty()) {
901 QueuedCheck check = queued_checks_.front();
902 DCHECK(!check.start.is_null());
903 LOCAL_HISTOGRAM_TIMES("SB.QueueDelay",
904 base::TimeTicks::Now() - check.start);
905 // If CheckUrl() determines the URL is safe immediately, it doesn't call the
906 // client's handler function (because normally it's being directly called by
907 // the client). Since we're not the client, we have to convey this result.
908 if (check.client && CheckBrowseUrl(check.url, check.client)) {
909 SafeBrowsingCheck sb_check(std::vector<GURL>(1, check.url),
910 std::vector<SBFullHash>(),
911 check.client,
912 check.check_type,
913 check.expected_threats);
914 check.client->OnSafeBrowsingResult(sb_check);
916 queued_checks_.pop_front();
920 void SafeBrowsingDatabaseManager::AddDatabaseChunks(
921 const std::string& list_name,
922 scoped_ptr<ScopedVector<SBChunkData> > chunks,
923 AddChunksCallback callback) {
924 DCHECK_EQ(base::MessageLoop::current(),
925 safe_browsing_thread_->message_loop());
926 if (chunks)
927 GetDatabase()->InsertChunks(list_name, chunks->get());
928 BrowserThread::PostTask(
929 BrowserThread::IO, FROM_HERE,
930 base::Bind(&SafeBrowsingDatabaseManager::OnAddChunksComplete, this,
931 callback));
934 void SafeBrowsingDatabaseManager::DeleteDatabaseChunks(
935 scoped_ptr<std::vector<SBChunkDelete> > chunk_deletes) {
936 DCHECK_EQ(base::MessageLoop::current(),
937 safe_browsing_thread_->message_loop());
938 if (chunk_deletes)
939 GetDatabase()->DeleteChunks(*chunk_deletes);
942 void SafeBrowsingDatabaseManager::DatabaseUpdateFinished(
943 bool update_succeeded) {
944 DCHECK_EQ(base::MessageLoop::current(),
945 safe_browsing_thread_->message_loop());
946 GetDatabase()->UpdateFinished(update_succeeded);
947 DCHECK(database_update_in_progress_);
948 database_update_in_progress_ = false;
949 BrowserThread::PostTask(
950 BrowserThread::UI, FROM_HERE,
951 base::Bind(&SafeBrowsingDatabaseManager::NotifyDatabaseUpdateFinished,
952 this, update_succeeded));
955 void SafeBrowsingDatabaseManager::OnCloseDatabase() {
956 DCHECK_EQ(base::MessageLoop::current(),
957 safe_browsing_thread_->message_loop());
958 DCHECK(closing_database_);
960 // Because |closing_database_| is true, nothing on the IO thread will be
961 // accessing the database, so it's safe to delete and then NULL the pointer.
962 delete database_;
963 database_ = NULL;
965 // Acquiring the lock here guarantees correct ordering between the resetting
966 // of |database_| above and of |closing_database_| below, which ensures there
967 // won't be a window during which the IO thread falsely believes the database
968 // is available.
969 base::AutoLock lock(database_lock_);
970 closing_database_ = false;
973 void SafeBrowsingDatabaseManager::OnResetDatabase() {
974 DCHECK_EQ(base::MessageLoop::current(),
975 safe_browsing_thread_->message_loop());
976 GetDatabase()->ResetDatabase();
979 void SafeBrowsingDatabaseManager::OnHandleGetHashResults(
980 SafeBrowsingCheck* check,
981 const std::vector<SBFullHashResult>& full_hashes) {
982 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
983 safe_browsing_util::ListType check_type = check->check_type;
984 SBPrefix prefix = check->prefix_hits[0];
985 GetHashRequests::iterator it = gethash_requests_.find(prefix);
986 if (check->prefix_hits.size() > 1 || it == gethash_requests_.end()) {
987 const bool hit = HandleOneCheck(check, full_hashes);
988 RecordGetHashCheckStatus(hit, check_type, full_hashes);
989 return;
992 // Call back all interested parties, noting if any has a hit.
993 GetHashRequestors& requestors = it->second;
994 bool hit = false;
995 for (GetHashRequestors::iterator r = requestors.begin();
996 r != requestors.end(); ++r) {
997 if (HandleOneCheck(*r, full_hashes))
998 hit = true;
1000 RecordGetHashCheckStatus(hit, check_type, full_hashes);
1002 gethash_requests_.erase(it);
1005 bool SafeBrowsingDatabaseManager::HandleOneCheck(
1006 SafeBrowsingCheck* check,
1007 const std::vector<SBFullHashResult>& full_hashes) {
1008 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
1009 DCHECK(check);
1011 bool is_threat = false;
1013 // TODO(shess): GetHashSeverestThreadListType() contains a loop,
1014 // GetUrlSeverestThreatListType() a loop around that loop. Having another
1015 // loop out here concerns me. It is likely that SAFE is an expected outcome,
1016 // which means all of those loops run to completion. Refactoring this to
1017 // generate a set of sorted items to compare in sequence would probably
1018 // improve things.
1020 // Additionally, the set of patterns generated from the urls is very similar
1021 // to the patterns generated in ContainsBrowseUrl() and other database checks,
1022 // which are called from this code. Refactoring that across the checks could
1023 // interact well with batching the checks here.
1025 // TODO(gab): Fix the fact that Get(Url|Hash)SeverestThreatType() may return a
1026 // threat for which IsExpectedThreat() returns false even if |full_hashes|
1027 // actually contains an expected threat.
1029 for (size_t i = 0; i < check->urls.size(); ++i) {
1030 size_t threat_index;
1031 SBThreatType threat =
1032 GetUrlSeverestThreatType(check->urls[i], full_hashes, &threat_index);
1033 if (threat != SB_THREAT_TYPE_SAFE &&
1034 IsExpectedThreat(threat, check->expected_threats)) {
1035 check->url_results[i] = threat;
1036 check->url_metadata[i] = full_hashes[threat_index].metadata;
1037 is_threat = true;
1041 for (size_t i = 0; i < check->full_hashes.size(); ++i) {
1042 SBThreatType threat =
1043 GetHashSeverestThreatType(check->full_hashes[i], full_hashes);
1044 if (threat != SB_THREAT_TYPE_SAFE &&
1045 IsExpectedThreat(threat, check->expected_threats)) {
1046 check->full_hash_results[i] = threat;
1047 is_threat = true;
1051 SafeBrowsingCheckDone(check);
1052 return is_threat;
1055 void SafeBrowsingDatabaseManager::CheckDownloadUrlOnSBThread(
1056 SafeBrowsingCheck* check) {
1057 DCHECK_EQ(base::MessageLoop::current(),
1058 safe_browsing_thread_->message_loop());
1059 DCHECK(enable_download_protection_);
1061 std::vector<SBPrefix> prefix_hits;
1063 if (!database_->ContainsDownloadUrl(check->urls, &prefix_hits)) {
1064 // Good, we don't have hash for this url prefix.
1065 BrowserThread::PostTask(
1066 BrowserThread::IO, FROM_HERE,
1067 base::Bind(&SafeBrowsingDatabaseManager::CheckDownloadUrlDone, this,
1068 check));
1069 return;
1072 check->need_get_hash = true;
1073 check->prefix_hits.clear();
1074 check->prefix_hits = prefix_hits;
1075 BrowserThread::PostTask(
1076 BrowserThread::IO, FROM_HERE,
1077 base::Bind(&SafeBrowsingDatabaseManager::OnCheckDone, this, check));
1080 void SafeBrowsingDatabaseManager::CheckExtensionIDsOnSBThread(
1081 SafeBrowsingCheck* check) {
1082 DCHECK_EQ(base::MessageLoop::current(),
1083 safe_browsing_thread_->message_loop());
1085 std::vector<SBPrefix> prefixes;
1086 for (std::vector<SBFullHash>::iterator it = check->full_hashes.begin();
1087 it != check->full_hashes.end(); ++it) {
1088 prefixes.push_back((*it).prefix);
1090 database_->ContainsExtensionPrefixes(prefixes, &check->prefix_hits);
1092 if (check->prefix_hits.empty()) {
1093 // No matches for any extensions.
1094 BrowserThread::PostTask(
1095 BrowserThread::IO,
1096 FROM_HERE,
1097 base::Bind(&SafeBrowsingDatabaseManager::SafeBrowsingCheckDone, this,
1098 check));
1099 } else {
1100 // Some prefixes matched, we need to ask Google whether they're legit.
1101 check->need_get_hash = true;
1102 BrowserThread::PostTask(
1103 BrowserThread::IO,
1104 FROM_HERE,
1105 base::Bind(&SafeBrowsingDatabaseManager::OnCheckDone, this, check));
1109 void SafeBrowsingDatabaseManager::TimeoutCallback(SafeBrowsingCheck* check) {
1110 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
1111 DCHECK(check);
1113 if (!enabled_)
1114 return;
1116 DCHECK(checks_.find(check) != checks_.end());
1117 if (check->client) {
1118 check->client->OnSafeBrowsingResult(*check);
1119 check->client = NULL;
1123 void SafeBrowsingDatabaseManager::CheckDownloadUrlDone(
1124 SafeBrowsingCheck* check) {
1125 DCHECK(enable_download_protection_);
1126 SafeBrowsingCheckDone(check);
1129 void SafeBrowsingDatabaseManager::SafeBrowsingCheckDone(
1130 SafeBrowsingCheck* check) {
1131 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
1132 DCHECK(check);
1134 if (!enabled_)
1135 return;
1137 DVLOG(1) << "SafeBrowsingCheckDone";
1138 DCHECK(checks_.find(check) != checks_.end());
1139 if (check->client)
1140 check->client->OnSafeBrowsingResult(*check);
1141 checks_.erase(check);
1142 delete check;
1145 void SafeBrowsingDatabaseManager::StartSafeBrowsingCheck(
1146 SafeBrowsingCheck* check,
1147 const base::Closure& task) {
1148 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
1149 check->timeout_factory_.reset(
1150 new base::WeakPtrFactory<SafeBrowsingDatabaseManager>(this));
1151 checks_.insert(check);
1153 safe_browsing_thread_->message_loop()->PostTask(FROM_HERE, task);
1155 base::MessageLoop::current()->PostDelayedTask(FROM_HERE,
1156 base::Bind(&SafeBrowsingDatabaseManager::TimeoutCallback,
1157 check->timeout_factory_->GetWeakPtr(), check),
1158 check_timeout_);