Remove 'RemoveTrailingSeparators' function from SimpleMenuModel
[chromium-blink-merge.git] / chrome / browser / safe_browsing / database_manager.cc
blobafcd0f69717fe06751fe7ca99e5bcb10d87cece9
1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "chrome/browser/safe_browsing/database_manager.h"
7 #include <algorithm>
9 #include "base/bind.h"
10 #include "base/bind_helpers.h"
11 #include "base/callback.h"
12 #include "base/command_line.h"
13 #include "base/debug/leak_tracker.h"
14 #include "base/metrics/histogram_macros.h"
15 #include "base/stl_util.h"
16 #include "base/strings/string_util.h"
17 #include "chrome/browser/browser_process.h"
18 #include "chrome/browser/chrome_notification_types.h"
19 #include "chrome/browser/prerender/prerender_field_trial.h"
20 #include "chrome/browser/safe_browsing/client_side_detection_service.h"
21 #include "chrome/browser/safe_browsing/download_protection_service.h"
22 #include "chrome/browser/safe_browsing/malware_details.h"
23 #include "chrome/browser/safe_browsing/protocol_manager.h"
24 #include "chrome/browser/safe_browsing/safe_browsing_database.h"
25 #include "chrome/browser/safe_browsing/safe_browsing_service.h"
26 #include "chrome/browser/safe_browsing/ui_manager.h"
27 #include "chrome/common/chrome_constants.h"
28 #include "chrome/common/chrome_paths.h"
29 #include "chrome/common/chrome_switches.h"
30 #include "components/startup_metric_utils/startup_metric_utils.h"
31 #include "content/public/browser/browser_thread.h"
32 #include "content/public/browser/notification_service.h"
33 #include "url/url_constants.h"
35 using content::BrowserThread;
37 namespace {
39 // Timeout for match checks, e.g. download URLs, hashes.
40 const int kCheckTimeoutMs = 10000;
42 // Records disposition information about the check. |hit| should be
43 // |true| if there were any prefix hits in |full_hashes|.
44 void RecordGetHashCheckStatus(
45 bool hit,
46 safe_browsing_util::ListType check_type,
47 const std::vector<SBFullHashResult>& full_hashes) {
48 SafeBrowsingProtocolManager::ResultType result;
49 if (full_hashes.empty()) {
50 result = SafeBrowsingProtocolManager::GET_HASH_FULL_HASH_EMPTY;
51 } else if (hit) {
52 result = SafeBrowsingProtocolManager::GET_HASH_FULL_HASH_HIT;
53 } else {
54 result = SafeBrowsingProtocolManager::GET_HASH_FULL_HASH_MISS;
56 bool is_download = check_type == safe_browsing_util::BINURL;
57 SafeBrowsingProtocolManager::RecordGetHashResult(is_download, result);
60 bool IsExpectedThreat(
61 const SBThreatType threat_type,
62 const std::vector<SBThreatType>& expected_threats) {
63 return expected_threats.end() != std::find(expected_threats.begin(),
64 expected_threats.end(),
65 threat_type);
68 // Return the severest list id from the results in |full_hashes| which matches
69 // |hash|, or INVALID if none match.
70 safe_browsing_util::ListType GetHashSeverestThreatListType(
71 const SBFullHash& hash,
72 const std::vector<SBFullHashResult>& full_hashes,
73 size_t* index) {
74 safe_browsing_util::ListType pending_threat = safe_browsing_util::INVALID;
75 for (size_t i = 0; i < full_hashes.size(); ++i) {
76 if (SBFullHashEqual(hash, full_hashes[i].hash)) {
77 const safe_browsing_util::ListType threat =
78 static_cast<safe_browsing_util::ListType>(full_hashes[i].list_id);
79 switch (threat) {
80 case safe_browsing_util::INVALID:
81 // |full_hashes| should never contain INVALID as a |list_id|.
82 NOTREACHED();
83 break;
84 case safe_browsing_util::MALWARE: // Falls through.
85 case safe_browsing_util::PHISH: // Falls through.
86 case safe_browsing_util::BINURL: // Falls through.
87 case safe_browsing_util::CSDWHITELIST: // Falls through.
88 case safe_browsing_util::DOWNLOADWHITELIST: // Falls through.
89 case safe_browsing_util::INCLUSIONWHITELIST: // Falls through.
90 case safe_browsing_util::EXTENSIONBLACKLIST: // Falls through.
91 case safe_browsing_util::SIDEEFFECTFREEWHITELIST: // Falls through.
92 case safe_browsing_util::IPBLACKLIST:
93 if (index)
94 *index = i;
95 return threat;
96 case safe_browsing_util::UNWANTEDURL:
97 // UNWANTEDURL is considered less severe than other threats, keep
98 // looking.
99 pending_threat = threat;
100 if (index)
101 *index = i;
102 break;
106 return pending_threat;
109 // Given a URL, compare all the possible host + path full hashes to the set of
110 // provided full hashes. Returns the list id of the severest matching result
111 // from |full_hashes|, or INVALID if none match.
112 safe_browsing_util::ListType GetUrlSeverestThreatListType(
113 const GURL& url,
114 const std::vector<SBFullHashResult>& full_hashes,
115 size_t* index) {
116 if (full_hashes.empty())
117 return safe_browsing_util::INVALID;
119 std::vector<std::string> patterns;
120 safe_browsing_util::GeneratePatternsToCheck(url, &patterns);
122 safe_browsing_util::ListType pending_threat = safe_browsing_util::INVALID;
123 for (size_t i = 0; i < patterns.size(); ++i) {
124 safe_browsing_util::ListType threat = GetHashSeverestThreatListType(
125 SBFullHashForString(patterns[i]), full_hashes, index);
126 switch (threat) {
127 case safe_browsing_util::INVALID:
128 // Ignore patterns with no matching threat.
129 break;
130 case safe_browsing_util::MALWARE: // Falls through.
131 case safe_browsing_util::PHISH: // Falls through.
132 case safe_browsing_util::BINURL: // Falls through.
133 case safe_browsing_util::CSDWHITELIST: // Falls through.
134 case safe_browsing_util::DOWNLOADWHITELIST: // Falls through.
135 case safe_browsing_util::INCLUSIONWHITELIST: // Falls through.
136 case safe_browsing_util::EXTENSIONBLACKLIST: // Falls through.
137 case safe_browsing_util::SIDEEFFECTFREEWHITELIST: // Falls through.
138 case safe_browsing_util::IPBLACKLIST:
139 return threat;
140 case safe_browsing_util::UNWANTEDURL:
141 // UNWANTEDURL is considered less severe than other threats, keep
142 // looking.
143 pending_threat = threat;
144 break;
147 return pending_threat;
150 SBThreatType GetThreatTypeFromListType(safe_browsing_util::ListType list_type) {
151 switch (list_type) {
152 case safe_browsing_util::PHISH:
153 return SB_THREAT_TYPE_URL_PHISHING;
154 case safe_browsing_util::MALWARE:
155 return SB_THREAT_TYPE_URL_MALWARE;
156 case safe_browsing_util::UNWANTEDURL:
157 return SB_THREAT_TYPE_URL_UNWANTED;
158 case safe_browsing_util::BINURL:
159 return SB_THREAT_TYPE_BINARY_MALWARE_URL;
160 case safe_browsing_util::EXTENSIONBLACKLIST:
161 return SB_THREAT_TYPE_EXTENSION;
162 default:
163 DVLOG(1) << "Unknown safe browsing list id " << list_type;
164 return SB_THREAT_TYPE_SAFE;
168 } // namespace
170 // static
171 SBThreatType SafeBrowsingDatabaseManager::GetHashSeverestThreatType(
172 const SBFullHash& hash,
173 const std::vector<SBFullHashResult>& full_hashes) {
174 return GetThreatTypeFromListType(
175 GetHashSeverestThreatListType(hash, full_hashes, NULL));
178 // static
179 SBThreatType SafeBrowsingDatabaseManager::GetUrlSeverestThreatType(
180 const GURL& url,
181 const std::vector<SBFullHashResult>& full_hashes,
182 size_t* index) {
183 return GetThreatTypeFromListType(
184 GetUrlSeverestThreatListType(url, full_hashes, index));
187 SafeBrowsingDatabaseManager::SafeBrowsingCheck::SafeBrowsingCheck(
188 const std::vector<GURL>& urls,
189 const std::vector<SBFullHash>& full_hashes,
190 Client* client,
191 safe_browsing_util::ListType check_type,
192 const std::vector<SBThreatType>& expected_threats)
193 : urls(urls),
194 url_results(urls.size(), SB_THREAT_TYPE_SAFE),
195 url_metadata(urls.size()),
196 full_hashes(full_hashes),
197 full_hash_results(full_hashes.size(), SB_THREAT_TYPE_SAFE),
198 client(client),
199 need_get_hash(false),
200 check_type(check_type),
201 expected_threats(expected_threats) {
202 DCHECK_EQ(urls.empty(), !full_hashes.empty())
203 << "Exactly one of urls and full_hashes must be set";
206 SafeBrowsingDatabaseManager::SafeBrowsingCheck::~SafeBrowsingCheck() {}
208 void SafeBrowsingDatabaseManager::Client::OnSafeBrowsingResult(
209 const SafeBrowsingCheck& check) {
210 DCHECK_CURRENTLY_ON(BrowserThread::IO);
212 DCHECK_EQ(check.urls.size(), check.url_results.size());
213 DCHECK_EQ(check.full_hashes.size(), check.full_hash_results.size());
214 if (!check.urls.empty()) {
215 DCHECK(check.full_hashes.empty());
216 switch (check.check_type) {
217 case safe_browsing_util::MALWARE:
218 case safe_browsing_util::PHISH:
219 case safe_browsing_util::UNWANTEDURL:
220 DCHECK_EQ(1u, check.urls.size());
221 OnCheckBrowseUrlResult(
222 check.urls[0], check.url_results[0], check.url_metadata[0]);
223 break;
224 case safe_browsing_util::BINURL:
225 DCHECK_EQ(check.urls.size(), check.url_results.size());
226 OnCheckDownloadUrlResult(
227 check.urls,
228 *std::max_element(check.url_results.begin(),
229 check.url_results.end()));
230 break;
231 default:
232 NOTREACHED();
234 } else if (!check.full_hashes.empty()) {
235 switch (check.check_type) {
236 case safe_browsing_util::EXTENSIONBLACKLIST: {
237 std::set<std::string> unsafe_extension_ids;
238 for (size_t i = 0; i < check.full_hashes.size(); ++i) {
239 std::string extension_id =
240 safe_browsing_util::SBFullHashToString(check.full_hashes[i]);
241 if (check.full_hash_results[i] == SB_THREAT_TYPE_EXTENSION)
242 unsafe_extension_ids.insert(extension_id);
244 OnCheckExtensionsResult(unsafe_extension_ids);
245 break;
247 default:
248 NOTREACHED();
250 } else {
251 NOTREACHED();
255 SafeBrowsingDatabaseManager::SafeBrowsingDatabaseManager(
256 const scoped_refptr<SafeBrowsingService>& service)
257 : sb_service_(service),
258 database_(NULL),
259 enabled_(false),
260 enable_download_protection_(false),
261 enable_csd_whitelist_(false),
262 enable_download_whitelist_(false),
263 enable_extension_blacklist_(false),
264 enable_side_effect_free_whitelist_(false),
265 enable_ip_blacklist_(false),
266 enable_unwanted_software_blacklist_(false),
267 update_in_progress_(false),
268 database_update_in_progress_(false),
269 closing_database_(false),
270 check_timeout_(base::TimeDelta::FromMilliseconds(kCheckTimeoutMs)) {
271 DCHECK_CURRENTLY_ON(BrowserThread::UI);
272 DCHECK(sb_service_.get() != NULL);
274 // Android only supports a subset of FULL_SAFE_BROWSING.
275 // TODO(shess): This shouldn't be OS-driven <http://crbug.com/394379>
276 #if !defined(OS_ANDROID)
277 base::CommandLine* cmdline = base::CommandLine::ForCurrentProcess();
278 enable_download_protection_ =
279 !cmdline->HasSwitch(switches::kSbDisableDownloadProtection);
281 // We only download the csd-whitelist if client-side phishing detection is
282 // enabled.
283 enable_csd_whitelist_ =
284 !cmdline->HasSwitch(switches::kDisableClientSidePhishingDetection);
286 // TODO(noelutz): remove this boolean variable since it should always be true
287 // if SafeBrowsing is enabled. Unfortunately, we have no test data for this
288 // list right now. This means that we need to be able to disable this list
289 // for the SafeBrowsing test to pass.
290 enable_download_whitelist_ = enable_csd_whitelist_;
292 // TODO(kalman): there really shouldn't be a flag for this.
293 enable_extension_blacklist_ =
294 !cmdline->HasSwitch(switches::kSbDisableExtensionBlacklist);
296 enable_side_effect_free_whitelist_ =
297 prerender::IsSideEffectFreeWhitelistEnabled() &&
298 !cmdline->HasSwitch(switches::kSbDisableSideEffectFreeWhitelist);
300 // The client-side IP blacklist feature is tightly integrated with client-side
301 // phishing protection for now.
302 enable_ip_blacklist_ = enable_csd_whitelist_;
304 // The UwS blacklist feature is controlled by a flag for M40.
305 enable_unwanted_software_blacklist_ =
306 safe_browsing_util::GetUnwantedTrialGroup() > safe_browsing_util::UWS_OFF;
308 enum SideEffectFreeWhitelistStatus {
309 SIDE_EFFECT_FREE_WHITELIST_ENABLED,
310 SIDE_EFFECT_FREE_WHITELIST_DISABLED,
311 SIDE_EFFECT_FREE_WHITELIST_STATUS_MAX
314 SideEffectFreeWhitelistStatus side_effect_free_whitelist_status =
315 enable_side_effect_free_whitelist_ ? SIDE_EFFECT_FREE_WHITELIST_ENABLED :
316 SIDE_EFFECT_FREE_WHITELIST_DISABLED;
318 UMA_HISTOGRAM_ENUMERATION("SB2.SideEffectFreeWhitelistStatus",
319 side_effect_free_whitelist_status,
320 SIDE_EFFECT_FREE_WHITELIST_STATUS_MAX);
321 #endif
324 SafeBrowsingDatabaseManager::~SafeBrowsingDatabaseManager() {
325 // The DCHECK is disabled due to crbug.com/438754.
326 // DCHECK_CURRENTLY_ON(BrowserThread::UI);
328 // We should have already been shut down. If we're still enabled, then the
329 // database isn't going to be closed properly, which could lead to corruption.
330 DCHECK(!enabled_);
333 bool SafeBrowsingDatabaseManager::CanCheckUrl(const GURL& url) const {
334 return url.SchemeIs(url::kFtpScheme) ||
335 url.SchemeIs(url::kHttpScheme) ||
336 url.SchemeIs(url::kHttpsScheme);
339 bool SafeBrowsingDatabaseManager::CheckDownloadUrl(
340 const std::vector<GURL>& url_chain,
341 Client* client) {
342 DCHECK_CURRENTLY_ON(BrowserThread::IO);
343 if (!enabled_ || !enable_download_protection_)
344 return true;
346 // We need to check the database for url prefix, and later may fetch the url
347 // from the safebrowsing backends. These need to be asynchronous.
348 SafeBrowsingCheck* check =
349 new SafeBrowsingCheck(url_chain,
350 std::vector<SBFullHash>(),
351 client,
352 safe_browsing_util::BINURL,
353 std::vector<SBThreatType>(1,
354 SB_THREAT_TYPE_BINARY_MALWARE_URL));
355 std::vector<SBPrefix> prefixes;
356 SafeBrowsingDatabase::GetDownloadUrlPrefixes(url_chain, &prefixes);
357 StartSafeBrowsingCheck(
358 check,
359 base::Bind(&SafeBrowsingDatabaseManager::CheckDownloadUrlOnSBThread, this,
360 prefixes));
361 return false;
364 bool SafeBrowsingDatabaseManager::CheckExtensionIDs(
365 const std::set<std::string>& extension_ids,
366 Client* client) {
367 DCHECK_CURRENTLY_ON(BrowserThread::IO);
369 if (!enabled_ || !enable_extension_blacklist_)
370 return true;
372 std::vector<SBFullHash> extension_id_hashes;
373 std::transform(extension_ids.begin(), extension_ids.end(),
374 std::back_inserter(extension_id_hashes),
375 safe_browsing_util::StringToSBFullHash);
376 std::vector<SBPrefix> prefixes;
377 for (const SBFullHash& hash : extension_id_hashes)
378 prefixes.push_back(hash.prefix);
380 SafeBrowsingCheck* check = new SafeBrowsingCheck(
381 std::vector<GURL>(),
382 extension_id_hashes,
383 client,
384 safe_browsing_util::EXTENSIONBLACKLIST,
385 std::vector<SBThreatType>(1, SB_THREAT_TYPE_EXTENSION));
386 StartSafeBrowsingCheck(
387 check,
388 base::Bind(&SafeBrowsingDatabaseManager::CheckExtensionIDsOnSBThread,
389 this, prefixes));
390 return false;
393 bool SafeBrowsingDatabaseManager::CheckSideEffectFreeWhitelistUrl(
394 const GURL& url) {
395 if (!enabled_)
396 return false;
398 if (!CanCheckUrl(url))
399 return false;
401 return database_->ContainsSideEffectFreeWhitelistUrl(url);
404 bool SafeBrowsingDatabaseManager::MatchMalwareIP(
405 const std::string& ip_address) {
406 DCHECK_CURRENTLY_ON(BrowserThread::IO);
407 if (!enabled_ || !enable_ip_blacklist_ || !MakeDatabaseAvailable()) {
408 return false; // Fail open.
410 return database_->ContainsMalwareIP(ip_address);
413 bool SafeBrowsingDatabaseManager::MatchCsdWhitelistUrl(const GURL& url) {
414 DCHECK_CURRENTLY_ON(BrowserThread::IO);
415 if (!enabled_ || !enable_csd_whitelist_ || !MakeDatabaseAvailable()) {
416 // There is something funky going on here -- for example, perhaps the user
417 // has not restarted since enabling metrics reporting, so we haven't
418 // enabled the csd whitelist yet. Just to be safe we return true in this
419 // case.
420 return true;
422 return database_->ContainsCsdWhitelistedUrl(url);
425 bool SafeBrowsingDatabaseManager::MatchDownloadWhitelistUrl(const GURL& url) {
426 DCHECK_CURRENTLY_ON(BrowserThread::IO);
427 if (!enabled_ || !enable_download_whitelist_ || !MakeDatabaseAvailable()) {
428 return true;
430 return database_->ContainsDownloadWhitelistedUrl(url);
433 bool SafeBrowsingDatabaseManager::MatchDownloadWhitelistString(
434 const std::string& str) {
435 DCHECK_CURRENTLY_ON(BrowserThread::IO);
436 if (!enabled_ || !enable_download_whitelist_ || !MakeDatabaseAvailable()) {
437 return true;
439 return database_->ContainsDownloadWhitelistedString(str);
442 bool SafeBrowsingDatabaseManager::MatchInclusionWhitelistUrl(const GURL& url) {
443 DCHECK_CURRENTLY_ON(BrowserThread::IO);
444 if (!enabled_ || !MakeDatabaseAvailable())
445 return true;
446 return database_->ContainsInclusionWhitelistedUrl(url);
449 bool SafeBrowsingDatabaseManager::IsMalwareKillSwitchOn() {
450 DCHECK_CURRENTLY_ON(BrowserThread::IO);
451 if (!enabled_ || !MakeDatabaseAvailable()) {
452 return true;
454 return database_->IsMalwareIPMatchKillSwitchOn();
457 bool SafeBrowsingDatabaseManager::IsCsdWhitelistKillSwitchOn() {
458 DCHECK_CURRENTLY_ON(BrowserThread::IO);
459 if (!enabled_ || !MakeDatabaseAvailable()) {
460 return true;
462 return database_->IsCsdWhitelistKillSwitchOn();
465 bool SafeBrowsingDatabaseManager::CheckBrowseUrl(const GURL& url,
466 Client* client) {
467 DCHECK_CURRENTLY_ON(BrowserThread::IO);
468 if (!enabled_)
469 return true;
471 if (!CanCheckUrl(url))
472 return true;
474 std::vector<SBThreatType> expected_threats;
475 expected_threats.push_back(SB_THREAT_TYPE_URL_MALWARE);
476 expected_threats.push_back(SB_THREAT_TYPE_URL_PHISHING);
477 expected_threats.push_back(SB_THREAT_TYPE_URL_UNWANTED);
479 const base::TimeTicks start = base::TimeTicks::Now();
480 if (!MakeDatabaseAvailable()) {
481 QueuedCheck queued_check(safe_browsing_util::MALWARE, // or PHISH
482 client,
483 url,
484 expected_threats,
485 start);
486 queued_checks_.push_back(queued_check);
487 return false;
490 // Cache hits should, in general, be the same for both (ignoring potential
491 // cache evictions in the second call for entries that were just about to be
492 // evicted in the first call).
493 // TODO(gab): Refactor SafeBrowsingDatabase to avoid depending on this here.
494 std::vector<SBFullHashResult> cache_hits;
496 std::vector<SBPrefix> browse_prefix_hits;
497 bool browse_prefix_match = database_->ContainsBrowseUrl(
498 url, &browse_prefix_hits, &cache_hits);
500 std::vector<SBPrefix> unwanted_prefix_hits;
501 std::vector<SBFullHashResult> unused_cache_hits;
502 bool unwanted_prefix_match = database_->ContainsUnwantedSoftwareUrl(
503 url, &unwanted_prefix_hits, &unused_cache_hits);
505 // Merge the two pre-sorted prefix hits lists.
506 // TODO(gab): Refactor SafeBrowsingDatabase for it to return this merged list
507 // by default rather than building it here.
508 std::vector<SBPrefix> prefix_hits(browse_prefix_hits.size() +
509 unwanted_prefix_hits.size());
510 std::merge(browse_prefix_hits.begin(),
511 browse_prefix_hits.end(),
512 unwanted_prefix_hits.begin(),
513 unwanted_prefix_hits.end(),
514 prefix_hits.begin());
515 prefix_hits.erase(std::unique(prefix_hits.begin(), prefix_hits.end()),
516 prefix_hits.end());
518 UMA_HISTOGRAM_TIMES("SB2.FilterCheck", base::TimeTicks::Now() - start);
520 if (!browse_prefix_match && !unwanted_prefix_match)
521 return true; // URL is okay.
523 // Needs to be asynchronous, since we could be in the constructor of a
524 // ResourceDispatcherHost event handler which can't pause there.
525 // This check will ping the Safe Browsing servers and get all lists which it
526 // matches. These lists will then be filtered against the |expected_threats|
527 // and the result callback for MALWARE (which is the same as for PHISH and
528 // UNWANTEDURL) will eventually be invoked with the final decision.
529 SafeBrowsingCheck* check = new SafeBrowsingCheck(std::vector<GURL>(1, url),
530 std::vector<SBFullHash>(),
531 client,
532 safe_browsing_util::MALWARE,
533 expected_threats);
534 check->need_get_hash = cache_hits.empty();
535 check->prefix_hits.swap(prefix_hits);
536 check->cache_hits.swap(cache_hits);
537 checks_.insert(check);
539 BrowserThread::PostTask(
540 BrowserThread::IO, FROM_HERE,
541 base::Bind(&SafeBrowsingDatabaseManager::OnCheckDone, this, check));
543 return false;
546 void SafeBrowsingDatabaseManager::CancelCheck(Client* client) {
547 DCHECK_CURRENTLY_ON(BrowserThread::IO);
548 for (CurrentChecks::iterator i = checks_.begin(); i != checks_.end(); ++i) {
549 // We can't delete matching checks here because the db thread has a copy of
550 // the pointer. Instead, we simply NULL out the client, and when the db
551 // thread calls us back, we'll clean up the check.
552 if ((*i)->client == client)
553 (*i)->client = NULL;
556 // Scan the queued clients store. Clients may be here if they requested a URL
557 // check before the database has finished loading.
558 for (std::deque<QueuedCheck>::iterator it(queued_checks_.begin());
559 it != queued_checks_.end(); ) {
560 // In this case it's safe to delete matches entirely since nothing has a
561 // pointer to them.
562 if (it->client == client)
563 it = queued_checks_.erase(it);
564 else
565 ++it;
569 void SafeBrowsingDatabaseManager::HandleGetHashResults(
570 SafeBrowsingCheck* check,
571 const std::vector<SBFullHashResult>& full_hashes,
572 const base::TimeDelta& cache_lifetime) {
573 DCHECK_CURRENTLY_ON(BrowserThread::IO);
575 if (!enabled_)
576 return;
578 // If the service has been shut down, |check| should have been deleted.
579 DCHECK(checks_.find(check) != checks_.end());
581 // |start| is set before calling |GetFullHash()|, which should be
582 // the only path which gets to here.
583 DCHECK(!check->start.is_null());
584 UMA_HISTOGRAM_LONG_TIMES("SB2.Network",
585 base::TimeTicks::Now() - check->start);
587 std::vector<SBPrefix> prefixes = check->prefix_hits;
588 OnHandleGetHashResults(check, full_hashes); // 'check' is deleted here.
590 // Cache the GetHash results.
591 if (cache_lifetime != base::TimeDelta() && MakeDatabaseAvailable())
592 database_->CacheHashResults(prefixes, full_hashes, cache_lifetime);
595 void SafeBrowsingDatabaseManager::GetChunks(GetChunksCallback callback) {
596 DCHECK_CURRENTLY_ON(BrowserThread::IO);
597 DCHECK(enabled_);
598 DCHECK(!callback.is_null());
599 safe_browsing_task_runner_->PostTask(
600 FROM_HERE,
601 base::Bind(&SafeBrowsingDatabaseManager::GetAllChunksFromDatabase, this,
602 callback));
605 void SafeBrowsingDatabaseManager::AddChunks(
606 const std::string& list,
607 scoped_ptr<ScopedVector<SBChunkData> > chunks,
608 AddChunksCallback callback) {
609 DCHECK_CURRENTLY_ON(BrowserThread::IO);
610 DCHECK(enabled_);
611 DCHECK(!callback.is_null());
612 safe_browsing_task_runner_->PostTask(
613 FROM_HERE, base::Bind(&SafeBrowsingDatabaseManager::AddDatabaseChunks,
614 this, list, base::Passed(&chunks), callback));
617 void SafeBrowsingDatabaseManager::DeleteChunks(
618 scoped_ptr<std::vector<SBChunkDelete> > chunk_deletes) {
619 DCHECK_CURRENTLY_ON(BrowserThread::IO);
620 DCHECK(enabled_);
621 safe_browsing_task_runner_->PostTask(
622 FROM_HERE, base::Bind(&SafeBrowsingDatabaseManager::DeleteDatabaseChunks,
623 this, base::Passed(&chunk_deletes)));
626 void SafeBrowsingDatabaseManager::UpdateStarted() {
627 DCHECK_CURRENTLY_ON(BrowserThread::IO);
628 DCHECK(enabled_);
629 DCHECK(!update_in_progress_);
630 update_in_progress_ = true;
633 void SafeBrowsingDatabaseManager::UpdateFinished(bool update_succeeded) {
634 DCHECK_CURRENTLY_ON(BrowserThread::IO);
635 DCHECK(enabled_);
636 if (update_in_progress_) {
637 update_in_progress_ = false;
638 safe_browsing_task_runner_->PostTask(
639 FROM_HERE,
640 base::Bind(&SafeBrowsingDatabaseManager::DatabaseUpdateFinished, this,
641 update_succeeded));
645 void SafeBrowsingDatabaseManager::ResetDatabase() {
646 DCHECK_CURRENTLY_ON(BrowserThread::IO);
647 DCHECK(enabled_);
648 safe_browsing_task_runner_->PostTask(
649 FROM_HERE,
650 base::Bind(&SafeBrowsingDatabaseManager::OnResetDatabase, this));
653 void SafeBrowsingDatabaseManager::StartOnIOThread() {
654 DCHECK_CURRENTLY_ON(BrowserThread::IO);
655 if (enabled_)
656 return;
658 // Only get a new task runner if there isn't one already. If the service has
659 // previously been started and stopped, a task runner could already exist.
660 if (!safe_browsing_task_runner_) {
661 base::SequencedWorkerPool* pool = BrowserThread::GetBlockingPool();
662 safe_browsing_task_runner_ =
663 pool->GetSequencedTaskRunnerWithShutdownBehavior(
664 pool->GetSequenceToken(),
665 base::SequencedWorkerPool::SKIP_ON_SHUTDOWN);
668 enabled_ = true;
670 MakeDatabaseAvailable();
673 void SafeBrowsingDatabaseManager::StopOnIOThread(bool shutdown) {
674 DCHECK_CURRENTLY_ON(BrowserThread::IO);
676 DoStopOnIOThread();
677 if (shutdown) {
678 sb_service_ = NULL;
682 void SafeBrowsingDatabaseManager::NotifyDatabaseUpdateFinished(
683 bool update_succeeded) {
684 DCHECK_CURRENTLY_ON(BrowserThread::UI);
685 content::NotificationService::current()->Notify(
686 chrome::NOTIFICATION_SAFE_BROWSING_UPDATE_COMPLETE,
687 content::Source<SafeBrowsingDatabaseManager>(this),
688 content::Details<bool>(&update_succeeded));
691 SafeBrowsingDatabaseManager::QueuedCheck::QueuedCheck(
692 const safe_browsing_util::ListType check_type,
693 Client* client,
694 const GURL& url,
695 const std::vector<SBThreatType>& expected_threats,
696 const base::TimeTicks& start)
697 : check_type(check_type),
698 client(client),
699 url(url),
700 expected_threats(expected_threats),
701 start(start) {
704 SafeBrowsingDatabaseManager::QueuedCheck::~QueuedCheck() {
707 void SafeBrowsingDatabaseManager::DoStopOnIOThread() {
708 DCHECK_CURRENTLY_ON(BrowserThread::IO);
710 if (!enabled_)
711 return;
713 enabled_ = false;
715 // Delete queued checks, calling back any clients with 'SB_THREAT_TYPE_SAFE'.
716 while (!queued_checks_.empty()) {
717 QueuedCheck queued = queued_checks_.front();
718 if (queued.client) {
719 SafeBrowsingCheck sb_check(std::vector<GURL>(1, queued.url),
720 std::vector<SBFullHash>(),
721 queued.client,
722 queued.check_type,
723 queued.expected_threats);
724 queued.client->OnSafeBrowsingResult(sb_check);
726 queued_checks_.pop_front();
729 // Close the database. Cases to avoid:
730 // * If |closing_database_| is true, continuing will queue up a second
731 // request, |closing_database_| will be reset after handling the first
732 // request, and if any functions on the db thread recreate the database, we
733 // could start using it on the IO thread and then have the second request
734 // handler delete it out from under us.
735 // * If |database_| is NULL, then either no creation request is in flight, in
736 // which case we don't need to do anything, or one is in flight, in which
737 // case the database will be recreated before our deletion request is
738 // handled, and could be used on the IO thread in that time period, leading
739 // to the same problem as above.
740 // Checking DatabaseAvailable() avoids both of these.
741 if (DatabaseAvailable()) {
742 closing_database_ = true;
743 safe_browsing_task_runner_->PostTask(
744 FROM_HERE,
745 base::Bind(&SafeBrowsingDatabaseManager::OnCloseDatabase, this));
748 // Delete pending checks, calling back any clients with 'SB_THREAT_TYPE_SAFE'.
749 // We have to do this after the db thread returns because methods on it can
750 // have copies of these pointers, so deleting them might lead to accessing
751 // garbage.
752 for (CurrentChecks::iterator it = checks_.begin();
753 it != checks_.end(); ++it) {
754 SafeBrowsingCheck* check = *it;
755 if (check->client)
756 check->client->OnSafeBrowsingResult(*check);
758 STLDeleteElements(&checks_);
760 gethash_requests_.clear();
763 bool SafeBrowsingDatabaseManager::DatabaseAvailable() const {
764 base::AutoLock lock(database_lock_);
765 return !closing_database_ && (database_ != NULL);
768 bool SafeBrowsingDatabaseManager::MakeDatabaseAvailable() {
769 DCHECK_CURRENTLY_ON(BrowserThread::IO);
770 DCHECK(enabled_);
771 if (DatabaseAvailable())
772 return true;
773 safe_browsing_task_runner_->PostTask(
774 FROM_HERE,
775 base::Bind(base::IgnoreResult(&SafeBrowsingDatabaseManager::GetDatabase),
776 this));
777 return false;
780 SafeBrowsingDatabase* SafeBrowsingDatabaseManager::GetDatabase() {
781 DCHECK(safe_browsing_task_runner_->RunsTasksOnCurrentThread());
783 if (database_)
784 return database_;
785 startup_metric_utils::ScopedSlowStartupUMA
786 scoped_timer("Startup.SlowStartupSafeBrowsingGetDatabase");
787 const base::TimeTicks before = base::TimeTicks::Now();
789 SafeBrowsingDatabase* database = SafeBrowsingDatabase::Create(
790 safe_browsing_task_runner_, enable_download_protection_,
791 enable_csd_whitelist_, enable_download_whitelist_,
792 enable_extension_blacklist_, enable_side_effect_free_whitelist_,
793 enable_ip_blacklist_, enable_unwanted_software_blacklist_);
795 database->Init(SafeBrowsingService::GetBaseFilename());
797 // Acquiring the lock here guarantees correct ordering between the writes to
798 // the new database object above, and the setting of |database_| below.
799 base::AutoLock lock(database_lock_);
800 database_ = database;
803 BrowserThread::PostTask(
804 BrowserThread::IO, FROM_HERE,
805 base::Bind(&SafeBrowsingDatabaseManager::DatabaseLoadComplete, this));
807 UMA_HISTOGRAM_TIMES("SB2.DatabaseOpen", base::TimeTicks::Now() - before);
808 return database_;
811 void SafeBrowsingDatabaseManager::OnCheckDone(SafeBrowsingCheck* check) {
812 DCHECK_CURRENTLY_ON(BrowserThread::IO);
814 if (!enabled_)
815 return;
817 // If the service has been shut down, |check| should have been deleted.
818 DCHECK(checks_.find(check) != checks_.end());
820 if (check->client && check->need_get_hash) {
821 // We have a partial match so we need to query Google for the full hash.
822 // Clean up will happen in HandleGetHashResults.
824 // See if we have a GetHash request already in progress for this particular
825 // prefix. If so, we just append ourselves to the list of interested parties
826 // when the results arrive. We only do this for checks involving one prefix,
827 // since that is the common case (multiple prefixes will issue the request
828 // as normal).
829 if (check->prefix_hits.size() == 1) {
830 SBPrefix prefix = check->prefix_hits[0];
831 GetHashRequests::iterator it = gethash_requests_.find(prefix);
832 if (it != gethash_requests_.end()) {
833 // There's already a request in progress.
834 it->second.push_back(check);
835 return;
838 // No request in progress, so we're the first for this prefix.
839 GetHashRequestors requestors;
840 requestors.push_back(check);
841 gethash_requests_[prefix] = requestors;
844 // Reset the start time so that we can measure the network time without the
845 // database time.
846 check->start = base::TimeTicks::Now();
847 // Note: If |this| is deleted or stopped, the protocol_manager will
848 // be destroyed as well - hence it's OK to do unretained in this case.
849 bool is_download = check->check_type == safe_browsing_util::BINURL;
850 sb_service_->protocol_manager()->GetFullHash(
851 check->prefix_hits,
852 base::Bind(&SafeBrowsingDatabaseManager::HandleGetHashResults,
853 base::Unretained(this),
854 check),
855 is_download);
856 } else {
857 // We may have cached results for previous GetHash queries. Since
858 // this data comes from cache, don't histogram hits.
859 HandleOneCheck(check, check->cache_hits);
863 void SafeBrowsingDatabaseManager::GetAllChunksFromDatabase(
864 GetChunksCallback callback) {
865 DCHECK(safe_browsing_task_runner_->RunsTasksOnCurrentThread());
867 bool database_error = true;
868 std::vector<SBListChunkRanges> lists;
869 DCHECK(!database_update_in_progress_);
870 database_update_in_progress_ = true;
871 GetDatabase(); // This guarantees that |database_| is non-NULL.
872 if (database_->UpdateStarted(&lists)) {
873 database_error = false;
874 } else {
875 database_->UpdateFinished(false);
878 BrowserThread::PostTask(
879 BrowserThread::IO, FROM_HERE,
880 base::Bind(&SafeBrowsingDatabaseManager::OnGetAllChunksFromDatabase,
881 this, lists, database_error, callback));
884 void SafeBrowsingDatabaseManager::OnGetAllChunksFromDatabase(
885 const std::vector<SBListChunkRanges>& lists, bool database_error,
886 GetChunksCallback callback) {
887 DCHECK_CURRENTLY_ON(BrowserThread::IO);
888 if (enabled_)
889 callback.Run(lists, database_error);
892 void SafeBrowsingDatabaseManager::OnAddChunksComplete(
893 AddChunksCallback callback) {
894 DCHECK_CURRENTLY_ON(BrowserThread::IO);
895 if (enabled_)
896 callback.Run();
899 void SafeBrowsingDatabaseManager::DatabaseLoadComplete() {
900 DCHECK_CURRENTLY_ON(BrowserThread::IO);
901 if (!enabled_)
902 return;
904 LOCAL_HISTOGRAM_COUNTS("SB.QueueDepth", queued_checks_.size());
905 if (queued_checks_.empty())
906 return;
908 // If the database isn't already available, calling CheckUrl() in the loop
909 // below will add the check back to the queue, and we'll infinite-loop.
910 DCHECK(DatabaseAvailable());
911 while (!queued_checks_.empty()) {
912 QueuedCheck check = queued_checks_.front();
913 DCHECK(!check.start.is_null());
914 LOCAL_HISTOGRAM_TIMES("SB.QueueDelay",
915 base::TimeTicks::Now() - check.start);
916 // If CheckUrl() determines the URL is safe immediately, it doesn't call the
917 // client's handler function (because normally it's being directly called by
918 // the client). Since we're not the client, we have to convey this result.
919 if (check.client && CheckBrowseUrl(check.url, check.client)) {
920 SafeBrowsingCheck sb_check(std::vector<GURL>(1, check.url),
921 std::vector<SBFullHash>(),
922 check.client,
923 check.check_type,
924 check.expected_threats);
925 check.client->OnSafeBrowsingResult(sb_check);
927 queued_checks_.pop_front();
931 void SafeBrowsingDatabaseManager::AddDatabaseChunks(
932 const std::string& list_name,
933 scoped_ptr<ScopedVector<SBChunkData> > chunks,
934 AddChunksCallback callback) {
935 DCHECK(safe_browsing_task_runner_->RunsTasksOnCurrentThread());
936 if (chunks)
937 GetDatabase()->InsertChunks(list_name, chunks->get());
938 BrowserThread::PostTask(
939 BrowserThread::IO, FROM_HERE,
940 base::Bind(&SafeBrowsingDatabaseManager::OnAddChunksComplete, this,
941 callback));
944 void SafeBrowsingDatabaseManager::DeleteDatabaseChunks(
945 scoped_ptr<std::vector<SBChunkDelete> > chunk_deletes) {
946 DCHECK(safe_browsing_task_runner_->RunsTasksOnCurrentThread());
947 if (chunk_deletes)
948 GetDatabase()->DeleteChunks(*chunk_deletes);
951 void SafeBrowsingDatabaseManager::DatabaseUpdateFinished(
952 bool update_succeeded) {
953 DCHECK(safe_browsing_task_runner_->RunsTasksOnCurrentThread());
954 GetDatabase()->UpdateFinished(update_succeeded);
955 DCHECK(database_update_in_progress_);
956 database_update_in_progress_ = false;
957 BrowserThread::PostTask(
958 BrowserThread::UI, FROM_HERE,
959 base::Bind(&SafeBrowsingDatabaseManager::NotifyDatabaseUpdateFinished,
960 this, update_succeeded));
963 void SafeBrowsingDatabaseManager::OnCloseDatabase() {
964 DCHECK(safe_browsing_task_runner_->RunsTasksOnCurrentThread());
965 DCHECK(closing_database_);
967 // Because |closing_database_| is true, nothing on the IO thread will be
968 // accessing the database, so it's safe to delete and then NULL the pointer.
969 delete database_;
970 database_ = NULL;
972 // Acquiring the lock here guarantees correct ordering between the resetting
973 // of |database_| above and of |closing_database_| below, which ensures there
974 // won't be a window during which the IO thread falsely believes the database
975 // is available.
976 base::AutoLock lock(database_lock_);
977 closing_database_ = false;
980 void SafeBrowsingDatabaseManager::OnResetDatabase() {
981 DCHECK(safe_browsing_task_runner_->RunsTasksOnCurrentThread());
983 GetDatabase()->ResetDatabase();
986 void SafeBrowsingDatabaseManager::OnHandleGetHashResults(
987 SafeBrowsingCheck* check,
988 const std::vector<SBFullHashResult>& full_hashes) {
989 DCHECK_CURRENTLY_ON(BrowserThread::IO);
990 safe_browsing_util::ListType check_type = check->check_type;
991 SBPrefix prefix = check->prefix_hits[0];
992 GetHashRequests::iterator it = gethash_requests_.find(prefix);
993 if (check->prefix_hits.size() > 1 || it == gethash_requests_.end()) {
994 const bool hit = HandleOneCheck(check, full_hashes);
995 RecordGetHashCheckStatus(hit, check_type, full_hashes);
996 return;
999 // Call back all interested parties, noting if any has a hit.
1000 GetHashRequestors& requestors = it->second;
1001 bool hit = false;
1002 for (GetHashRequestors::iterator r = requestors.begin();
1003 r != requestors.end(); ++r) {
1004 if (HandleOneCheck(*r, full_hashes))
1005 hit = true;
1007 RecordGetHashCheckStatus(hit, check_type, full_hashes);
1009 gethash_requests_.erase(it);
1012 bool SafeBrowsingDatabaseManager::HandleOneCheck(
1013 SafeBrowsingCheck* check,
1014 const std::vector<SBFullHashResult>& full_hashes) {
1015 DCHECK_CURRENTLY_ON(BrowserThread::IO);
1016 DCHECK(check);
1018 bool is_threat = false;
1020 // TODO(shess): GetHashSeverestThreadListType() contains a loop,
1021 // GetUrlSeverestThreatListType() a loop around that loop. Having another
1022 // loop out here concerns me. It is likely that SAFE is an expected outcome,
1023 // which means all of those loops run to completion. Refactoring this to
1024 // generate a set of sorted items to compare in sequence would probably
1025 // improve things.
1027 // Additionally, the set of patterns generated from the urls is very similar
1028 // to the patterns generated in ContainsBrowseUrl() and other database checks,
1029 // which are called from this code. Refactoring that across the checks could
1030 // interact well with batching the checks here.
1032 // TODO(gab): Fix the fact that Get(Url|Hash)SeverestThreatType() may return a
1033 // threat for which IsExpectedThreat() returns false even if |full_hashes|
1034 // actually contains an expected threat.
1036 for (size_t i = 0; i < check->urls.size(); ++i) {
1037 size_t threat_index;
1038 SBThreatType threat =
1039 GetUrlSeverestThreatType(check->urls[i], full_hashes, &threat_index);
1040 if (threat != SB_THREAT_TYPE_SAFE &&
1041 IsExpectedThreat(threat, check->expected_threats)) {
1042 check->url_results[i] = threat;
1043 check->url_metadata[i] = full_hashes[threat_index].metadata;
1044 is_threat = true;
1048 for (size_t i = 0; i < check->full_hashes.size(); ++i) {
1049 SBThreatType threat =
1050 GetHashSeverestThreatType(check->full_hashes[i], full_hashes);
1051 if (threat != SB_THREAT_TYPE_SAFE &&
1052 IsExpectedThreat(threat, check->expected_threats)) {
1053 check->full_hash_results[i] = threat;
1054 is_threat = true;
1058 SafeBrowsingCheckDone(check);
1059 return is_threat;
1062 void SafeBrowsingDatabaseManager::OnAsyncCheckDone(
1063 SafeBrowsingCheck* check,
1064 const std::vector<SBPrefix>& prefix_hits) {
1065 DCHECK_CURRENTLY_ON(BrowserThread::IO);
1066 DCHECK(enable_download_protection_);
1068 check->prefix_hits = prefix_hits;
1069 if (check->prefix_hits.empty()) {
1070 SafeBrowsingCheckDone(check);
1071 } else {
1072 check->need_get_hash = true;
1073 OnCheckDone(check);
1077 std::vector<SBPrefix> SafeBrowsingDatabaseManager::CheckDownloadUrlOnSBThread(
1078 const std::vector<SBPrefix>& prefixes) {
1079 DCHECK(safe_browsing_task_runner_->RunsTasksOnCurrentThread());
1080 DCHECK(enable_download_protection_);
1082 std::vector<SBPrefix> prefix_hits;
1083 const bool result =
1084 database_->ContainsDownloadUrlPrefixes(prefixes, &prefix_hits);
1085 DCHECK_EQ(result, !prefix_hits.empty());
1086 return prefix_hits;
1089 std::vector<SBPrefix> SafeBrowsingDatabaseManager::CheckExtensionIDsOnSBThread(
1090 const std::vector<SBPrefix>& prefixes) {
1091 DCHECK(safe_browsing_task_runner_->RunsTasksOnCurrentThread());
1093 std::vector<SBPrefix> prefix_hits;
1094 const bool result =
1095 database_->ContainsExtensionPrefixes(prefixes, &prefix_hits);
1096 DCHECK_EQ(result, !prefix_hits.empty());
1097 return prefix_hits;
1100 void SafeBrowsingDatabaseManager::TimeoutCallback(SafeBrowsingCheck* check) {
1101 DCHECK_CURRENTLY_ON(BrowserThread::IO);
1102 DCHECK(check);
1104 if (!enabled_)
1105 return;
1107 DCHECK(checks_.find(check) != checks_.end());
1108 if (check->client) {
1109 check->client->OnSafeBrowsingResult(*check);
1110 check->client = NULL;
1114 void SafeBrowsingDatabaseManager::SafeBrowsingCheckDone(
1115 SafeBrowsingCheck* check) {
1116 DCHECK_CURRENTLY_ON(BrowserThread::IO);
1117 DCHECK(check);
1119 if (!enabled_)
1120 return;
1122 DVLOG(1) << "SafeBrowsingCheckDone";
1123 DCHECK(checks_.find(check) != checks_.end());
1124 if (check->client)
1125 check->client->OnSafeBrowsingResult(*check);
1126 checks_.erase(check);
1127 delete check;
1130 void SafeBrowsingDatabaseManager::StartSafeBrowsingCheck(
1131 SafeBrowsingCheck* check,
1132 const base::Callback<std::vector<SBPrefix>(void)>& task) {
1133 DCHECK_CURRENTLY_ON(BrowserThread::IO);
1134 check->weak_ptr_factory_.reset(
1135 new base::WeakPtrFactory<SafeBrowsingDatabaseManager>(this));
1136 checks_.insert(check);
1138 base::PostTaskAndReplyWithResult(
1139 safe_browsing_task_runner_.get(), FROM_HERE, task,
1140 base::Bind(&SafeBrowsingDatabaseManager::OnAsyncCheckDone,
1141 check->weak_ptr_factory_->GetWeakPtr(), check));
1142 base::MessageLoop::current()->PostDelayedTask(FROM_HERE,
1143 base::Bind(&SafeBrowsingDatabaseManager::TimeoutCallback,
1144 check->weak_ptr_factory_->GetWeakPtr(), check),
1145 check_timeout_);