chrome/browser/safe_browsing/database_manager.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "chrome/browser/safe_browsing/database_manager.h"
   6
   7 #include <algorithm>
   8
   9 #include "base/bind.h"
  10 #include "base/bind_helpers.h"
  11 #include "base/callback.h"
  12 #include "base/command_line.h"
  13 #include "base/debug/leak_tracker.h"
  14 #include "base/path_service.h"
  15 #include "base/stl_util.h"
  16 #include "base/strings/string_util.h"
  17 #include "base/threading/thread.h"
  18 #include "base/threading/thread_restrictions.h"
  19 #include "chrome/browser/browser_process.h"
  20 #include "chrome/browser/chrome_notification_types.h"
  21 #include "chrome/browser/metrics/metrics_service.h"
  22 #include "chrome/browser/prerender/prerender_field_trial.h"
  23 #include "chrome/browser/safe_browsing/client_side_detection_service.h"
  24 #include "chrome/browser/safe_browsing/download_protection_service.h"
  25 #include "chrome/browser/safe_browsing/malware_details.h"
  26 #include "chrome/browser/safe_browsing/protocol_manager.h"
  27 #include "chrome/browser/safe_browsing/safe_browsing_database.h"
  28 #include "chrome/browser/safe_browsing/safe_browsing_service.h"
  29 #include "chrome/browser/safe_browsing/ui_manager.h"
  30 #include "chrome/common/chrome_constants.h"
  31 #include "chrome/common/chrome_paths.h"
  32 #include "chrome/common/chrome_switches.h"
  33 #include "chrome/common/url_constants.h"
  34 #include "components/startup_metric_utils/startup_metric_utils.h"
  35 #include "content/public/browser/browser_thread.h"
  36 #include "content/public/browser/notification_service.h"
  37
  38 using content::BrowserThread;
  39
  40 namespace {
  41
  42 // Timeout for match checks, e.g. download URLs, hashes.
  43 const int kCheckTimeoutMs = 10000;
  44
  45 // Records disposition information about the check.  |hit| should be
  46 // |true| if there were any prefix hits in |full_hashes|.
  47 void RecordGetHashCheckStatus(
  48     bool hit,
  49     safe_browsing_util::ListType check_type,
  50     const std::vector<SBFullHashResult>& full_hashes) {
  51   SafeBrowsingProtocolManager::ResultType result;
  52   if (full_hashes.empty()) {
  53     result = SafeBrowsingProtocolManager::GET_HASH_FULL_HASH_EMPTY;
  54   } else if (hit) {
  55     result = SafeBrowsingProtocolManager::GET_HASH_FULL_HASH_HIT;
  56   } else {
  57     result = SafeBrowsingProtocolManager::GET_HASH_FULL_HASH_MISS;
  58   }
  59   bool is_download = check_type == safe_browsing_util::BINURL ||
  60                      check_type == safe_browsing_util::BINHASH;
  61   SafeBrowsingProtocolManager::RecordGetHashResult(is_download, result);
  62 }
  63
  64 bool IsExpectedThreat(
  65     const SBThreatType threat_type,
  66     const std::vector<SBThreatType>& expected_threats) {
  67   return expected_threats.end() != std::find(expected_threats.begin(),
  68                                              expected_threats.end(),
  69                                              threat_type);
  70 }
  71
  72 }  // namespace
  73
  74 SafeBrowsingDatabaseManager::SafeBrowsingCheck::SafeBrowsingCheck(
  75     const std::vector<GURL>& urls,
  76     const std::vector<SBFullHash>& full_hashes,
  77     Client* client,
  78     safe_browsing_util::ListType check_type,
  79     const std::vector<SBThreatType>& expected_threats)
  80     : urls(urls),
  81       url_results(urls.size(), SB_THREAT_TYPE_SAFE),
  82       full_hashes(full_hashes),
  83       full_hash_results(full_hashes.size(), SB_THREAT_TYPE_SAFE),
  84       client(client),
  85       need_get_hash(false),
  86       check_type(check_type),
  87       expected_threats(expected_threats) {
  88   DCHECK_EQ(urls.empty(), !full_hashes.empty())
  89       << "Exactly one of urls and full_hashes must be set";
  90 }
  91
  92 SafeBrowsingDatabaseManager::SafeBrowsingCheck::~SafeBrowsingCheck() {}
  93
  94 void SafeBrowsingDatabaseManager::Client::OnSafeBrowsingResult(
  95     const SafeBrowsingCheck& check) {
  96   DCHECK_EQ(check.urls.size(), check.url_results.size());
  97   DCHECK_EQ(check.full_hashes.size(), check.full_hash_results.size());
  98   if (!check.urls.empty()) {
  99     DCHECK(check.full_hashes.empty());
 100     switch (check.check_type) {
 101       case safe_browsing_util::MALWARE:
 102       case safe_browsing_util::PHISH:
 103         DCHECK_EQ(1u, check.urls.size());
 104         OnCheckBrowseUrlResult(check.urls[0], check.url_results[0]);
 105         break;
 106       case safe_browsing_util::BINURL:
 107         DCHECK_EQ(check.urls.size(), check.url_results.size());
 108         OnCheckDownloadUrlResult(
 109             check.urls,
 110             *std::max_element(check.url_results.begin(),
 111                               check.url_results.end()));
 112         break;
 113       default:
 114         NOTREACHED();
 115     }
 116   } else if (!check.full_hashes.empty()) {
 117     switch (check.check_type) {
 118       case safe_browsing_util::BINHASH:
 119         DCHECK_EQ(1u, check.full_hashes.size());
 120         OnCheckDownloadHashResult(
 121             safe_browsing_util::SBFullHashToString(check.full_hashes[0]),
 122             check.full_hash_results[0]);
 123         break;
 124       case safe_browsing_util::EXTENSIONBLACKLIST: {
 125         std::set<std::string> unsafe_extension_ids;
 126         for (size_t i = 0; i < check.full_hashes.size(); ++i) {
 127           std::string extension_id =
 128               safe_browsing_util::SBFullHashToString(check.full_hashes[i]);
 129           if (check.full_hash_results[i] == SB_THREAT_TYPE_EXTENSION)
 130             unsafe_extension_ids.insert(extension_id);
 131         }
 132         OnCheckExtensionsResult(unsafe_extension_ids);
 133         break;
 134       }
 135       default:
 136         NOTREACHED();
 137     }
 138   } else {
 139     NOTREACHED();
 140   }
 141 }
 142
 143 SafeBrowsingDatabaseManager::SafeBrowsingDatabaseManager(
 144     const scoped_refptr<SafeBrowsingService>& service)
 145     : sb_service_(service),
 146       database_(NULL),
 147       enabled_(false),
 148       enable_download_protection_(false),
 149       enable_csd_whitelist_(false),
 150       enable_download_whitelist_(false),
 151       enable_extension_blacklist_(false),
 152       enable_side_effect_free_whitelist_(false),
 153       enable_ip_blacklist_(false),
 154       update_in_progress_(false),
 155       database_update_in_progress_(false),
 156       closing_database_(false),
 157       check_timeout_(base::TimeDelta::FromMilliseconds(kCheckTimeoutMs)) {
 158   DCHECK(sb_service_.get() != NULL);
 159
 160   CommandLine* cmdline = CommandLine::ForCurrentProcess();
 161   enable_download_protection_ =
 162       !cmdline->HasSwitch(switches::kSbDisableDownloadProtection);
 163
 164   // We only download the csd-whitelist if client-side phishing detection is
 165   // enabled.
 166   enable_csd_whitelist_ =
 167       !cmdline->HasSwitch(switches::kDisableClientSidePhishingDetection);
 168
 169   // TODO(noelutz): remove this boolean variable since it should always be true
 170   // if SafeBrowsing is enabled.  Unfortunately, we have no test data for this
 171   // list right now.  This means that we need to be able to disable this list
 172   // for the SafeBrowsing test to pass.
 173   enable_download_whitelist_ = enable_csd_whitelist_;
 174
 175   // TODO(kalman): there really shouldn't be a flag for this.
 176   enable_extension_blacklist_ =
 177       !cmdline->HasSwitch(switches::kSbDisableExtensionBlacklist);
 178
 179   enable_side_effect_free_whitelist_ =
 180       prerender::IsSideEffectFreeWhitelistEnabled() &&
 181       !cmdline->HasSwitch(switches::kSbDisableSideEffectFreeWhitelist);
 182
 183   // The client-side IP blacklist feature is tightly integrated with client-side
 184   // phishing protection for now.
 185   enable_ip_blacklist_ = enable_csd_whitelist_;
 186
 187   enum SideEffectFreeWhitelistStatus {
 188     SIDE_EFFECT_FREE_WHITELIST_ENABLED,
 189     SIDE_EFFECT_FREE_WHITELIST_DISABLED,
 190     SIDE_EFFECT_FREE_WHITELIST_STATUS_MAX
 191   };
 192
 193   SideEffectFreeWhitelistStatus side_effect_free_whitelist_status =
 194       enable_side_effect_free_whitelist_ ? SIDE_EFFECT_FREE_WHITELIST_ENABLED :
 195       SIDE_EFFECT_FREE_WHITELIST_DISABLED;
 196
 197   UMA_HISTOGRAM_ENUMERATION("SB2.SideEffectFreeWhitelistStatus",
 198                             side_effect_free_whitelist_status,
 199                             SIDE_EFFECT_FREE_WHITELIST_STATUS_MAX);
 200 }
 201
 202 SafeBrowsingDatabaseManager::~SafeBrowsingDatabaseManager() {
 203   // We should have already been shut down. If we're still enabled, then the
 204   // database isn't going to be closed properly, which could lead to corruption.
 205   DCHECK(!enabled_);
 206 }
 207
 208 bool SafeBrowsingDatabaseManager::CanCheckUrl(const GURL& url) const {
 209   return url.SchemeIs(content::kFtpScheme) ||
 210          url.SchemeIs(content::kHttpScheme) ||
 211          url.SchemeIs(content::kHttpsScheme);
 212 }
 213
 214 bool SafeBrowsingDatabaseManager::CheckDownloadUrl(
 215     const std::vector<GURL>& url_chain,
 216     Client* client) {
 217   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 218   if (!enabled_ || !enable_download_protection_)
 219     return true;
 220
 221   // We need to check the database for url prefix, and later may fetch the url
 222   // from the safebrowsing backends. These need to be asynchronous.
 223   SafeBrowsingCheck* check =
 224       new SafeBrowsingCheck(url_chain,
 225                             std::vector<SBFullHash>(),
 226                             client,
 227                             safe_browsing_util::BINURL,
 228                             std::vector<SBThreatType>(1,
 229                                 SB_THREAT_TYPE_BINARY_MALWARE_URL));
 230   StartSafeBrowsingCheck(
 231       check,
 232       base::Bind(&SafeBrowsingDatabaseManager::CheckDownloadUrlOnSBThread, this,
 233                  check));
 234   return false;
 235 }
 236
 237 bool SafeBrowsingDatabaseManager::CheckDownloadHash(
 238     const std::string& full_hash,
 239     Client* client) {
 240   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 241   DCHECK(!full_hash.empty());
 242   if (!enabled_ || !enable_download_protection_ || full_hash.empty())
 243     return true;
 244
 245   // We need to check the database for url prefix, and later may fetch the url
 246   // from the safebrowsing backends. These need to be asynchronous.
 247   std::vector<SBFullHash> full_hashes(
 248       1, safe_browsing_util::StringToSBFullHash(full_hash));
 249   SafeBrowsingCheck* check =
 250       new SafeBrowsingCheck(std::vector<GURL>(),
 251                             full_hashes,
 252                             client,
 253                             safe_browsing_util::BINHASH,
 254                             std::vector<SBThreatType>(1,
 255                                 SB_THREAT_TYPE_BINARY_MALWARE_HASH));
 256   StartSafeBrowsingCheck(
 257       check,
 258       base::Bind(&SafeBrowsingDatabaseManager::CheckDownloadHashOnSBThread,this,
 259                  check));
 260   return false;
 261 }
 262
 263 bool SafeBrowsingDatabaseManager::CheckExtensionIDs(
 264     const std::set<std::string>& extension_ids,
 265     Client* client) {
 266   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 267
 268   if (!enabled_ || !enable_extension_blacklist_)
 269     return true;
 270
 271   std::vector<SBFullHash> extension_id_hashes;
 272   std::transform(extension_ids.begin(), extension_ids.end(),
 273                  std::back_inserter(extension_id_hashes),
 274                  safe_browsing_util::StringToSBFullHash);
 275
 276   SafeBrowsingCheck* check = new SafeBrowsingCheck(
 277       std::vector<GURL>(),
 278       extension_id_hashes,
 279       client,
 280       safe_browsing_util::EXTENSIONBLACKLIST,
 281       std::vector<SBThreatType>(1, SB_THREAT_TYPE_EXTENSION));
 282
 283   StartSafeBrowsingCheck(
 284       check,
 285       base::Bind(&SafeBrowsingDatabaseManager::CheckExtensionIDsOnSBThread,
 286                  this,
 287                  check));
 288   return false;
 289 }
 290
 291 bool SafeBrowsingDatabaseManager::CheckSideEffectFreeWhitelistUrl(
 292     const GURL& url) {
 293   if (!enabled_)
 294     return false;
 295
 296   if (!CanCheckUrl(url))
 297     return false;
 298
 299   return database_->ContainsSideEffectFreeWhitelistUrl(url);
 300 }
 301
 302 bool SafeBrowsingDatabaseManager::MatchMalwareIP(
 303     const std::string& ip_address) {
 304   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 305   if (!enabled_ || !enable_ip_blacklist_ || !MakeDatabaseAvailable()) {
 306     return false;  // Fail open.
 307   }
 308   return database_->ContainsMalwareIP(ip_address);
 309 }
 310
 311 bool SafeBrowsingDatabaseManager::MatchCsdWhitelistUrl(const GURL& url) {
 312   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 313   if (!enabled_ || !enable_csd_whitelist_ || !MakeDatabaseAvailable()) {
 314     // There is something funky going on here -- for example, perhaps the user
 315     // has not restarted since enabling metrics reporting, so we haven't
 316     // enabled the csd whitelist yet.  Just to be safe we return true in this
 317     // case.
 318     return true;
 319   }
 320   return database_->ContainsCsdWhitelistedUrl(url);
 321 }
 322
 323 bool SafeBrowsingDatabaseManager::MatchDownloadWhitelistUrl(const GURL& url) {
 324   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 325   if (!enabled_ || !enable_download_whitelist_ || !MakeDatabaseAvailable()) {
 326     return true;
 327   }
 328   return database_->ContainsDownloadWhitelistedUrl(url);
 329 }
 330
 331 bool SafeBrowsingDatabaseManager::MatchDownloadWhitelistString(
 332     const std::string& str) {
 333   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 334   if (!enabled_ || !enable_download_whitelist_ || !MakeDatabaseAvailable()) {
 335     return true;
 336   }
 337   return database_->ContainsDownloadWhitelistedString(str);
 338 }
 339
 340 bool SafeBrowsingDatabaseManager::IsMalwareKillSwitchOn() {
 341   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 342   if (!enabled_ || !MakeDatabaseAvailable()) {
 343     return true;
 344   }
 345   return database_->IsMalwareIPMatchKillSwitchOn();
 346 }
 347
 348 bool SafeBrowsingDatabaseManager::CheckBrowseUrl(const GURL& url,
 349                                                  Client* client) {
 350   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 351   if (!enabled_)
 352     return true;
 353
 354   if (!CanCheckUrl(url))
 355     return true;
 356
 357   std::vector<SBThreatType> expected_threats;
 358   expected_threats.push_back(SB_THREAT_TYPE_URL_MALWARE);
 359   expected_threats.push_back(SB_THREAT_TYPE_URL_PHISHING);
 360
 361   const base::TimeTicks start = base::TimeTicks::Now();
 362   if (!MakeDatabaseAvailable()) {
 363     QueuedCheck queued_check(safe_browsing_util::MALWARE,  // or PHISH
 364                              client,
 365                              url,
 366                              expected_threats,
 367                              start);
 368     queued_checks_.push_back(queued_check);
 369     return false;
 370   }
 371
 372   std::string list;
 373   std::vector<SBPrefix> prefix_hits;
 374   std::vector<SBFullHashResult> full_hits;
 375
 376   bool prefix_match =
 377       database_->ContainsBrowseUrl(url, &list, &prefix_hits, &full_hits,
 378           sb_service_->protocol_manager()->last_update());
 379
 380   UMA_HISTOGRAM_TIMES("SB2.FilterCheck", base::TimeTicks::Now() - start);
 381
 382   if (!prefix_match)
 383     return true;  // URL is okay.
 384
 385   // Needs to be asynchronous, since we could be in the constructor of a
 386   // ResourceDispatcherHost event handler which can't pause there.
 387   SafeBrowsingCheck* check = new SafeBrowsingCheck(std::vector<GURL>(1, url),
 388                                                    std::vector<SBFullHash>(),
 389                                                    client,
 390                                                    safe_browsing_util::MALWARE,
 391                                                    expected_threats);
 392   check->need_get_hash = full_hits.empty();
 393   check->prefix_hits.swap(prefix_hits);
 394   check->full_hits.swap(full_hits);
 395   checks_.insert(check);
 396
 397   BrowserThread::PostTask(
 398       BrowserThread::IO, FROM_HERE,
 399       base::Bind(&SafeBrowsingDatabaseManager::OnCheckDone, this, check));
 400
 401   return false;
 402 }
 403
 404 void SafeBrowsingDatabaseManager::CancelCheck(Client* client) {
 405   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 406   for (CurrentChecks::iterator i = checks_.begin(); i != checks_.end(); ++i) {
 407     // We can't delete matching checks here because the db thread has a copy of
 408     // the pointer.  Instead, we simply NULL out the client, and when the db
 409     // thread calls us back, we'll clean up the check.
 410     if ((*i)->client == client)
 411       (*i)->client = NULL;
 412   }
 413
 414   // Scan the queued clients store. Clients may be here if they requested a URL
 415   // check before the database has finished loading.
 416   for (std::deque<QueuedCheck>::iterator it(queued_checks_.begin());
 417        it != queued_checks_.end(); ) {
 418     // In this case it's safe to delete matches entirely since nothing has a
 419     // pointer to them.
 420     if (it->client == client)
 421       it = queued_checks_.erase(it);
 422     else
 423       ++it;
 424   }
 425 }
 426
 427 void SafeBrowsingDatabaseManager::HandleGetHashResults(
 428     SafeBrowsingCheck* check,
 429     const std::vector<SBFullHashResult>& full_hashes,
 430     bool can_cache) {
 431   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 432
 433   if (!enabled_)
 434     return;
 435
 436   // If the service has been shut down, |check| should have been deleted.
 437   DCHECK(checks_.find(check) != checks_.end());
 438
 439   // |start| is set before calling |GetFullHash()|, which should be
 440   // the only path which gets to here.
 441   DCHECK(!check->start.is_null());
 442   UMA_HISTOGRAM_LONG_TIMES("SB2.Network",
 443                            base::TimeTicks::Now() - check->start);
 444
 445   std::vector<SBPrefix> prefixes = check->prefix_hits;
 446   OnHandleGetHashResults(check, full_hashes);  // 'check' is deleted here.
 447
 448   if (can_cache && MakeDatabaseAvailable()) {
 449     // Cache the GetHash results in memory:
 450     database_->CacheHashResults(prefixes, full_hashes);
 451   }
 452 }
 453
 454 void SafeBrowsingDatabaseManager::GetChunks(GetChunksCallback callback) {
 455   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 456   DCHECK(enabled_);
 457   DCHECK(!callback.is_null());
 458   safe_browsing_thread_->message_loop()->PostTask(FROM_HERE, base::Bind(
 459       &SafeBrowsingDatabaseManager::GetAllChunksFromDatabase, this, callback));
 460 }
 461
 462 void SafeBrowsingDatabaseManager::AddChunks(const std::string& list,
 463                                             SBChunkList* chunks,
 464                                             AddChunksCallback callback) {
 465   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 466   DCHECK(enabled_);
 467   DCHECK(!callback.is_null());
 468   safe_browsing_thread_->message_loop()->PostTask(FROM_HERE, base::Bind(
 469       &SafeBrowsingDatabaseManager::AddDatabaseChunks, this, list,
 470       chunks, callback));
 471 }
 472
 473 void SafeBrowsingDatabaseManager::DeleteChunks(
 474     std::vector<SBChunkDelete>* chunk_deletes) {
 475   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 476   DCHECK(enabled_);
 477   safe_browsing_thread_->message_loop()->PostTask(FROM_HERE, base::Bind(
 478       &SafeBrowsingDatabaseManager::DeleteDatabaseChunks, this, chunk_deletes));
 479 }
 480
 481 void SafeBrowsingDatabaseManager::UpdateStarted() {
 482   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 483   DCHECK(enabled_);
 484   DCHECK(!update_in_progress_);
 485   update_in_progress_ = true;
 486 }
 487
 488 void SafeBrowsingDatabaseManager::UpdateFinished(bool update_succeeded) {
 489   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 490   DCHECK(enabled_);
 491   if (update_in_progress_) {
 492     update_in_progress_ = false;
 493     safe_browsing_thread_->message_loop()->PostTask(FROM_HERE,
 494       base::Bind(&SafeBrowsingDatabaseManager::DatabaseUpdateFinished,
 495                  this, update_succeeded));
 496   }
 497 }
 498
 499 void SafeBrowsingDatabaseManager::ResetDatabase() {
 500   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 501   DCHECK(enabled_);
 502   safe_browsing_thread_->message_loop()->PostTask(FROM_HERE, base::Bind(
 503       &SafeBrowsingDatabaseManager::OnResetDatabase, this));
 504 }
 505
 506 void SafeBrowsingDatabaseManager::PurgeMemory() {
 507   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 508   CloseDatabase();
 509 }
 510
 511 void SafeBrowsingDatabaseManager::LogPauseDelay(base::TimeDelta time) {
 512   UMA_HISTOGRAM_LONG_TIMES("SB2.Delay", time);
 513 }
 514
 515 void SafeBrowsingDatabaseManager::StartOnIOThread() {
 516   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 517   if (enabled_)
 518     return;
 519
 520   DCHECK(!safe_browsing_thread_.get());
 521   safe_browsing_thread_.reset(new base::Thread("Chrome_SafeBrowsingThread"));
 522   if (!safe_browsing_thread_->Start())
 523     return;
 524   enabled_ = true;
 525
 526   MakeDatabaseAvailable();
 527 }
 528
 529 void SafeBrowsingDatabaseManager::StopOnIOThread(bool shutdown) {
 530   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 531
 532   DoStopOnIOThread();
 533   if (shutdown) {
 534     sb_service_ = NULL;
 535   }
 536 }
 537
 538 void SafeBrowsingDatabaseManager::NotifyDatabaseUpdateFinished(
 539     bool update_succeeded) {
 540   content::NotificationService::current()->Notify(
 541       chrome::NOTIFICATION_SAFE_BROWSING_UPDATE_COMPLETE,
 542       content::Source<SafeBrowsingDatabaseManager>(this),
 543       content::Details<bool>(&update_succeeded));
 544 }
 545
 546 SafeBrowsingDatabaseManager::QueuedCheck::QueuedCheck(
 547     const safe_browsing_util::ListType check_type,
 548     Client* client,
 549     const GURL& url,
 550     const std::vector<SBThreatType>& expected_threats,
 551     const base::TimeTicks& start)
 552     : check_type(check_type),
 553       client(client),
 554       url(url),
 555       expected_threats(expected_threats),
 556       start(start) {
 557 }
 558
 559 SafeBrowsingDatabaseManager::QueuedCheck::~QueuedCheck() {
 560 }
 561
 562 void SafeBrowsingDatabaseManager::DoStopOnIOThread() {
 563   if (!enabled_)
 564     return;
 565
 566   enabled_ = false;
 567
 568   // Delete queued checks, calling back any clients with 'SB_THREAT_TYPE_SAFE'.
 569   // If we don't do this here we may fail to close the database below.
 570   while (!queued_checks_.empty()) {
 571     QueuedCheck queued = queued_checks_.front();
 572     if (queued.client) {
 573       SafeBrowsingCheck sb_check(std::vector<GURL>(1, queued.url),
 574                                  std::vector<SBFullHash>(),
 575                                  queued.client,
 576                                  queued.check_type,
 577                                  queued.expected_threats);
 578       queued.client->OnSafeBrowsingResult(sb_check);
 579     }
 580     queued_checks_.pop_front();
 581   }
 582
 583   // Close the database.  We don't simply DeleteSoon() because if a close is
 584   // already pending, we'll double-free, and we don't set |database_| to NULL
 585   // because if there is still anything running on the db thread, it could
 586   // create a new database object (via GetDatabase()) that would then leak.
 587   CloseDatabase();
 588
 589   // Flush the database thread. Any in-progress database check results will be
 590   // ignored and cleaned up below.
 591   //
 592   // Note that to avoid leaking the database, we rely on the fact that no new
 593   // tasks will be added to the db thread between the call above and this one.
 594   // See comments on the declaration of |safe_browsing_thread_|.
 595   {
 596     // A ScopedAllowIO object is required to join the thread when calling Stop.
 597     // See http://crbug.com/72696. Note that we call Stop() first to clear out
 598     // any remaining tasks before clearing safe_browsing_thread_.
 599     base::ThreadRestrictions::ScopedAllowIO allow_io_for_thread_join;
 600     safe_browsing_thread_->Stop();
 601     safe_browsing_thread_.reset();
 602   }
 603
 604   // Delete pending checks, calling back any clients with 'SB_THREAT_TYPE_SAFE'.
 605   // We have to do this after the db thread returns because methods on it can
 606   // have copies of these pointers, so deleting them might lead to accessing
 607   // garbage.
 608   for (CurrentChecks::iterator it = checks_.begin();
 609        it != checks_.end(); ++it) {
 610     SafeBrowsingCheck* check = *it;
 611     if (check->client)
 612       check->client->OnSafeBrowsingResult(*check);
 613   }
 614   STLDeleteElements(&checks_);
 615
 616   gethash_requests_.clear();
 617 }
 618
 619 bool SafeBrowsingDatabaseManager::DatabaseAvailable() const {
 620   base::AutoLock lock(database_lock_);
 621   return !closing_database_ && (database_ != NULL);
 622 }
 623
 624 bool SafeBrowsingDatabaseManager::MakeDatabaseAvailable() {
 625   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 626   DCHECK(enabled_);
 627   if (DatabaseAvailable())
 628     return true;
 629   safe_browsing_thread_->message_loop()->PostTask(
 630       FROM_HERE,
 631       base::Bind(base::IgnoreResult(&SafeBrowsingDatabaseManager::GetDatabase),
 632                  this));
 633   return false;
 634 }
 635
 636 void SafeBrowsingDatabaseManager::CloseDatabase() {
 637   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 638
 639   // Cases to avoid:
 640   //  * If |closing_database_| is true, continuing will queue up a second
 641   //    request, |closing_database_| will be reset after handling the first
 642   //    request, and if any functions on the db thread recreate the database, we
 643   //    could start using it on the IO thread and then have the second request
 644   //    handler delete it out from under us.
 645   //  * If |database_| is NULL, then either no creation request is in flight, in
 646   //    which case we don't need to do anything, or one is in flight, in which
 647   //    case the database will be recreated before our deletion request is
 648   //    handled, and could be used on the IO thread in that time period, leading
 649   //    to the same problem as above.
 650   //  * If |queued_checks_| is non-empty and |database_| is non-NULL, we're
 651   //    about to be called back (in DatabaseLoadComplete()).  This will call
 652   //    CheckUrl(), which will want the database.  Closing the database here
 653   //    would lead to an infinite loop in DatabaseLoadComplete(), and even if it
 654   //    didn't, it would be pointless since we'd just want to recreate.
 655   //
 656   // The first two cases above are handled by checking DatabaseAvailable().
 657   if (!DatabaseAvailable() || !queued_checks_.empty())
 658     return;
 659
 660   closing_database_ = true;
 661   if (safe_browsing_thread_.get()) {
 662     safe_browsing_thread_->message_loop()->PostTask(FROM_HERE,
 663         base::Bind(&SafeBrowsingDatabaseManager::OnCloseDatabase, this));
 664   }
 665 }
 666
 667 SafeBrowsingDatabase* SafeBrowsingDatabaseManager::GetDatabase() {
 668   DCHECK_EQ(base::MessageLoop::current(),
 669             safe_browsing_thread_->message_loop());
 670   if (database_)
 671     return database_;
 672   startup_metric_utils::ScopedSlowStartupUMA
 673       scoped_timer("Startup.SlowStartupSafeBrowsingGetDatabase");
 674   const base::TimeTicks before = base::TimeTicks::Now();
 675
 676   SafeBrowsingDatabase* database =
 677       SafeBrowsingDatabase::Create(enable_download_protection_,
 678                                    enable_csd_whitelist_,
 679                                    enable_download_whitelist_,
 680                                    enable_extension_blacklist_,
 681                                    enable_side_effect_free_whitelist_,
 682                                    enable_ip_blacklist_);
 683
 684   database->Init(SafeBrowsingService::GetBaseFilename());
 685   {
 686     // Acquiring the lock here guarantees correct ordering between the writes to
 687     // the new database object above, and the setting of |databse_| below.
 688     base::AutoLock lock(database_lock_);
 689     database_ = database;
 690   }
 691
 692   BrowserThread::PostTask(
 693       BrowserThread::IO, FROM_HERE,
 694       base::Bind(&SafeBrowsingDatabaseManager::DatabaseLoadComplete, this));
 695
 696   UMA_HISTOGRAM_TIMES("SB2.DatabaseOpen", base::TimeTicks::Now() - before);
 697   return database_;
 698 }
 699
 700 void SafeBrowsingDatabaseManager::OnCheckDone(SafeBrowsingCheck* check) {
 701   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 702
 703   if (!enabled_)
 704     return;
 705
 706   // If the service has been shut down, |check| should have been deleted.
 707   DCHECK(checks_.find(check) != checks_.end());
 708
 709   if (check->client && check->need_get_hash) {
 710     // We have a partial match so we need to query Google for the full hash.
 711     // Clean up will happen in HandleGetHashResults.
 712
 713     // See if we have a GetHash request already in progress for this particular
 714     // prefix. If so, we just append ourselves to the list of interested parties
 715     // when the results arrive. We only do this for checks involving one prefix,
 716     // since that is the common case (multiple prefixes will issue the request
 717     // as normal).
 718     if (check->prefix_hits.size() == 1) {
 719       SBPrefix prefix = check->prefix_hits[0];
 720       GetHashRequests::iterator it = gethash_requests_.find(prefix);
 721       if (it != gethash_requests_.end()) {
 722         // There's already a request in progress.
 723         it->second.push_back(check);
 724         return;
 725       }
 726
 727       // No request in progress, so we're the first for this prefix.
 728       GetHashRequestors requestors;
 729       requestors.push_back(check);
 730       gethash_requests_[prefix] = requestors;
 731     }
 732
 733     // Reset the start time so that we can measure the network time without the
 734     // database time.
 735     check->start = base::TimeTicks::Now();
 736     // Note: If |this| is deleted or stopped, the protocol_manager will
 737     // be destroyed as well - hence it's OK to do unretained in this case.
 738     bool is_download = check->check_type == safe_browsing_util::BINURL ||
 739                        check->check_type == safe_browsing_util::BINHASH;
 740     sb_service_->protocol_manager()->GetFullHash(
 741         check->prefix_hits,
 742         base::Bind(&SafeBrowsingDatabaseManager::HandleGetHashResults,
 743                    base::Unretained(this),
 744                    check),
 745         is_download);
 746   } else {
 747     // We may have cached results for previous GetHash queries.  Since
 748     // this data comes from cache, don't histogram hits.
 749     HandleOneCheck(check, check->full_hits);
 750   }
 751 }
 752
 753 void SafeBrowsingDatabaseManager::GetAllChunksFromDatabase(
 754     GetChunksCallback callback) {
 755   DCHECK_EQ(base::MessageLoop::current(),
 756             safe_browsing_thread_->message_loop());
 757
 758   bool database_error = true;
 759   std::vector<SBListChunkRanges> lists;
 760   DCHECK(!database_update_in_progress_);
 761   database_update_in_progress_ = true;
 762   GetDatabase();  // This guarantees that |database_| is non-NULL.
 763   if (database_->UpdateStarted(&lists)) {
 764     database_error = false;
 765   } else {
 766     database_->UpdateFinished(false);
 767   }
 768
 769   BrowserThread::PostTask(
 770       BrowserThread::IO, FROM_HERE,
 771       base::Bind(&SafeBrowsingDatabaseManager::OnGetAllChunksFromDatabase,
 772                  this, lists, database_error, callback));
 773 }
 774
 775 void SafeBrowsingDatabaseManager::OnGetAllChunksFromDatabase(
 776     const std::vector<SBListChunkRanges>& lists, bool database_error,
 777     GetChunksCallback callback) {
 778   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 779   if (enabled_)
 780     callback.Run(lists, database_error);
 781 }
 782
 783 void SafeBrowsingDatabaseManager::OnAddChunksComplete(
 784     AddChunksCallback callback) {
 785   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 786   if (enabled_)
 787     callback.Run();
 788 }
 789
 790 void SafeBrowsingDatabaseManager::DatabaseLoadComplete() {
 791   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 792   if (!enabled_)
 793     return;
 794
 795   HISTOGRAM_COUNTS("SB.QueueDepth", queued_checks_.size());
 796   if (queued_checks_.empty())
 797     return;
 798
 799   // If the database isn't already available, calling CheckUrl() in the loop
 800   // below will add the check back to the queue, and we'll infinite-loop.
 801   DCHECK(DatabaseAvailable());
 802   while (!queued_checks_.empty()) {
 803     QueuedCheck check = queued_checks_.front();
 804     DCHECK(!check.start.is_null());
 805     HISTOGRAM_TIMES("SB.QueueDelay", base::TimeTicks::Now() - check.start);
 806     // If CheckUrl() determines the URL is safe immediately, it doesn't call the
 807     // client's handler function (because normally it's being directly called by
 808     // the client).  Since we're not the client, we have to convey this result.
 809     if (check.client && CheckBrowseUrl(check.url, check.client)) {
 810       SafeBrowsingCheck sb_check(std::vector<GURL>(1, check.url),
 811                                  std::vector<SBFullHash>(),
 812                                  check.client,
 813                                  check.check_type,
 814                                  check.expected_threats);
 815       check.client->OnSafeBrowsingResult(sb_check);
 816     }
 817     queued_checks_.pop_front();
 818   }
 819 }
 820
 821 void SafeBrowsingDatabaseManager::AddDatabaseChunks(
 822     const std::string& list_name, SBChunkList* chunks,
 823     AddChunksCallback callback) {
 824   DCHECK_EQ(base::MessageLoop::current(),
 825             safe_browsing_thread_->message_loop());
 826   if (chunks) {
 827     GetDatabase()->InsertChunks(list_name, *chunks);
 828     delete chunks;
 829   }
 830   BrowserThread::PostTask(
 831       BrowserThread::IO, FROM_HERE,
 832       base::Bind(&SafeBrowsingDatabaseManager::OnAddChunksComplete, this,
 833                  callback));
 834 }
 835
 836 void SafeBrowsingDatabaseManager::DeleteDatabaseChunks(
 837     std::vector<SBChunkDelete>* chunk_deletes) {
 838   DCHECK_EQ(base::MessageLoop::current(),
 839             safe_browsing_thread_->message_loop());
 840   if (chunk_deletes) {
 841     GetDatabase()->DeleteChunks(*chunk_deletes);
 842     delete chunk_deletes;
 843   }
 844 }
 845
 846 SBThreatType SafeBrowsingDatabaseManager::GetThreatTypeFromListname(
 847     const std::string& list_name) {
 848   if (safe_browsing_util::IsPhishingList(list_name)) {
 849     return SB_THREAT_TYPE_URL_PHISHING;
 850   }
 851
 852   if (safe_browsing_util::IsMalwareList(list_name)) {
 853     return SB_THREAT_TYPE_URL_MALWARE;
 854   }
 855
 856   if (safe_browsing_util::IsBadbinurlList(list_name)) {
 857     return SB_THREAT_TYPE_BINARY_MALWARE_URL;
 858   }
 859
 860   if (safe_browsing_util::IsBadbinhashList(list_name)) {
 861     return SB_THREAT_TYPE_BINARY_MALWARE_HASH;
 862   }
 863
 864   if (safe_browsing_util::IsExtensionList(list_name)) {
 865     return SB_THREAT_TYPE_EXTENSION;
 866   }
 867
 868   DVLOG(1) << "Unknown safe browsing list " << list_name;
 869   return SB_THREAT_TYPE_SAFE;
 870 }
 871
 872 void SafeBrowsingDatabaseManager::DatabaseUpdateFinished(
 873     bool update_succeeded) {
 874   DCHECK_EQ(base::MessageLoop::current(),
 875             safe_browsing_thread_->message_loop());
 876   GetDatabase()->UpdateFinished(update_succeeded);
 877   DCHECK(database_update_in_progress_);
 878   database_update_in_progress_ = false;
 879   BrowserThread::PostTask(
 880       BrowserThread::UI, FROM_HERE,
 881       base::Bind(&SafeBrowsingDatabaseManager::NotifyDatabaseUpdateFinished,
 882                  this, update_succeeded));
 883 }
 884
 885 void SafeBrowsingDatabaseManager::OnCloseDatabase() {
 886   DCHECK_EQ(base::MessageLoop::current(),
 887             safe_browsing_thread_->message_loop());
 888   DCHECK(closing_database_);
 889
 890   // Because |closing_database_| is true, nothing on the IO thread will be
 891   // accessing the database, so it's safe to delete and then NULL the pointer.
 892   delete database_;
 893   database_ = NULL;
 894
 895   // Acquiring the lock here guarantees correct ordering between the resetting
 896   // of |database_| above and of |closing_database_| below, which ensures there
 897   // won't be a window during which the IO thread falsely believes the database
 898   // is available.
 899   base::AutoLock lock(database_lock_);
 900   closing_database_ = false;
 901 }
 902
 903 void SafeBrowsingDatabaseManager::OnResetDatabase() {
 904   DCHECK_EQ(base::MessageLoop::current(),
 905             safe_browsing_thread_->message_loop());
 906   GetDatabase()->ResetDatabase();
 907 }
 908
 909 void SafeBrowsingDatabaseManager::CacheHashResults(
 910   const std::vector<SBPrefix>& prefixes,
 911   const std::vector<SBFullHashResult>& full_hashes) {
 912   DCHECK_EQ(base::MessageLoop::current(),
 913             safe_browsing_thread_->message_loop());
 914   GetDatabase()->CacheHashResults(prefixes, full_hashes);
 915 }
 916
 917 void SafeBrowsingDatabaseManager::OnHandleGetHashResults(
 918     SafeBrowsingCheck* check,
 919     const std::vector<SBFullHashResult>& full_hashes) {
 920   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 921   safe_browsing_util::ListType check_type = check->check_type;
 922   SBPrefix prefix = check->prefix_hits[0];
 923   GetHashRequests::iterator it = gethash_requests_.find(prefix);
 924   if (check->prefix_hits.size() > 1 || it == gethash_requests_.end()) {
 925     const bool hit = HandleOneCheck(check, full_hashes);
 926     RecordGetHashCheckStatus(hit, check_type, full_hashes);
 927     return;
 928   }
 929
 930   // Call back all interested parties, noting if any has a hit.
 931   GetHashRequestors& requestors = it->second;
 932   bool hit = false;
 933   for (GetHashRequestors::iterator r = requestors.begin();
 934        r != requestors.end(); ++r) {
 935     if (HandleOneCheck(*r, full_hashes))
 936       hit = true;
 937   }
 938   RecordGetHashCheckStatus(hit, check_type, full_hashes);
 939
 940   gethash_requests_.erase(it);
 941 }
 942
 943 bool SafeBrowsingDatabaseManager::HandleOneCheck(
 944     SafeBrowsingCheck* check,
 945     const std::vector<SBFullHashResult>& full_hashes) {
 946   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 947   DCHECK(check);
 948
 949   bool is_threat = false;
 950
 951   for (size_t i = 0; i < check->urls.size(); ++i) {
 952     int index =
 953         safe_browsing_util::GetUrlHashIndex(check->urls[i], full_hashes);
 954     if (index == -1)
 955       continue;
 956     SBThreatType threat =
 957         GetThreatTypeFromListname(full_hashes[index].list_name);
 958     if (threat != SB_THREAT_TYPE_SAFE &&
 959         IsExpectedThreat(threat, check->expected_threats)) {
 960       check->url_results[i] = threat;
 961       is_threat = true;
 962     }
 963   }
 964
 965   for (size_t i = 0; i < check->full_hashes.size(); ++i) {
 966     int index =
 967         safe_browsing_util::GetHashIndex(check->full_hashes[i], full_hashes);
 968     if (index == -1)
 969       continue;
 970     SBThreatType threat =
 971         GetThreatTypeFromListname(full_hashes[index].list_name);
 972     if (threat != SB_THREAT_TYPE_SAFE &&
 973         IsExpectedThreat(threat, check->expected_threats)) {
 974       check->full_hash_results[i] = threat;
 975       is_threat = true;
 976     }
 977   }
 978
 979   SafeBrowsingCheckDone(check);
 980   return is_threat;
 981 }
 982
 983 void SafeBrowsingDatabaseManager::CheckDownloadHashOnSBThread(
 984     SafeBrowsingCheck* check) {
 985   DCHECK_EQ(base::MessageLoop::current(),
 986             safe_browsing_thread_->message_loop());
 987   DCHECK(enable_download_protection_);
 988
 989   DCHECK_EQ(1u, check->full_hashes.size());
 990   SBFullHash full_hash = check->full_hashes[0];
 991
 992   if (!database_->ContainsDownloadHashPrefix(full_hash.prefix)) {
 993     // Good, we don't have hash for this url prefix.
 994     BrowserThread::PostTask(
 995         BrowserThread::IO, FROM_HERE,
 996         base::Bind(&SafeBrowsingDatabaseManager::CheckDownloadHashDone, this,
 997                    check));
 998     return;
 999   }
1000
1001   check->need_get_hash = true;
1002   check->prefix_hits.push_back(full_hash.prefix);
1003   BrowserThread::PostTask(
1004       BrowserThread::IO, FROM_HERE,
1005       base::Bind(&SafeBrowsingDatabaseManager::OnCheckDone, this, check));
1006 }
1007
1008 void SafeBrowsingDatabaseManager::CheckDownloadUrlOnSBThread(
1009     SafeBrowsingCheck* check) {
1010   DCHECK_EQ(base::MessageLoop::current(),
1011             safe_browsing_thread_->message_loop());
1012   DCHECK(enable_download_protection_);
1013
1014   std::vector<SBPrefix> prefix_hits;
1015
1016   if (!database_->ContainsDownloadUrl(check->urls, &prefix_hits)) {
1017     // Good, we don't have hash for this url prefix.
1018     BrowserThread::PostTask(
1019         BrowserThread::IO, FROM_HERE,
1020         base::Bind(&SafeBrowsingDatabaseManager::CheckDownloadUrlDone, this,
1021                    check));
1022     return;
1023   }
1024
1025   check->need_get_hash = true;
1026   check->prefix_hits.clear();
1027   check->prefix_hits = prefix_hits;
1028   BrowserThread::PostTask(
1029       BrowserThread::IO, FROM_HERE,
1030       base::Bind(&SafeBrowsingDatabaseManager::OnCheckDone, this, check));
1031 }
1032
1033 void SafeBrowsingDatabaseManager::CheckExtensionIDsOnSBThread(
1034     SafeBrowsingCheck* check) {
1035   DCHECK_EQ(base::MessageLoop::current(),
1036             safe_browsing_thread_->message_loop());
1037
1038   std::vector<SBPrefix> prefixes;
1039   for (std::vector<SBFullHash>::iterator it = check->full_hashes.begin();
1040        it != check->full_hashes.end(); ++it) {
1041     prefixes.push_back((*it).prefix);
1042   }
1043   database_->ContainsExtensionPrefixes(prefixes, &check->prefix_hits);
1044
1045   if (check->prefix_hits.empty()) {
1046     // No matches for any extensions.
1047     BrowserThread::PostTask(
1048         BrowserThread::IO,
1049         FROM_HERE,
1050         base::Bind(&SafeBrowsingDatabaseManager::SafeBrowsingCheckDone, this,
1051                    check));
1052   } else {
1053     // Some prefixes matched, we need to ask Google whether they're legit.
1054     check->need_get_hash = true;
1055     BrowserThread::PostTask(
1056         BrowserThread::IO,
1057         FROM_HERE,
1058         base::Bind(&SafeBrowsingDatabaseManager::OnCheckDone, this, check));
1059   }
1060 }
1061
1062 void SafeBrowsingDatabaseManager::TimeoutCallback(SafeBrowsingCheck* check) {
1063   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
1064   DCHECK(check);
1065
1066   if (!enabled_)
1067     return;
1068
1069   DCHECK(checks_.find(check) != checks_.end());
1070   if (check->client) {
1071     check->client->OnSafeBrowsingResult(*check);
1072     check->client = NULL;
1073   }
1074 }
1075
1076 void SafeBrowsingDatabaseManager::CheckDownloadUrlDone(
1077     SafeBrowsingCheck* check) {
1078   DCHECK(enable_download_protection_);
1079   SafeBrowsingCheckDone(check);
1080 }
1081
1082 void SafeBrowsingDatabaseManager::CheckDownloadHashDone(
1083     SafeBrowsingCheck* check) {
1084   DCHECK(enable_download_protection_);
1085   SafeBrowsingCheckDone(check);
1086 }
1087
1088 void SafeBrowsingDatabaseManager::SafeBrowsingCheckDone(
1089     SafeBrowsingCheck* check) {
1090   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
1091   DCHECK(check);
1092
1093   if (!enabled_)
1094     return;
1095
1096   VLOG(1) << "SafeBrowsingCheckDone";
1097   DCHECK(checks_.find(check) != checks_.end());
1098   if (check->client)
1099     check->client->OnSafeBrowsingResult(*check);
1100   checks_.erase(check);
1101   delete check;
1102 }
1103
1104 void SafeBrowsingDatabaseManager::StartSafeBrowsingCheck(
1105     SafeBrowsingCheck* check,
1106     const base::Closure& task) {
1107   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
1108   check->timeout_factory_.reset(
1109       new base::WeakPtrFactory<SafeBrowsingDatabaseManager>(this));
1110   checks_.insert(check);
1111
1112   safe_browsing_thread_->message_loop()->PostTask(FROM_HERE, task);
1113
1114   base::MessageLoop::current()->PostDelayedTask(FROM_HERE,
1115       base::Bind(&SafeBrowsingDatabaseManager::TimeoutCallback,
1116                  check->timeout_factory_->GetWeakPtr(), check),
1117       check_timeout_);
1118 }