chrome/browser/safe_browsing/client_side_detection_service.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "chrome/browser/safe_browsing/client_side_detection_service.h"
   6
   7 #include <algorithm>
   8
   9 #include "base/bind.h"
  10 #include "base/command_line.h"
  11 #include "base/logging.h"
  12 #include "base/memory/scoped_ptr.h"
  13 #include "base/message_loop/message_loop.h"
  14 #include "base/metrics/histogram.h"
  15 #include "base/metrics/sparse_histogram.h"
  16 #include "base/prefs/pref_service.h"
  17 #include "base/profiler/scoped_profile.h"
  18 #include "base/stl_util.h"
  19 #include "base/strings/string_util.h"
  20 #include "base/time/time.h"
  21 #include "chrome/browser/browser_process.h"
  22 #include "chrome/browser/profiles/profile.h"
  23 #include "chrome/common/chrome_switches.h"
  24 #include "chrome/common/pref_names.h"
  25 #include "chrome/common/safe_browsing/client_model.pb.h"
  26 #include "chrome/common/safe_browsing/csd.pb.h"
  27 #include "chrome/common/safe_browsing/safebrowsing_messages.h"
  28 #include "content/public/browser/browser_thread.h"
  29 #include "content/public/browser/notification_service.h"
  30 #include "content/public/browser/notification_types.h"
  31 #include "content/public/browser/render_process_host.h"
  32 #include "crypto/sha2.h"
  33 #include "google_apis/google_api_keys.h"
  34 #include "net/base/escape.h"
  35 #include "net/base/load_flags.h"
  36 #include "net/base/net_util.h"
  37 #include "net/http/http_response_headers.h"
  38 #include "net/http/http_status_code.h"
  39 #include "net/url_request/url_fetcher.h"
  40 #include "net/url_request/url_request_context_getter.h"
  41 #include "net/url_request/url_request_status.h"
  42 #include "url/gurl.h"
  43
  44 using content::BrowserThread;
  45
  46 namespace safe_browsing {
  47
  48 namespace {
  49
  50   // malware report type for UMA histogram counting.
  51   enum MalwareReportTypes {
  52     REPORT_SENT,
  53     REPORT_HIT_LIMIT,
  54     REPORT_FAILED_SERIALIZATION,
  55
  56     // Always at the end
  57     REPORT_RESULT_MAX
  58   };
  59
  60   void UpdateEnumUMAHistogram(MalwareReportTypes report_type) {
  61     DCHECK(report_type >= 0 && report_type < REPORT_RESULT_MAX);
  62     UMA_HISTOGRAM_ENUMERATION("SBClientMalware.SentReports",
  63                               report_type, REPORT_RESULT_MAX);
  64   }
  65
  66 }  // namespace
  67
  68 const size_t ClientSideDetectionService::kMaxModelSizeBytes = 90 * 1024;
  69 const int ClientSideDetectionService::kMaxReportsPerInterval = 3;
  70 // TODO(noelutz): once we know this mechanism works as intended we should fetch
  71 // the model much more frequently.  E.g., every 5 minutes or so.
  72 const int ClientSideDetectionService::kClientModelFetchIntervalMs = 3600 * 1000;
  73 const int ClientSideDetectionService::kInitialClientModelFetchDelayMs = 10000;
  74
  75 const int ClientSideDetectionService::kReportsIntervalDays = 1;
  76 const int ClientSideDetectionService::kNegativeCacheIntervalDays = 1;
  77 const int ClientSideDetectionService::kPositiveCacheIntervalMinutes = 30;
  78
  79 const char ClientSideDetectionService::kClientReportPhishingUrl[] =
  80     "https://sb-ssl.google.com/safebrowsing/clientreport/phishing";
  81 const char ClientSideDetectionService::kClientReportMalwareUrl[] =
  82     "https://sb-ssl.google.com/safebrowsing/clientreport/malware-check";
  83 const char ClientSideDetectionService::kClientModelUrl[] =
  84     "https://ssl.gstatic.com/safebrowsing/csd/client_model_v5.pb";
  85
  86 struct ClientSideDetectionService::ClientReportInfo {
  87   ClientReportPhishingRequestCallback callback;
  88   GURL phishing_url;
  89 };
  90
  91 struct ClientSideDetectionService::ClientMalwareReportInfo {
  92   ClientReportMalwareRequestCallback callback;
  93   // This is the original landing url, may not be the malware url.
  94   GURL original_url;
  95 };
  96
  97 ClientSideDetectionService::CacheState::CacheState(bool phish, base::Time time)
  98     : is_phishing(phish),
  99       timestamp(time) {}
 100
 101 ClientSideDetectionService::ClientSideDetectionService(
 102     net::URLRequestContextGetter* request_context_getter)
 103     : enabled_(false),
 104       request_context_getter_(request_context_getter),
 105       weak_factory_(this) {
 106   registrar_.Add(this, content::NOTIFICATION_RENDERER_PROCESS_CREATED,
 107                  content::NotificationService::AllBrowserContextsAndSources());
 108 }
 109
 110 ClientSideDetectionService::~ClientSideDetectionService() {
 111   weak_factory_.InvalidateWeakPtrs();
 112   STLDeleteContainerPairPointers(client_phishing_reports_.begin(),
 113                                  client_phishing_reports_.end());
 114   client_phishing_reports_.clear();
 115   STLDeleteContainerPairPointers(client_malware_reports_.begin(),
 116                                  client_malware_reports_.end());
 117   client_malware_reports_.clear();
 118 }
 119
 120 // static
 121 ClientSideDetectionService* ClientSideDetectionService::Create(
 122     net::URLRequestContextGetter* request_context_getter) {
 123   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 124   return new ClientSideDetectionService(request_context_getter);
 125 }
 126
 127 void ClientSideDetectionService::SetEnabledAndRefreshState(bool enabled) {
 128   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 129   SendModelToRenderers();  // always refresh the renderer state
 130   if (enabled == enabled_)
 131     return;
 132   enabled_ = enabled;
 133   if (enabled_) {
 134     // Refresh the model when the service is enabled.  This can happen when the
 135     // preference is toggled, or early during startup if the preference is
 136     // already enabled. In a lot of cases the model will be in the cache so it
 137     // won't actually be fetched from the network.
 138     // We delay the first model fetch to avoid slowing down browser startup.
 139     ScheduleFetchModel(kInitialClientModelFetchDelayMs);
 140   } else {
 141     // Cancel pending requests.
 142     model_fetcher_.reset();
 143     // Invoke pending callbacks with a false verdict.
 144     for (std::map<const net::URLFetcher*, ClientReportInfo*>::iterator it =
 145              client_phishing_reports_.begin();
 146          it != client_phishing_reports_.end(); ++it) {
 147       ClientReportInfo* info = it->second;
 148       if (!info->callback.is_null())
 149         info->callback.Run(info->phishing_url, false);
 150     }
 151     STLDeleteContainerPairPointers(client_phishing_reports_.begin(),
 152                                    client_phishing_reports_.end());
 153     client_phishing_reports_.clear();
 154     for (std::map<const net::URLFetcher*, ClientMalwareReportInfo*>::iterator it
 155              = client_malware_reports_.begin();
 156          it != client_malware_reports_.end(); ++it) {
 157       ClientMalwareReportInfo* info = it->second;
 158       if (!info->callback.is_null())
 159         info->callback.Run(info->original_url, info->original_url, false);
 160     }
 161     STLDeleteContainerPairPointers(client_malware_reports_.begin(),
 162                                    client_malware_reports_.end());
 163     client_malware_reports_.clear();
 164     cache_.clear();
 165   }
 166 }
 167
 168 void ClientSideDetectionService::SendClientReportPhishingRequest(
 169     ClientPhishingRequest* verdict,
 170     const ClientReportPhishingRequestCallback& callback) {
 171   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 172   base::MessageLoop::current()->PostTask(
 173       FROM_HERE,
 174       base::Bind(&ClientSideDetectionService::StartClientReportPhishingRequest,
 175                  weak_factory_.GetWeakPtr(), verdict, callback));
 176 }
 177
 178 void ClientSideDetectionService::SendClientReportMalwareRequest(
 179     ClientMalwareRequest* verdict,
 180     const ClientReportMalwareRequestCallback& callback) {
 181   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 182   base::MessageLoop::current()->PostTask(
 183       FROM_HERE,
 184       base::Bind(&ClientSideDetectionService::StartClientReportMalwareRequest,
 185                  weak_factory_.GetWeakPtr(), verdict, callback));
 186 }
 187
 188 bool ClientSideDetectionService::IsPrivateIPAddress(
 189     const std::string& ip_address) const {
 190   net::IPAddressNumber ip_number;
 191   if (!net::ParseIPLiteralToNumber(ip_address, &ip_number)) {
 192     VLOG(2) << "Unable to parse IP address: '" << ip_address << "'";
 193     // Err on the side of safety and assume this might be private.
 194     return true;
 195   }
 196
 197   return net::IsIPAddressReserved(ip_number);
 198 }
 199
 200 void ClientSideDetectionService::OnURLFetchComplete(
 201     const net::URLFetcher* source) {
 202   // TODO(vadimt): Remove ScopedProfile below once crbug.com/422577 is fixed.
 203   tracked_objects::ScopedProfile tracking_profile(
 204       FROM_HERE_WITH_EXPLICIT_FUNCTION(
 205           "422577 ClientSideDetectionService::OnURLFetchComplete"));
 206
 207   std::string data;
 208   source->GetResponseAsString(&data);
 209   if (source == model_fetcher_.get()) {
 210     HandleModelResponse(
 211         source, source->GetURL(), source->GetStatus(),
 212         source->GetResponseCode(), source->GetCookies(), data);
 213   } else if (client_phishing_reports_.find(source) !=
 214              client_phishing_reports_.end()) {
 215     HandlePhishingVerdict(
 216         source, source->GetURL(), source->GetStatus(),
 217         source->GetResponseCode(), source->GetCookies(), data);
 218   } else if (client_malware_reports_.find(source) !=
 219              client_malware_reports_.end()) {
 220     HandleMalwareVerdict(
 221         source, source->GetURL(), source->GetStatus(),
 222         source->GetResponseCode(), source->GetCookies(), data);
 223   } else {
 224     NOTREACHED();
 225   }
 226 }
 227
 228 void ClientSideDetectionService::Observe(
 229     int type,
 230     const content::NotificationSource& source,
 231     const content::NotificationDetails& details) {
 232   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 233   DCHECK(type == content::NOTIFICATION_RENDERER_PROCESS_CREATED);
 234   if (!model_.get()) {
 235     // Model might not be ready or maybe there was an error.
 236     return;
 237   }
 238   SendModelToProcess(
 239       content::Source<content::RenderProcessHost>(source).ptr());
 240 }
 241
 242 void ClientSideDetectionService::SendModelToProcess(
 243     content::RenderProcessHost* process) {
 244   // The ClientSideDetectionService is enabled if _any_ active profile has
 245   // SafeBrowsing turned on.  Here we check the profile for each renderer
 246   // process and only send the model to those that have SafeBrowsing enabled.
 247   Profile* profile = Profile::FromBrowserContext(process->GetBrowserContext());
 248   std::string model;
 249   if (profile->GetPrefs()->GetBoolean(prefs::kSafeBrowsingEnabled)) {
 250     VLOG(2) << "Sending phishing model to RenderProcessHost @" << process;
 251     model = model_str_;
 252   } else {
 253     VLOG(2) << "Disabling client-side phishing detection for "
 254             << "RenderProcessHost @" << process;
 255   }
 256   process->Send(new SafeBrowsingMsg_SetPhishingModel(model));
 257 }
 258
 259 void ClientSideDetectionService::SendModelToRenderers() {
 260   for (content::RenderProcessHost::iterator i(
 261           content::RenderProcessHost::AllHostsIterator());
 262        !i.IsAtEnd(); i.Advance()) {
 263     SendModelToProcess(i.GetCurrentValue());
 264   }
 265 }
 266
 267 void ClientSideDetectionService::ScheduleFetchModel(int64 delay_ms) {
 268   if (CommandLine::ForCurrentProcess()->HasSwitch(
 269       switches::kSbDisableAutoUpdate))
 270     return;
 271   base::MessageLoop::current()->PostDelayedTask(
 272       FROM_HERE,
 273       base::Bind(&ClientSideDetectionService::StartFetchModel,
 274                  weak_factory_.GetWeakPtr()),
 275       base::TimeDelta::FromMilliseconds(delay_ms));
 276 }
 277
 278 void ClientSideDetectionService::StartFetchModel() {
 279   if (enabled_) {
 280     // Start fetching the model either from the cache or possibly from the
 281     // network if the model isn't in the cache.
 282     model_fetcher_.reset(net::URLFetcher::Create(
 283         0 /* ID used for testing */, GURL(kClientModelUrl),
 284         net::URLFetcher::GET, this));
 285     model_fetcher_->SetRequestContext(request_context_getter_.get());
 286     model_fetcher_->Start();
 287   }
 288 }
 289
 290 void ClientSideDetectionService::EndFetchModel(ClientModelStatus status) {
 291   UMA_HISTOGRAM_ENUMERATION("SBClientPhishing.ClientModelStatus",
 292                             status,
 293                             MODEL_STATUS_MAX);
 294   if (status == MODEL_SUCCESS) {
 295     SetBadSubnets(*model_, &bad_subnets_);
 296     SendModelToRenderers();
 297   }
 298   int delay_ms = kClientModelFetchIntervalMs;
 299   // If the most recently fetched model had a valid max-age and the model was
 300   // valid we're scheduling the next model update for after the max-age expired.
 301   if (model_max_age_.get() &&
 302       (status == MODEL_SUCCESS || status == MODEL_NOT_CHANGED)) {
 303     // We're adding 60s of additional delay to make sure we're past
 304     // the model's age.
 305     *model_max_age_ += base::TimeDelta::FromMinutes(1);
 306     delay_ms = model_max_age_->InMilliseconds();
 307   }
 308   model_max_age_.reset();
 309
 310   // Schedule the next model reload.
 311   ScheduleFetchModel(delay_ms);
 312 }
 313
 314 void ClientSideDetectionService::StartClientReportPhishingRequest(
 315     ClientPhishingRequest* verdict,
 316     const ClientReportPhishingRequestCallback& callback) {
 317   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 318   scoped_ptr<ClientPhishingRequest> request(verdict);
 319
 320   if (!enabled_) {
 321     if (!callback.is_null())
 322       callback.Run(GURL(request->url()), false);
 323     return;
 324   }
 325
 326   std::string request_data;
 327   if (!request->SerializeToString(&request_data)) {
 328     UMA_HISTOGRAM_COUNTS("SBClientPhishing.RequestNotSerialized", 1);
 329     VLOG(1) << "Unable to serialize the CSD request. Proto file changed?";
 330     if (!callback.is_null())
 331       callback.Run(GURL(request->url()), false);
 332     return;
 333   }
 334
 335   net::URLFetcher* fetcher = net::URLFetcher::Create(
 336       0 /* ID used for testing */,
 337       GetClientReportUrl(kClientReportPhishingUrl),
 338       net::URLFetcher::POST, this);
 339
 340   // Remember which callback and URL correspond to the current fetcher object.
 341   ClientReportInfo* info = new ClientReportInfo;
 342   info->callback = callback;
 343   info->phishing_url = GURL(request->url());
 344   client_phishing_reports_[fetcher] = info;
 345
 346   fetcher->SetLoadFlags(net::LOAD_DISABLE_CACHE);
 347   fetcher->SetRequestContext(request_context_getter_.get());
 348   fetcher->SetUploadData("application/octet-stream", request_data);
 349   fetcher->Start();
 350
 351   // Record that we made a request
 352   phishing_report_times_.push(base::Time::Now());
 353 }
 354
 355 void ClientSideDetectionService::StartClientReportMalwareRequest(
 356     ClientMalwareRequest* verdict,
 357     const ClientReportMalwareRequestCallback& callback) {
 358   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 359   scoped_ptr<ClientMalwareRequest> request(verdict);
 360
 361   if (!enabled_) {
 362     if (!callback.is_null())
 363       callback.Run(GURL(request->url()), GURL(request->url()), false);
 364     return;
 365   }
 366
 367   std::string request_data;
 368   if (!request->SerializeToString(&request_data)) {
 369     UpdateEnumUMAHistogram(REPORT_FAILED_SERIALIZATION);
 370     DVLOG(1) << "Unable to serialize the CSD request. Proto file changed?";
 371     if (!callback.is_null())
 372       callback.Run(GURL(request->url()), GURL(request->url()), false);
 373     return;
 374   }
 375
 376   net::URLFetcher* fetcher = net::URLFetcher::Create(
 377       0 /* ID used for testing */,
 378       GetClientReportUrl(kClientReportMalwareUrl),
 379       net::URLFetcher::POST, this);
 380
 381   // Remember which callback and URL correspond to the current fetcher object.
 382   ClientMalwareReportInfo* info = new ClientMalwareReportInfo;
 383   info->callback = callback;
 384   info->original_url = GURL(request->url());
 385   client_malware_reports_[fetcher] = info;
 386
 387   fetcher->SetLoadFlags(net::LOAD_DISABLE_CACHE);
 388   fetcher->SetRequestContext(request_context_getter_.get());
 389   fetcher->SetUploadData("application/octet-stream", request_data);
 390   fetcher->Start();
 391
 392   UMA_HISTOGRAM_ENUMERATION("SBClientMalware.SentReports",
 393                             REPORT_SENT, REPORT_RESULT_MAX);
 394
 395   UMA_HISTOGRAM_COUNTS("SBClientMalware.IPBlacklistRequestPayloadSize",
 396                        request_data.size());
 397
 398   // Record that we made a malware request
 399   malware_report_times_.push(base::Time::Now());
 400 }
 401
 402 void ClientSideDetectionService::HandleModelResponse(
 403     const net::URLFetcher* source,
 404     const GURL& url,
 405     const net::URLRequestStatus& status,
 406     int response_code,
 407     const net::ResponseCookies& cookies,
 408     const std::string& data) {
 409   base::TimeDelta max_age;
 410   if (status.is_success() && net::HTTP_OK == response_code &&
 411       source->GetResponseHeaders() &&
 412       source->GetResponseHeaders()->GetMaxAgeValue(&max_age)) {
 413     model_max_age_.reset(new base::TimeDelta(max_age));
 414   }
 415   scoped_ptr<ClientSideModel> model(new ClientSideModel());
 416   ClientModelStatus model_status;
 417   if (!status.is_success() || net::HTTP_OK != response_code) {
 418     model_status = MODEL_FETCH_FAILED;
 419   } else if (data.empty()) {
 420     model_status = MODEL_EMPTY;
 421   } else if (data.size() > kMaxModelSizeBytes) {
 422     model_status = MODEL_TOO_LARGE;
 423   } else if (!model->ParseFromString(data)) {
 424     model_status = MODEL_PARSE_ERROR;
 425   } else if (!model->IsInitialized() || !model->has_version()) {
 426     model_status = MODEL_MISSING_FIELDS;
 427   } else if (!ModelHasValidHashIds(*model)) {
 428     model_status = MODEL_BAD_HASH_IDS;
 429   } else if (model->version() < 0 ||
 430              (model_.get() && model->version() < model_->version())) {
 431     model_status = MODEL_INVALID_VERSION_NUMBER;
 432   } else if (model_.get() && model->version() == model_->version()) {
 433     model_status = MODEL_NOT_CHANGED;
 434   } else {
 435     // The model is valid => replace the existing model with the new one.
 436     model_str_.assign(data);
 437     model_.swap(model);
 438     model_status = MODEL_SUCCESS;
 439   }
 440   EndFetchModel(model_status);
 441 }
 442
 443 void ClientSideDetectionService::HandlePhishingVerdict(
 444     const net::URLFetcher* source,
 445     const GURL& url,
 446     const net::URLRequestStatus& status,
 447     int response_code,
 448     const net::ResponseCookies& cookies,
 449     const std::string& data) {
 450   ClientPhishingResponse response;
 451   scoped_ptr<ClientReportInfo> info(client_phishing_reports_[source]);
 452   bool is_phishing = false;
 453   if (status.is_success() && net::HTTP_OK == response_code &&
 454       response.ParseFromString(data)) {
 455     // Cache response, possibly flushing an old one.
 456     cache_[info->phishing_url] =
 457         make_linked_ptr(new CacheState(response.phishy(), base::Time::Now()));
 458     is_phishing = response.phishy();
 459   } else {
 460     DLOG(ERROR) << "Unable to get the server verdict for URL: "
 461                 << info->phishing_url << " status: " << status.status() << " "
 462                 << "response_code:" << response_code;
 463   }
 464   if (!info->callback.is_null())
 465     info->callback.Run(info->phishing_url, is_phishing);
 466   client_phishing_reports_.erase(source);
 467   delete source;
 468 }
 469
 470 void ClientSideDetectionService::HandleMalwareVerdict(
 471     const net::URLFetcher* source,
 472     const GURL& url,
 473     const net::URLRequestStatus& status,
 474     int response_code,
 475     const net::ResponseCookies& cookies,
 476     const std::string& data) {
 477   if (status.is_success()) {
 478     UMA_HISTOGRAM_SPARSE_SLOWLY(
 479         "SBClientMalware.IPBlacklistRequestResponseCode", response_code);
 480   }
 481   // status error is negative, so we put - in front of it.
 482   UMA_HISTOGRAM_SPARSE_SLOWLY(
 483       "SBClientMalware.IPBlacklistRequestNetError", -status.error());
 484
 485   ClientMalwareResponse response;
 486   scoped_ptr<ClientMalwareReportInfo> info(client_malware_reports_[source]);
 487   bool should_blacklist = false;
 488   if (status.is_success() && net::HTTP_OK == response_code &&
 489       response.ParseFromString(data)) {
 490     should_blacklist = response.blacklist();
 491   } else {
 492     DLOG(ERROR) << "Unable to get the server verdict for URL: "
 493                 << info->original_url << " status: " << status.status() << " "
 494                 << "response_code:" << response_code;
 495   }
 496
 497   if (!info->callback.is_null()) {
 498     if (response.has_bad_url())
 499       info->callback.Run(info->original_url, GURL(response.bad_url()),
 500                          should_blacklist);
 501     else
 502       info->callback.Run(info->original_url, info->original_url, false);
 503   }
 504
 505   client_malware_reports_.erase(source);
 506   delete source;
 507 }
 508
 509 bool ClientSideDetectionService::IsInCache(const GURL& url) {
 510   UpdateCache();
 511
 512   return cache_.find(url) != cache_.end();
 513 }
 514
 515 bool ClientSideDetectionService::GetValidCachedResult(const GURL& url,
 516                                                       bool* is_phishing) {
 517   UpdateCache();
 518
 519   PhishingCache::iterator it = cache_.find(url);
 520   if (it == cache_.end()) {
 521     return false;
 522   }
 523
 524   // We still need to check if the result is valid.
 525   const CacheState& cache_state = *it->second;
 526   if (cache_state.is_phishing ?
 527       cache_state.timestamp > base::Time::Now() -
 528           base::TimeDelta::FromMinutes(kPositiveCacheIntervalMinutes) :
 529       cache_state.timestamp > base::Time::Now() -
 530           base::TimeDelta::FromDays(kNegativeCacheIntervalDays)) {
 531     *is_phishing = cache_state.is_phishing;
 532     return true;
 533   }
 534   return false;
 535 }
 536
 537 void ClientSideDetectionService::UpdateCache() {
 538   // Since we limit the number of requests but allow pass-through for cache
 539   // refreshes, we don't want to remove elements from the cache if they
 540   // could be used for this purpose even if we will not use the entry to
 541   // satisfy the request from the cache.
 542   base::TimeDelta positive_cache_interval =
 543       std::max(base::TimeDelta::FromMinutes(kPositiveCacheIntervalMinutes),
 544                base::TimeDelta::FromDays(kReportsIntervalDays));
 545   base::TimeDelta negative_cache_interval =
 546       std::max(base::TimeDelta::FromDays(kNegativeCacheIntervalDays),
 547                base::TimeDelta::FromDays(kReportsIntervalDays));
 548
 549   // Remove elements from the cache that will no longer be used.
 550   for (PhishingCache::iterator it = cache_.begin(); it != cache_.end();) {
 551     const CacheState& cache_state = *it->second;
 552     if (cache_state.is_phishing ?
 553         cache_state.timestamp > base::Time::Now() - positive_cache_interval :
 554         cache_state.timestamp > base::Time::Now() - negative_cache_interval) {
 555       ++it;
 556     } else {
 557       cache_.erase(it++);
 558     }
 559   }
 560 }
 561
 562 bool ClientSideDetectionService::OverMalwareReportLimit() {
 563   return GetMalwareNumReports() > kMaxReportsPerInterval;
 564 }
 565
 566 bool ClientSideDetectionService::OverPhishingReportLimit() {
 567   return GetPhishingNumReports() > kMaxReportsPerInterval;
 568 }
 569
 570 int ClientSideDetectionService::GetMalwareNumReports() {
 571   return GetNumReports(&malware_report_times_);
 572 }
 573
 574 int ClientSideDetectionService::GetPhishingNumReports() {
 575   return GetNumReports(&phishing_report_times_);
 576 }
 577
 578 int ClientSideDetectionService::GetNumReports(
 579     std::queue<base::Time>* report_times) {
 580   base::Time cutoff =
 581       base::Time::Now() - base::TimeDelta::FromDays(kReportsIntervalDays);
 582
 583   // Erase items older than cutoff because we will never care about them again.
 584   while (!report_times->empty() &&
 585          report_times->front() < cutoff) {
 586     report_times->pop();
 587   }
 588
 589   // Return the number of elements that are above the cutoff.
 590   return report_times->size();
 591 }
 592
 593 // static
 594 void ClientSideDetectionService::SetBadSubnets(const ClientSideModel& model,
 595                                                BadSubnetMap* bad_subnets) {
 596   bad_subnets->clear();
 597   for (int i = 0; i < model.bad_subnet_size(); ++i) {
 598     int size = model.bad_subnet(i).size();
 599     if (size < 0 || size > static_cast<int>(net::kIPv6AddressSize) * 8) {
 600       DLOG(ERROR) << "Invalid bad subnet size: " << size;
 601       continue;
 602     }
 603     if (model.bad_subnet(i).prefix().size() != crypto::kSHA256Length) {
 604       DLOG(ERROR) << "Invalid bad subnet prefix length: "
 605                   << model.bad_subnet(i).prefix().size();
 606       continue;
 607     }
 608     // We precompute the mask for the given subnet size to speed up lookups.
 609     // Basically we need to create a 16B long string which has the highest
 610     // |size| bits sets to one.
 611     std::string mask(net::kIPv6AddressSize, '\x00');
 612     mask.replace(0, size / 8, size / 8, '\xFF');
 613     if (size % 8) {
 614       mask[size / 8] = 0xFF << (8 - (size % 8));
 615     }
 616     (*bad_subnets)[mask].insert(model.bad_subnet(i).prefix());
 617   }
 618 }
 619
 620 // static
 621 bool ClientSideDetectionService::ModelHasValidHashIds(
 622     const ClientSideModel& model) {
 623   const int max_index = model.hashes_size() - 1;
 624   for (int i = 0; i < model.rule_size(); ++i) {
 625     for (int j = 0; j < model.rule(i).feature_size(); ++j) {
 626       if (model.rule(i).feature(j) < 0 ||
 627           model.rule(i).feature(j) > max_index) {
 628         return false;
 629       }
 630     }
 631   }
 632   for (int i = 0; i < model.page_term_size(); ++i) {
 633     if (model.page_term(i) < 0 || model.page_term(i) > max_index) {
 634       return false;
 635     }
 636   }
 637   return true;
 638 }
 639
 640 // static
 641 GURL ClientSideDetectionService::GetClientReportUrl(
 642     const std::string& report_url) {
 643   GURL url(report_url);
 644   std::string api_key = google_apis::GetAPIKey();
 645   if (!api_key.empty())
 646     url = url.Resolve("?key=" + net::EscapeQueryParamValue(api_key, true));
 647
 648   return url;
 649 }
 650 }  // namespace safe_browsing