chrome/browser/safe_browsing/client_side_detection_service_unittest.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include <map>
   6 #include <queue>
   7 #include <string>
   8
   9 #include "base/bind.h"
  10 #include "base/callback.h"
  11 #include "base/logging.h"
  12 #include "base/memory/scoped_ptr.h"
  13 #include "base/message_loop/message_loop.h"
  14 #include "base/metrics/field_trial.h"
  15 #include "base/strings/string_number_conversions.h"
  16 #include "base/time/time.h"
  17 #include "chrome/browser/safe_browsing/client_side_detection_service.h"
  18 #include "chrome/common/safe_browsing/client_model.pb.h"
  19 #include "chrome/common/safe_browsing/csd.pb.h"
  20 #include "components/variations/variations_associated_data.h"
  21 #include "content/public/test/test_browser_thread.h"
  22 #include "crypto/sha2.h"
  23 #include "net/http/http_status_code.h"
  24 #include "net/url_request/test_url_fetcher_factory.h"
  25 #include "net/url_request/url_request_status.h"
  26 #include "testing/gmock/include/gmock/gmock.h"
  27 #include "testing/gtest/include/gtest/gtest.h"
  28 #include "url/gurl.h"
  29
  30 using ::testing::Invoke;
  31 using ::testing::Mock;
  32 using ::testing::StrictMock;
  33 using ::testing::_;
  34 using content::BrowserThread;
  35
  36 namespace safe_browsing {
  37 namespace {
  38
  39 class MockModelLoader : public ModelLoader {
  40  public:
  41   explicit MockModelLoader(const std::string model_name)
  42       : ModelLoader(base::Closure(), model_name) {}
  43   ~MockModelLoader() override {}
  44
  45   MOCK_METHOD1(ScheduleFetch, void(int64));
  46   MOCK_METHOD0(CancelFetcher, void());
  47
  48  private:
  49   DISALLOW_COPY_AND_ASSIGN(MockModelLoader);
  50 };
  51
  52 class MockClientSideDetectionService : public ClientSideDetectionService {
  53  public:
  54   MockClientSideDetectionService() : ClientSideDetectionService(NULL) {}
  55
  56   ~MockClientSideDetectionService() override {}
  57
  58  private:
  59   DISALLOW_COPY_AND_ASSIGN(MockClientSideDetectionService);
  60 };
  61
  62 }  // namespace
  63
  64 class ClientSideDetectionServiceTest : public testing::Test {
  65  protected:
  66   void SetUp() override {
  67     file_thread_.reset(new content::TestBrowserThread(BrowserThread::FILE,
  68                                                       &msg_loop_));
  69     factory_.reset(new net::FakeURLFetcherFactory(NULL));
  70     browser_thread_.reset(new content::TestBrowserThread(BrowserThread::UI,
  71                                                          &msg_loop_));
  72   }
  73
  74   void TearDown() override {
  75     msg_loop_.RunUntilIdle();
  76     csd_service_.reset();
  77     file_thread_.reset();
  78     browser_thread_.reset();
  79   }
  80
  81   bool SendClientReportPhishingRequest(const GURL& phishing_url,
  82                                        float score) {
  83     ClientPhishingRequest* request = new ClientPhishingRequest();
  84     request->set_url(phishing_url.spec());
  85     request->set_client_score(score);
  86     request->set_is_phishing(true);  // client thinks the URL is phishing.
  87     csd_service_->SendClientReportPhishingRequest(
  88         request,
  89         false,
  90         base::Bind(&ClientSideDetectionServiceTest::SendRequestDone,
  91                    base::Unretained(this)));
  92     phishing_url_ = phishing_url;
  93     msg_loop_.Run();  // Waits until callback is called.
  94     return is_phishing_;
  95   }
  96
  97   bool SendClientReportMalwareRequest(const GURL& url) {
  98     scoped_ptr<ClientMalwareRequest> request(new ClientMalwareRequest());
  99     request->set_url(url.spec());
 100     csd_service_->SendClientReportMalwareRequest(
 101         request.release(),
 102         base::Bind(&ClientSideDetectionServiceTest::SendMalwareRequestDone,
 103                    base::Unretained(this)));
 104     phishing_url_ = url;
 105     msg_loop_.Run();  // Waits until callback is called.
 106     return is_malware_;
 107   }
 108
 109   void SetModelFetchResponses() {
 110     // Set reponses for both models.
 111     factory_->SetFakeResponse(GURL(ModelLoader::kClientModelUrlPrefix +
 112                                    ModelLoader::FillInModelName(false, 0)),
 113                               "bogusmodel", net::HTTP_OK,
 114                               net::URLRequestStatus::SUCCESS);
 115     factory_->SetFakeResponse(GURL(ModelLoader::kClientModelUrlPrefix +
 116                                    ModelLoader::FillInModelName(true, 0)),
 117                               "bogusmodel", net::HTTP_OK,
 118                               net::URLRequestStatus::SUCCESS);
 119   }
 120
 121   void SetClientReportPhishingResponse(std::string response_data,
 122                                        net::HttpStatusCode response_code,
 123                                        net::URLRequestStatus::Status status) {
 124     factory_->SetFakeResponse(
 125         ClientSideDetectionService::GetClientReportUrl(
 126             ClientSideDetectionService::kClientReportPhishingUrl),
 127         response_data, response_code, status);
 128   }
 129
 130   void SetClientReportMalwareResponse(std::string response_data,
 131                                       net::HttpStatusCode response_code,
 132                                       net::URLRequestStatus::Status status) {
 133     factory_->SetFakeResponse(
 134         ClientSideDetectionService::GetClientReportUrl(
 135             ClientSideDetectionService::kClientReportMalwareUrl),
 136         response_data, response_code, status);
 137   }
 138
 139   int GetNumReports(std::queue<base::Time>* report_times) {
 140     return csd_service_->GetNumReports(report_times);
 141   }
 142
 143   std::queue<base::Time>& GetPhishingReportTimes() {
 144     return csd_service_->phishing_report_times_;
 145   }
 146
 147   std::queue<base::Time>& GetMalwareReportTimes() {
 148     return csd_service_->malware_report_times_;
 149   }
 150
 151   void SetCache(const GURL& gurl, bool is_phishing, base::Time time) {
 152     csd_service_->cache_[gurl] =
 153         make_linked_ptr(new ClientSideDetectionService::CacheState(is_phishing,
 154                                                                    time));
 155   }
 156
 157   void TestCache() {
 158     ClientSideDetectionService::PhishingCache& cache = csd_service_->cache_;
 159     base::Time now = base::Time::Now();
 160     base::Time time =
 161         now - base::TimeDelta::FromDays(
 162             ClientSideDetectionService::kNegativeCacheIntervalDays) +
 163         base::TimeDelta::FromMinutes(5);
 164     cache[GURL("http://first.url.com/")] =
 165         make_linked_ptr(new ClientSideDetectionService::CacheState(false,
 166                                                                    time));
 167
 168     time =
 169         now - base::TimeDelta::FromDays(
 170             ClientSideDetectionService::kNegativeCacheIntervalDays) -
 171         base::TimeDelta::FromHours(1);
 172     cache[GURL("http://second.url.com/")] =
 173         make_linked_ptr(new ClientSideDetectionService::CacheState(false,
 174                                                                    time));
 175
 176     time =
 177         now - base::TimeDelta::FromMinutes(
 178             ClientSideDetectionService::kPositiveCacheIntervalMinutes) -
 179         base::TimeDelta::FromMinutes(5);
 180     cache[GURL("http://third.url.com/")] =
 181         make_linked_ptr(new ClientSideDetectionService::CacheState(true, time));
 182
 183     time =
 184         now - base::TimeDelta::FromMinutes(
 185             ClientSideDetectionService::kPositiveCacheIntervalMinutes) +
 186         base::TimeDelta::FromMinutes(5);
 187     cache[GURL("http://fourth.url.com/")] =
 188         make_linked_ptr(new ClientSideDetectionService::CacheState(true, time));
 189
 190     csd_service_->UpdateCache();
 191
 192     // 3 elements should be in the cache, the first, third, and fourth.
 193     EXPECT_EQ(3U, cache.size());
 194     EXPECT_TRUE(cache.find(GURL("http://first.url.com/")) != cache.end());
 195     EXPECT_TRUE(cache.find(GURL("http://third.url.com/")) != cache.end());
 196     EXPECT_TRUE(cache.find(GURL("http://fourth.url.com/")) != cache.end());
 197
 198     // While 3 elements remain, only the first and the fourth are actually
 199     // valid.
 200     bool is_phishing;
 201     EXPECT_TRUE(csd_service_->GetValidCachedResult(
 202         GURL("http://first.url.com"), &is_phishing));
 203     EXPECT_FALSE(is_phishing);
 204     EXPECT_FALSE(csd_service_->GetValidCachedResult(
 205         GURL("http://third.url.com"), &is_phishing));
 206     EXPECT_TRUE(csd_service_->GetValidCachedResult(
 207         GURL("http://fourth.url.com"), &is_phishing));
 208     EXPECT_TRUE(is_phishing);
 209   }
 210
 211   void AddFeature(const std::string& name, double value,
 212                   ClientPhishingRequest* request) {
 213     ClientPhishingRequest_Feature* feature = request->add_feature_map();
 214     feature->set_name(name);
 215     feature->set_value(value);
 216   }
 217
 218   void AddNonModelFeature(const std::string& name, double value,
 219                           ClientPhishingRequest* request) {
 220     ClientPhishingRequest_Feature* feature =
 221         request->add_non_model_feature_map();
 222     feature->set_name(name);
 223     feature->set_value(value);
 224   }
 225
 226   void CheckConfirmedMalwareUrl(GURL url) {
 227     ASSERT_EQ(confirmed_malware_url_, url);
 228   }
 229
 230  protected:
 231   scoped_ptr<ClientSideDetectionService> csd_service_;
 232   scoped_ptr<net::FakeURLFetcherFactory> factory_;
 233   base::MessageLoop msg_loop_;
 234
 235  private:
 236   void SendRequestDone(GURL phishing_url, bool is_phishing) {
 237     ASSERT_EQ(phishing_url, phishing_url_);
 238     is_phishing_ = is_phishing;
 239     msg_loop_.Quit();
 240   }
 241
 242   void SendMalwareRequestDone(GURL original_url, GURL malware_url,
 243                               bool is_malware) {
 244     ASSERT_EQ(phishing_url_, original_url);
 245     confirmed_malware_url_ = malware_url;
 246     is_malware_ = is_malware;
 247     msg_loop_.Quit();
 248   }
 249
 250   scoped_ptr<content::TestBrowserThread> browser_thread_;
 251   scoped_ptr<content::TestBrowserThread> file_thread_;
 252   scoped_ptr<base::FieldTrialList> field_trials_;
 253
 254   GURL phishing_url_;
 255   GURL confirmed_malware_url_;
 256   bool is_phishing_;
 257   bool is_malware_;
 258 };
 259
 260
 261 TEST_F(ClientSideDetectionServiceTest, ServiceObjectDeletedBeforeCallbackDone) {
 262   SetModelFetchResponses();
 263   csd_service_.reset(ClientSideDetectionService::Create(NULL));
 264   csd_service_->SetEnabledAndRefreshState(true);
 265   EXPECT_TRUE(csd_service_.get() != NULL);
 266   // We delete the client-side detection service class even though the callbacks
 267   // haven't run yet.
 268   csd_service_.reset();
 269   // Waiting for the callbacks to run should not crash even if the service
 270   // object is gone.
 271   msg_loop_.RunUntilIdle();
 272 }
 273
 274 TEST_F(ClientSideDetectionServiceTest, SendClientReportPhishingRequest) {
 275   SetModelFetchResponses();
 276   csd_service_.reset(ClientSideDetectionService::Create(NULL));
 277   csd_service_->SetEnabledAndRefreshState(true);
 278
 279   GURL url("http://a.com/");
 280   float score = 0.4f;  // Some random client score.
 281
 282   base::Time before = base::Time::Now();
 283
 284   // Invalid response body from the server.
 285   SetClientReportPhishingResponse("invalid proto response", net::HTTP_OK,
 286                                   net::URLRequestStatus::SUCCESS);
 287   EXPECT_FALSE(SendClientReportPhishingRequest(url, score));
 288
 289   // Normal behavior.
 290   ClientPhishingResponse response;
 291   response.set_phishy(true);
 292   SetClientReportPhishingResponse(response.SerializeAsString(), net::HTTP_OK,
 293                                   net::URLRequestStatus::SUCCESS);
 294   EXPECT_TRUE(SendClientReportPhishingRequest(url, score));
 295
 296   // This request will fail
 297   GURL second_url("http://b.com/");
 298   response.set_phishy(false);
 299   SetClientReportPhishingResponse(response.SerializeAsString(),
 300                                   net::HTTP_INTERNAL_SERVER_ERROR,
 301                                   net::URLRequestStatus::FAILED);
 302   EXPECT_FALSE(SendClientReportPhishingRequest(second_url, score));
 303
 304   base::Time after = base::Time::Now();
 305
 306   // Check that we have recorded all 3 requests within the correct time range.
 307   std::queue<base::Time>& report_times = GetPhishingReportTimes();
 308   EXPECT_EQ(3U, report_times.size());
 309   while (!report_times.empty()) {
 310     base::Time time = report_times.back();
 311     report_times.pop();
 312     EXPECT_LE(before, time);
 313     EXPECT_GE(after, time);
 314   }
 315
 316   // Only the first url should be in the cache.
 317   bool is_phishing;
 318   EXPECT_TRUE(csd_service_->IsInCache(url));
 319   EXPECT_TRUE(csd_service_->GetValidCachedResult(url, &is_phishing));
 320   EXPECT_TRUE(is_phishing);
 321   EXPECT_FALSE(csd_service_->IsInCache(second_url));
 322 }
 323
 324 TEST_F(ClientSideDetectionServiceTest, SendClientReportMalwareRequest) {
 325   SetModelFetchResponses();
 326   csd_service_.reset(ClientSideDetectionService::Create(NULL));
 327   csd_service_->SetEnabledAndRefreshState(true);
 328   GURL url("http://a.com/");
 329
 330   base::Time before = base::Time::Now();
 331   // Invalid response body from the server.
 332   SetClientReportMalwareResponse("invalid proto response", net::HTTP_OK,
 333                                  net::URLRequestStatus::SUCCESS);
 334   EXPECT_FALSE(SendClientReportMalwareRequest(url));
 335
 336   // Missing bad_url.
 337   ClientMalwareResponse response;
 338   response.set_blacklist(true);
 339   SetClientReportMalwareResponse(response.SerializeAsString(), net::HTTP_OK,
 340                                  net::URLRequestStatus::SUCCESS);
 341   EXPECT_FALSE(SendClientReportMalwareRequest(url));
 342
 343   // Normal behavior.
 344   response.set_blacklist(true);
 345   response.set_bad_url("http://response-bad.com/");
 346   SetClientReportMalwareResponse(response.SerializeAsString(), net::HTTP_OK,
 347                                  net::URLRequestStatus::SUCCESS);
 348   EXPECT_TRUE(SendClientReportMalwareRequest(url));
 349   CheckConfirmedMalwareUrl(GURL("http://response-bad.com/"));
 350
 351   // This request will fail
 352   response.set_blacklist(false);
 353   SetClientReportMalwareResponse(response.SerializeAsString(),
 354                                  net::HTTP_INTERNAL_SERVER_ERROR,
 355                                  net::URLRequestStatus::FAILED);
 356   EXPECT_FALSE(SendClientReportMalwareRequest(url));
 357
 358   // Server blacklist decision is false, and response is successful
 359   response.set_blacklist(false);
 360   SetClientReportMalwareResponse(response.SerializeAsString(), net::HTTP_OK,
 361                                  net::URLRequestStatus::SUCCESS);
 362   EXPECT_FALSE(SendClientReportMalwareRequest(url));
 363
 364   // Check that we have recorded all 5 requests within the correct time range.
 365   base::Time after = base::Time::Now();
 366   std::queue<base::Time>& report_times = GetMalwareReportTimes();
 367   EXPECT_EQ(5U, report_times.size());
 368
 369   // Check that the malware report limit was reached.
 370   EXPECT_TRUE(csd_service_->OverMalwareReportLimit());
 371
 372   report_times = GetMalwareReportTimes();
 373   EXPECT_EQ(5U, report_times.size());
 374   while (!report_times.empty()) {
 375     base::Time time = report_times.back();
 376     report_times.pop();
 377     EXPECT_LE(before, time);
 378     EXPECT_GE(after, time);
 379   }
 380 }
 381
 382 TEST_F(ClientSideDetectionServiceTest, GetNumReportTest) {
 383   SetModelFetchResponses();
 384   csd_service_.reset(ClientSideDetectionService::Create(NULL));
 385
 386   std::queue<base::Time>& report_times = GetPhishingReportTimes();
 387   base::Time now = base::Time::Now();
 388   base::TimeDelta twenty_five_hours = base::TimeDelta::FromHours(25);
 389   report_times.push(now - twenty_five_hours);
 390   report_times.push(now - twenty_five_hours);
 391   report_times.push(now);
 392   report_times.push(now);
 393
 394   EXPECT_EQ(2, GetNumReports(&report_times));
 395 }
 396
 397 TEST_F(ClientSideDetectionServiceTest, CacheTest) {
 398   SetModelFetchResponses();
 399   csd_service_.reset(ClientSideDetectionService::Create(NULL));
 400
 401   TestCache();
 402 }
 403
 404 TEST_F(ClientSideDetectionServiceTest, IsPrivateIPAddress) {
 405   SetModelFetchResponses();
 406   csd_service_.reset(ClientSideDetectionService::Create(NULL));
 407
 408   EXPECT_TRUE(csd_service_->IsPrivateIPAddress("10.1.2.3"));
 409   EXPECT_TRUE(csd_service_->IsPrivateIPAddress("127.0.0.1"));
 410   EXPECT_TRUE(csd_service_->IsPrivateIPAddress("172.24.3.4"));
 411   EXPECT_TRUE(csd_service_->IsPrivateIPAddress("192.168.1.1"));
 412   EXPECT_TRUE(csd_service_->IsPrivateIPAddress("fc00::"));
 413   EXPECT_TRUE(csd_service_->IsPrivateIPAddress("fec0::"));
 414   EXPECT_TRUE(csd_service_->IsPrivateIPAddress("fec0:1:2::3"));
 415   EXPECT_TRUE(csd_service_->IsPrivateIPAddress("::1"));
 416
 417   EXPECT_FALSE(csd_service_->IsPrivateIPAddress("1.2.3.4"));
 418   EXPECT_FALSE(csd_service_->IsPrivateIPAddress("200.1.1.1"));
 419   EXPECT_FALSE(csd_service_->IsPrivateIPAddress("2001:0db8:ac10:fe01::"));
 420
 421   // If the address can't be parsed, the default is true.
 422   EXPECT_TRUE(csd_service_->IsPrivateIPAddress("blah"));
 423 }
 424
 425 TEST_F(ClientSideDetectionServiceTest, SetEnabledAndRefreshState) {
 426   // Check that the model isn't downloaded until the service is enabled.
 427   csd_service_.reset(ClientSideDetectionService::Create(NULL));
 428   EXPECT_FALSE(csd_service_->enabled());
 429   EXPECT_TRUE(csd_service_->model_loader_standard_->fetcher_.get() == NULL);
 430
 431   // Use a MockClientSideDetectionService for the rest of the test, to avoid
 432   // the scheduling delay.
 433   MockClientSideDetectionService* service =
 434       new StrictMock<MockClientSideDetectionService>();
 435   // Inject mock loaders.
 436   MockModelLoader* loader_1 = new StrictMock<MockModelLoader>("model1");
 437   MockModelLoader* loader_2 = new StrictMock<MockModelLoader>("model2");
 438   service->model_loader_standard_.reset(loader_1);
 439   service->model_loader_extended_.reset(loader_2);
 440   csd_service_.reset(service);
 441
 442   EXPECT_FALSE(csd_service_->enabled());
 443   // No calls expected yet.
 444   Mock::VerifyAndClearExpectations(service);
 445   Mock::VerifyAndClearExpectations(loader_1);
 446   Mock::VerifyAndClearExpectations(loader_2);
 447
 448   // Check that initial ScheduleFetch() calls are made.
 449   EXPECT_CALL(*loader_1,
 450               ScheduleFetch(
 451                   ClientSideDetectionService::kInitialClientModelFetchDelayMs));
 452   EXPECT_CALL(*loader_2,
 453               ScheduleFetch(
 454                   ClientSideDetectionService::kInitialClientModelFetchDelayMs));
 455   csd_service_->SetEnabledAndRefreshState(true);
 456   msg_loop_.RunUntilIdle();
 457   Mock::VerifyAndClearExpectations(service);
 458   Mock::VerifyAndClearExpectations(loader_1);
 459   Mock::VerifyAndClearExpectations(loader_2);
 460
 461   // Check that enabling again doesn't request the model.
 462   csd_service_->SetEnabledAndRefreshState(true);
 463   // No calls expected.
 464   msg_loop_.RunUntilIdle();
 465   Mock::VerifyAndClearExpectations(service);
 466   Mock::VerifyAndClearExpectations(loader_1);
 467   Mock::VerifyAndClearExpectations(loader_2);
 468
 469   // Check that disabling the service cancels pending requests.
 470   EXPECT_CALL(*loader_1, CancelFetcher());
 471   EXPECT_CALL(*loader_2, CancelFetcher());
 472   csd_service_->SetEnabledAndRefreshState(false);
 473   msg_loop_.RunUntilIdle();
 474   Mock::VerifyAndClearExpectations(service);
 475   Mock::VerifyAndClearExpectations(loader_1);
 476   Mock::VerifyAndClearExpectations(loader_2);
 477
 478   // Check that disabling again doesn't request the model.
 479   csd_service_->SetEnabledAndRefreshState(false);
 480   // No calls expected.
 481   msg_loop_.RunUntilIdle();
 482   Mock::VerifyAndClearExpectations(service);
 483   Mock::VerifyAndClearExpectations(loader_1);
 484   Mock::VerifyAndClearExpectations(loader_2);
 485 }
 486 }  // namespace safe_browsing