chrome/browser/safe_browsing/browser_feature_extractor_unittest.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "chrome/browser/safe_browsing/browser_feature_extractor.h"
   6
   7 #include <map>
   8 #include <string>
   9 #include <vector>
  10
  11 #include "base/memory/scoped_ptr.h"
  12 #include "base/message_loop/message_loop.h"
  13 #include "base/strings/stringprintf.h"
  14 #include "base/time/time.h"
  15 #include "chrome/browser/history/history_service_factory.h"
  16 #include "chrome/browser/profiles/profile.h"
  17 #include "chrome/browser/safe_browsing/browser_features.h"
  18 #include "chrome/browser/safe_browsing/client_side_detection_host.h"
  19 #include "chrome/browser/safe_browsing/database_manager.h"
  20 #include "chrome/browser/safe_browsing/safe_browsing_service.h"
  21 #include "chrome/browser/safe_browsing/test_database_manager.h"
  22 #include "chrome/browser/safe_browsing/ui_manager.h"
  23 #include "chrome/common/safe_browsing/csd.pb.h"
  24 #include "chrome/test/base/chrome_render_view_host_test_harness.h"
  25 #include "chrome/test/base/testing_profile.h"
  26 #include "components/history/core/browser/history_backend.h"
  27 #include "components/history/core/browser/history_service.h"
  28 #include "content/public/browser/navigation_controller.h"
  29 #include "content/public/browser/navigation_entry.h"
  30 #include "content/public/browser/web_contents.h"
  31 #include "content/public/common/referrer.h"
  32 #include "content/public/test/test_browser_thread.h"
  33 #include "content/public/test/web_contents_tester.h"
  34 #include "testing/gmock/include/gmock/gmock.h"
  35 #include "testing/gtest/include/gtest/gtest.h"
  36 #include "ui/base/page_transition_types.h"
  37 #include "url/gurl.h"
  38
  39 using content::BrowserThread;
  40 using content::ResourceType;
  41 using content::WebContentsTester;
  42
  43 using testing::DoAll;
  44 using testing::Return;
  45 using testing::StrictMock;
  46
  47 namespace safe_browsing {
  48
  49 namespace {
  50
  51 class MockSafeBrowsingDatabaseManager : public TestSafeBrowsingDatabaseManager {
  52  public:
  53   MockSafeBrowsingDatabaseManager() {}
  54
  55   MOCK_METHOD1(MatchMalwareIP, bool(const std::string& ip_address));
  56
  57  protected:
  58   virtual ~MockSafeBrowsingDatabaseManager() {}
  59
  60  private:
  61   DISALLOW_COPY_AND_ASSIGN(MockSafeBrowsingDatabaseManager);
  62 };
  63
  64 class MockClientSideDetectionHost : public ClientSideDetectionHost {
  65  public:
  66   MockClientSideDetectionHost(
  67       content::WebContents* tab,
  68       SafeBrowsingDatabaseManager* database_manager)
  69       : ClientSideDetectionHost(tab) {
  70     set_safe_browsing_managers(NULL, database_manager);
  71   }
  72
  73   virtual ~MockClientSideDetectionHost() {}
  74
  75   MOCK_METHOD1(IsBadIpAddress, bool(const std::string&));
  76 };
  77 }  // namespace
  78
  79 class BrowserFeatureExtractorTest : public ChromeRenderViewHostTestHarness {
  80  protected:
  81   void SetUp() override {
  82     ChromeRenderViewHostTestHarness::SetUp();
  83     ASSERT_TRUE(profile()->CreateHistoryService(
  84         true /* delete_file */, false /* no_db */));
  85
  86     db_manager_ = new StrictMock<MockSafeBrowsingDatabaseManager>();
  87     host_.reset(new StrictMock<MockClientSideDetectionHost>(
  88         web_contents(), db_manager_.get()));
  89     extractor_.reset(
  90         new BrowserFeatureExtractor(web_contents(), host_.get()));
  91     num_pending_ = 0;
  92     browse_info_.reset(new BrowseInfo);
  93   }
  94
  95   void TearDown() override {
  96     extractor_.reset();
  97     host_.reset();
  98     db_manager_ = NULL;
  99     profile()->DestroyHistoryService();
 100     ChromeRenderViewHostTestHarness::TearDown();
 101     ASSERT_EQ(0, num_pending_);
 102   }
 103
 104   history::HistoryService* history_service() {
 105     return HistoryServiceFactory::GetForProfile(
 106         profile(), ServiceAccessType::EXPLICIT_ACCESS);
 107   }
 108
 109   void SetRedirectChain(const std::vector<GURL>& redirect_chain,
 110                         bool new_host) {
 111     browse_info_->url_redirects = redirect_chain;
 112     if (new_host) {
 113       browse_info_->host_redirects = redirect_chain;
 114     }
 115   }
 116
 117   // Wrapper around NavigateAndCommit that also sets the redirect chain to
 118   // a sane value.
 119   void SimpleNavigateAndCommit(const GURL& url) {
 120     std::vector<GURL> redirect_chain;
 121     redirect_chain.push_back(url);
 122     SetRedirectChain(redirect_chain, true);
 123     NavigateAndCommit(url, GURL(), ui::PAGE_TRANSITION_LINK);
 124   }
 125
 126   // This is similar to NavigateAndCommit that is in WebContentsTester, but
 127   // allows us to specify the referrer and page_transition_type.
 128   void NavigateAndCommit(const GURL& url,
 129                          const GURL& referrer,
 130                          ui::PageTransition type) {
 131     web_contents()->GetController().LoadURL(
 132         url, content::Referrer(referrer, blink::WebReferrerPolicyDefault),
 133         type, std::string());
 134     int pending_id =
 135         web_contents()->GetController().GetPendingEntry()->GetUniqueID();
 136
 137     static int page_id = 0;
 138     content::RenderFrameHost* rfh =
 139         WebContentsTester::For(web_contents())->GetPendingMainFrame();
 140     if (!rfh) {
 141       rfh = web_contents()->GetMainFrame();
 142     }
 143     WebContentsTester::For(web_contents())->ProceedWithCrossSiteNavigation();
 144     WebContentsTester::For(web_contents())->TestDidNavigateWithReferrer(
 145         rfh, ++page_id, pending_id, true, url,
 146         content::Referrer(referrer, blink::WebReferrerPolicyDefault), type);
 147   }
 148
 149   bool ExtractFeatures(ClientPhishingRequest* request) {
 150     StartExtractFeatures(request);
 151     base::MessageLoop::current()->Run();
 152     EXPECT_EQ(1U, success_.count(request));
 153     return success_.count(request) ? success_[request] : false;
 154   }
 155
 156   void StartExtractFeatures(ClientPhishingRequest* request) {
 157     success_.erase(request);
 158     ++num_pending_;
 159     extractor_->ExtractFeatures(
 160         browse_info_.get(),
 161         request,
 162         base::Bind(&BrowserFeatureExtractorTest::ExtractFeaturesDone,
 163                    base::Unretained(this)));
 164   }
 165
 166   void GetFeatureMap(const ClientPhishingRequest& request,
 167                      std::map<std::string, double>* features) {
 168     for (int i = 0; i < request.non_model_feature_map_size(); ++i) {
 169       const ClientPhishingRequest::Feature& feature =
 170           request.non_model_feature_map(i);
 171       EXPECT_EQ(0U, features->count(feature.name()));
 172       (*features)[feature.name()] = feature.value();
 173     }
 174   }
 175
 176   void ExtractMalwareFeatures(ClientMalwareRequest* request) {
 177     // Feature extraction takes ownership of the request object
 178     // and passes it along to the done callback in the end.
 179     StartExtractMalwareFeatures(request);
 180     base::MessageLoopForUI::current()->Run();
 181     EXPECT_EQ(1U, success_.count(request));
 182     EXPECT_TRUE(success_[request]);
 183   }
 184
 185   void StartExtractMalwareFeatures(ClientMalwareRequest* request) {
 186     success_.erase(request);
 187     ++num_pending_;
 188     // We temporarily give up ownership of request to ExtractMalwareFeatures
 189     // but we'll regain ownership of it in ExtractMalwareFeaturesDone.
 190     extractor_->ExtractMalwareFeatures(
 191         browse_info_.get(),
 192         request,
 193         base::Bind(&BrowserFeatureExtractorTest::ExtractMalwareFeaturesDone,
 194                    base::Unretained(this)));
 195   }
 196
 197   void GetMalwareUrls(
 198       const ClientMalwareRequest& request,
 199       std::map<std::string, std::set<std::string> >* urls) {
 200     for (int i = 0; i < request.bad_ip_url_info_size(); ++i) {
 201       const ClientMalwareRequest::UrlInfo& urlinfo =
 202           request.bad_ip_url_info(i);
 203       (*urls)[urlinfo.ip()].insert(urlinfo.url());
 204     }
 205   }
 206
 207   int num_pending_;  // Number of pending feature extractions.
 208   scoped_ptr<BrowserFeatureExtractor> extractor_;
 209   std::map<void*, bool> success_;
 210   scoped_ptr<BrowseInfo> browse_info_;
 211   scoped_ptr<StrictMock<MockClientSideDetectionHost> > host_;
 212   scoped_refptr<StrictMock<MockSafeBrowsingDatabaseManager> > db_manager_;
 213
 214  private:
 215   void ExtractFeaturesDone(bool success,
 216                            scoped_ptr<ClientPhishingRequest> request) {
 217     EXPECT_TRUE(BrowserThread::CurrentlyOn(BrowserThread::UI));
 218     ASSERT_EQ(0U, success_.count(request.get()));
 219     // The pointer doesn't really belong to us.  It belongs to
 220     // the test case which passed it to ExtractFeatures above.
 221     success_[request.release()] = success;
 222     if (--num_pending_ == 0) {
 223       base::MessageLoop::current()->Quit();
 224     }
 225   }
 226
 227   void ExtractMalwareFeaturesDone(
 228       bool success,
 229       scoped_ptr<ClientMalwareRequest> request) {
 230     EXPECT_TRUE(BrowserThread::CurrentlyOn(BrowserThread::UI));
 231     ASSERT_EQ(0U, success_.count(request.get()));
 232     // The pointer doesn't really belong to us.  It belongs to
 233     // the test case which passed it to ExtractMalwareFeatures above.
 234     success_[request.release()] = success;
 235     if (--num_pending_ == 0) {
 236       base::MessageLoopForUI::current()->Quit();
 237     }
 238   }
 239 };
 240
 241 TEST_F(BrowserFeatureExtractorTest, UrlNotInHistory) {
 242   ClientPhishingRequest request;
 243   SimpleNavigateAndCommit(GURL("http://www.google.com"));
 244   request.set_url("http://www.google.com/");
 245   request.set_client_score(0.5);
 246   EXPECT_FALSE(ExtractFeatures(&request));
 247 }
 248
 249 TEST_F(BrowserFeatureExtractorTest, RequestNotInitialized) {
 250   ClientPhishingRequest request;
 251   request.set_url("http://www.google.com/");
 252   // Request is missing the score value.
 253   SimpleNavigateAndCommit(GURL("http://www.google.com"));
 254   EXPECT_FALSE(ExtractFeatures(&request));
 255 }
 256
 257 TEST_F(BrowserFeatureExtractorTest, UrlInHistory) {
 258   history_service()->AddPage(GURL("http://www.foo.com/bar.html"),
 259                              base::Time::Now(),
 260                              history::SOURCE_BROWSED);
 261   history_service()->AddPage(GURL("https://www.foo.com/gaa.html"),
 262                              base::Time::Now(),
 263                              history::SOURCE_BROWSED);  // same host HTTPS.
 264   history_service()->AddPage(GURL("http://www.foo.com/gaa.html"),
 265                              base::Time::Now(),
 266                              history::SOURCE_BROWSED);  // same host HTTP.
 267   history_service()->AddPage(GURL("http://bar.foo.com/gaa.html"),
 268                              base::Time::Now(),
 269                              history::SOURCE_BROWSED);  // different host.
 270   history_service()->AddPage(GURL("http://www.foo.com/bar.html?a=b"),
 271                              base::Time::Now() - base::TimeDelta::FromHours(23),
 272                              NULL, 0, GURL(), history::RedirectList(),
 273                              ui::PAGE_TRANSITION_LINK,
 274                              history::SOURCE_BROWSED, false);
 275   history_service()->AddPage(GURL("http://www.foo.com/bar.html"),
 276                              base::Time::Now() - base::TimeDelta::FromHours(25),
 277                              NULL, 0, GURL(), history::RedirectList(),
 278                              ui::PAGE_TRANSITION_TYPED,
 279                              history::SOURCE_BROWSED, false);
 280   history_service()->AddPage(GURL("https://www.foo.com/goo.html"),
 281                              base::Time::Now() - base::TimeDelta::FromDays(5),
 282                              NULL, 0, GURL(), history::RedirectList(),
 283                              ui::PAGE_TRANSITION_TYPED,
 284                              history::SOURCE_BROWSED, false);
 285
 286   SimpleNavigateAndCommit(GURL("http://www.foo.com/bar.html"));
 287
 288   ClientPhishingRequest request;
 289   request.set_url("http://www.foo.com/bar.html");
 290   request.set_client_score(0.5);
 291   EXPECT_TRUE(ExtractFeatures(&request));
 292   std::map<std::string, double> features;
 293   GetFeatureMap(request, &features);
 294
 295   EXPECT_EQ(12U, features.size());
 296   EXPECT_DOUBLE_EQ(2.0, features[features::kUrlHistoryVisitCount]);
 297   EXPECT_DOUBLE_EQ(1.0,
 298                    features[features::kUrlHistoryVisitCountMoreThan24hAgo]);
 299   EXPECT_DOUBLE_EQ(1.0, features[features::kUrlHistoryTypedCount]);
 300   EXPECT_DOUBLE_EQ(1.0, features[features::kUrlHistoryLinkCount]);
 301   EXPECT_DOUBLE_EQ(4.0, features[features::kHttpHostVisitCount]);
 302   EXPECT_DOUBLE_EQ(2.0, features[features::kHttpsHostVisitCount]);
 303   EXPECT_DOUBLE_EQ(1.0, features[features::kFirstHttpHostVisitMoreThan24hAgo]);
 304   EXPECT_DOUBLE_EQ(1.0, features[features::kFirstHttpsHostVisitMoreThan24hAgo]);
 305
 306   request.Clear();
 307   request.set_url("http://bar.foo.com/gaa.html");
 308   request.set_client_score(0.5);
 309   EXPECT_TRUE(ExtractFeatures(&request));
 310   features.clear();
 311   GetFeatureMap(request, &features);
 312   // We have less features because we didn't Navigate to this page, so we don't
 313   // have Referrer, IsFirstNavigation, HasSSLReferrer, etc.
 314   EXPECT_EQ(7U, features.size());
 315   EXPECT_DOUBLE_EQ(1.0, features[features::kUrlHistoryVisitCount]);
 316   EXPECT_DOUBLE_EQ(0.0,
 317                    features[features::kUrlHistoryVisitCountMoreThan24hAgo]);
 318   EXPECT_DOUBLE_EQ(0.0, features[features::kUrlHistoryTypedCount]);
 319   EXPECT_DOUBLE_EQ(1.0, features[features::kUrlHistoryLinkCount]);
 320   EXPECT_DOUBLE_EQ(1.0, features[features::kHttpHostVisitCount]);
 321   EXPECT_DOUBLE_EQ(0.0, features[features::kHttpsHostVisitCount]);
 322   EXPECT_DOUBLE_EQ(0.0, features[features::kFirstHttpHostVisitMoreThan24hAgo]);
 323   EXPECT_FALSE(features.count(features::kFirstHttpsHostVisitMoreThan24hAgo));
 324 }
 325
 326 TEST_F(BrowserFeatureExtractorTest, MultipleRequestsAtOnce) {
 327   history_service()->AddPage(GURL("http://www.foo.com/bar.html"),
 328                              base::Time::Now(),
 329                              history::SOURCE_BROWSED);
 330   SimpleNavigateAndCommit(GURL("http:/www.foo.com/bar.html"));
 331   ClientPhishingRequest request;
 332   request.set_url("http://www.foo.com/bar.html");
 333   request.set_client_score(0.5);
 334   StartExtractFeatures(&request);
 335
 336   SimpleNavigateAndCommit(GURL("http://www.foo.com/goo.html"));
 337   ClientPhishingRequest request2;
 338   request2.set_url("http://www.foo.com/goo.html");
 339   request2.set_client_score(1.0);
 340   StartExtractFeatures(&request2);
 341
 342   base::MessageLoop::current()->Run();
 343   EXPECT_TRUE(success_[&request]);
 344   // Success is false because the second URL is not in the history and we are
 345   // not able to distinguish between a missing URL in the history and an error.
 346   EXPECT_FALSE(success_[&request2]);
 347 }
 348
 349 TEST_F(BrowserFeatureExtractorTest, BrowseFeatures) {
 350   history_service()->AddPage(GURL("http://www.foo.com/"),
 351                              base::Time::Now(),
 352                              history::SOURCE_BROWSED);
 353   history_service()->AddPage(GURL("http://www.foo.com/page.html"),
 354                              base::Time::Now(),
 355                              history::SOURCE_BROWSED);
 356   history_service()->AddPage(GURL("http://www.bar.com/"),
 357                              base::Time::Now(),
 358                              history::SOURCE_BROWSED);
 359   history_service()->AddPage(GURL("http://www.bar.com/other_page.html"),
 360                              base::Time::Now(),
 361                              history::SOURCE_BROWSED);
 362   history_service()->AddPage(GURL("http://www.baz.com/"),
 363                              base::Time::Now(),
 364                              history::SOURCE_BROWSED);
 365
 366   ClientPhishingRequest request;
 367   request.set_url("http://www.foo.com/");
 368   request.set_client_score(0.5);
 369   std::vector<GURL> redirect_chain;
 370   redirect_chain.push_back(GURL("http://somerandomwebsite.com/"));
 371   redirect_chain.push_back(GURL("http://www.foo.com/"));
 372   SetRedirectChain(redirect_chain, true);
 373   browse_info_->http_status_code = 200;
 374   NavigateAndCommit(GURL("http://www.foo.com/"),
 375                     GURL("http://google.com/"),
 376                     ui::PageTransitionFromInt(
 377                         ui::PAGE_TRANSITION_AUTO_BOOKMARK |
 378                         ui::PAGE_TRANSITION_FORWARD_BACK));
 379
 380   EXPECT_TRUE(ExtractFeatures(&request));
 381   std::map<std::string, double> features;
 382   GetFeatureMap(request, &features);
 383
 384   EXPECT_EQ(1.0,
 385             features[base::StringPrintf("%s=%s",
 386                                         features::kReferrer,
 387                                         "http://google.com/")]);
 388   EXPECT_EQ(1.0,
 389             features[base::StringPrintf("%s[0]=%s",
 390                                         features::kRedirect,
 391                                         "http://somerandomwebsite.com/")]);
 392   // We shouldn't have a feature for the last redirect in the chain, since it
 393   // should always be the URL that we navigated to.
 394   EXPECT_EQ(0.0,
 395             features[base::StringPrintf("%s[1]=%s",
 396                                         features::kRedirect,
 397                                         "http://foo.com/")]);
 398   EXPECT_EQ(0.0, features[features::kHasSSLReferrer]);
 399   EXPECT_EQ(2.0, features[features::kPageTransitionType]);
 400   EXPECT_EQ(1.0, features[features::kIsFirstNavigation]);
 401   EXPECT_EQ(200.0, features[features::kHttpStatusCode]);
 402
 403   request.Clear();
 404   request.set_url("http://www.foo.com/page.html");
 405   request.set_client_score(0.5);
 406   redirect_chain.clear();
 407   redirect_chain.push_back(GURL("http://www.foo.com/redirect"));
 408   redirect_chain.push_back(GURL("http://www.foo.com/second_redirect"));
 409   redirect_chain.push_back(GURL("http://www.foo.com/page.html"));
 410   SetRedirectChain(redirect_chain, false);
 411   browse_info_->http_status_code = 404;
 412   NavigateAndCommit(GURL("http://www.foo.com/page.html"),
 413                     GURL("http://www.foo.com"),
 414                     ui::PageTransitionFromInt(
 415                         ui::PAGE_TRANSITION_TYPED |
 416                         ui::PAGE_TRANSITION_CHAIN_START |
 417                         ui::PAGE_TRANSITION_CLIENT_REDIRECT));
 418
 419   EXPECT_TRUE(ExtractFeatures(&request));
 420   features.clear();
 421   GetFeatureMap(request, &features);
 422
 423   EXPECT_EQ(1,
 424             features[base::StringPrintf("%s=%s",
 425                                         features::kReferrer,
 426                                         "http://www.foo.com/")]);
 427   EXPECT_EQ(1.0,
 428             features[base::StringPrintf("%s[0]=%s",
 429                                         features::kRedirect,
 430                                         "http://www.foo.com/redirect")]);
 431   EXPECT_EQ(1.0,
 432             features[base::StringPrintf("%s[1]=%s",
 433                                         features::kRedirect,
 434                                         "http://www.foo.com/second_redirect")]);
 435   EXPECT_EQ(0.0, features[features::kHasSSLReferrer]);
 436   EXPECT_EQ(1.0, features[features::kPageTransitionType]);
 437   EXPECT_EQ(0.0, features[features::kIsFirstNavigation]);
 438   EXPECT_EQ(1.0,
 439             features[base::StringPrintf("%s%s=%s",
 440                                         features::kHostPrefix,
 441                                         features::kReferrer,
 442                                         "http://google.com/")]);
 443   EXPECT_EQ(1.0,
 444             features[base::StringPrintf("%s%s[0]=%s",
 445                                         features::kHostPrefix,
 446                                         features::kRedirect,
 447                                         "http://somerandomwebsite.com/")]);
 448   EXPECT_EQ(2.0,
 449             features[base::StringPrintf("%s%s",
 450                                         features::kHostPrefix,
 451                                         features::kPageTransitionType)]);
 452   EXPECT_EQ(1.0,
 453             features[base::StringPrintf("%s%s",
 454                                         features::kHostPrefix,
 455                                         features::kIsFirstNavigation)]);
 456   EXPECT_EQ(404.0, features[features::kHttpStatusCode]);
 457
 458   request.Clear();
 459   request.set_url("http://www.bar.com/");
 460   request.set_client_score(0.5);
 461   redirect_chain.clear();
 462   redirect_chain.push_back(GURL("http://www.foo.com/page.html"));
 463   redirect_chain.push_back(GURL("http://www.bar.com/"));
 464   SetRedirectChain(redirect_chain, true);
 465   NavigateAndCommit(GURL("http://www.bar.com/"),
 466                     GURL("http://www.foo.com/page.html"),
 467                     ui::PageTransitionFromInt(
 468                         ui::PAGE_TRANSITION_LINK |
 469                         ui::PAGE_TRANSITION_CHAIN_END |
 470                         ui::PAGE_TRANSITION_CLIENT_REDIRECT));
 471
 472   EXPECT_TRUE(ExtractFeatures(&request));
 473   features.clear();
 474   GetFeatureMap(request, &features);
 475
 476   EXPECT_EQ(1.0,
 477             features[base::StringPrintf("%s=%s",
 478                                         features::kReferrer,
 479                                         "http://www.foo.com/page.html")]);
 480   EXPECT_EQ(1.0,
 481             features[base::StringPrintf("%s[0]=%s",
 482                                         features::kRedirect,
 483                                         "http://www.foo.com/page.html")]);
 484   EXPECT_EQ(0.0, features[features::kHasSSLReferrer]);
 485   EXPECT_EQ(0.0, features[features::kPageTransitionType]);
 486   EXPECT_EQ(0.0, features[features::kIsFirstNavigation]);
 487
 488   // Should not have host features.
 489   EXPECT_EQ(0U,
 490             features.count(base::StringPrintf("%s%s",
 491                                               features::kHostPrefix,
 492                                               features::kPageTransitionType)));
 493   EXPECT_EQ(0U,
 494             features.count(base::StringPrintf("%s%s",
 495                                               features::kHostPrefix,
 496                                               features::kIsFirstNavigation)));
 497
 498   request.Clear();
 499   request.set_url("http://www.bar.com/other_page.html");
 500   request.set_client_score(0.5);
 501   redirect_chain.clear();
 502   redirect_chain.push_back(GURL("http://www.bar.com/other_page.html"));
 503   SetRedirectChain(redirect_chain, false);
 504   NavigateAndCommit(GURL("http://www.bar.com/other_page.html"),
 505                     GURL("http://www.bar.com/"),
 506                     ui::PAGE_TRANSITION_LINK);
 507
 508   EXPECT_TRUE(ExtractFeatures(&request));
 509   features.clear();
 510   GetFeatureMap(request, &features);
 511
 512   EXPECT_EQ(1.0,
 513             features[base::StringPrintf("%s=%s",
 514                                         features::kReferrer,
 515                                         "http://www.bar.com/")]);
 516   EXPECT_EQ(0.0, features[features::kHasSSLReferrer]);
 517   EXPECT_EQ(0.0, features[features::kPageTransitionType]);
 518   EXPECT_EQ(0.0, features[features::kIsFirstNavigation]);
 519   EXPECT_EQ(1.0,
 520             features[base::StringPrintf("%s%s=%s",
 521                                         features::kHostPrefix,
 522                                         features::kReferrer,
 523                                         "http://www.foo.com/page.html")]);
 524   EXPECT_EQ(1.0,
 525             features[base::StringPrintf("%s%s[0]=%s",
 526                                         features::kHostPrefix,
 527                                         features::kRedirect,
 528                                         "http://www.foo.com/page.html")]);
 529   EXPECT_EQ(0.0,
 530             features[base::StringPrintf("%s%s",
 531                                         features::kHostPrefix,
 532                                         features::kPageTransitionType)]);
 533   EXPECT_EQ(0.0,
 534             features[base::StringPrintf("%s%s",
 535                                         features::kHostPrefix,
 536                                         features::kIsFirstNavigation)]);
 537   request.Clear();
 538   request.set_url("http://www.baz.com/");
 539   request.set_client_score(0.5);
 540   redirect_chain.clear();
 541   redirect_chain.push_back(GURL("https://bankofamerica.com"));
 542   redirect_chain.push_back(GURL("http://www.baz.com/"));
 543   SetRedirectChain(redirect_chain, true);
 544   NavigateAndCommit(GURL("http://www.baz.com"),
 545                     GURL("https://bankofamerica.com"),
 546                     ui::PAGE_TRANSITION_GENERATED);
 547
 548   EXPECT_TRUE(ExtractFeatures(&request));
 549   features.clear();
 550   GetFeatureMap(request, &features);
 551
 552   EXPECT_EQ(1.0,
 553             features[base::StringPrintf("%s[0]=%s",
 554                                         features::kRedirect,
 555                                         features::kSecureRedirectValue)]);
 556   EXPECT_EQ(1.0, features[features::kHasSSLReferrer]);
 557   EXPECT_EQ(5.0, features[features::kPageTransitionType]);
 558   // Should not have redirect or host features.
 559   EXPECT_EQ(0U,
 560             features.count(base::StringPrintf("%s%s",
 561                                               features::kHostPrefix,
 562                                               features::kPageTransitionType)));
 563   EXPECT_EQ(0U,
 564             features.count(base::StringPrintf("%s%s",
 565                                               features::kHostPrefix,
 566                                               features::kIsFirstNavigation)));
 567   EXPECT_EQ(5.0, features[features::kPageTransitionType]);
 568 }
 569
 570 TEST_F(BrowserFeatureExtractorTest, SafeBrowsingFeatures) {
 571   SimpleNavigateAndCommit(GURL("http://www.foo.com/malware.html"));
 572   ClientPhishingRequest request;
 573   request.set_url("http://www.foo.com/malware.html");
 574   request.set_client_score(0.5);
 575
 576   browse_info_->unsafe_resource.reset(
 577       new SafeBrowsingUIManager::UnsafeResource);
 578   browse_info_->unsafe_resource->url = GURL("http://www.malware.com/");
 579   browse_info_->unsafe_resource->original_url = GURL("http://www.good.com/");
 580   browse_info_->unsafe_resource->is_subresource = true;
 581   browse_info_->unsafe_resource->threat_type = SB_THREAT_TYPE_URL_MALWARE;
 582
 583   ExtractFeatures(&request);
 584   std::map<std::string, double> features;
 585   GetFeatureMap(request, &features);
 586   EXPECT_TRUE(features.count(base::StringPrintf(
 587       "%s%s",
 588       features::kSafeBrowsingMaliciousUrl,
 589       "http://www.malware.com/")));
 590   EXPECT_TRUE(features.count(base::StringPrintf(
 591       "%s%s",
 592        features::kSafeBrowsingOriginalUrl,
 593         "http://www.good.com/")));
 594   EXPECT_DOUBLE_EQ(1.0, features[features::kSafeBrowsingIsSubresource]);
 595   EXPECT_DOUBLE_EQ(2.0, features[features::kSafeBrowsingThreatType]);
 596 }
 597
 598 TEST_F(BrowserFeatureExtractorTest, MalwareFeatures) {
 599   ClientMalwareRequest request;
 600   request.set_url("http://www.foo.com/");
 601
 602   std::vector<IPUrlInfo> bad_urls;
 603   bad_urls.push_back(
 604       IPUrlInfo("http://bad.com", "GET", "", content::RESOURCE_TYPE_SCRIPT));
 605   bad_urls.push_back(
 606       IPUrlInfo("http://evil.com", "GET", "", content::RESOURCE_TYPE_SCRIPT));
 607   browse_info_->ips.insert(std::make_pair("193.5.163.8", bad_urls));
 608   browse_info_->ips.insert(std::make_pair("92.92.92.92", bad_urls));
 609   std::vector<IPUrlInfo> good_urls;
 610   good_urls.push_back(
 611       IPUrlInfo("http://ok.com", "GET", "", content::RESOURCE_TYPE_SCRIPT));
 612   browse_info_->ips.insert(std::make_pair("23.94.78.1", good_urls));
 613   EXPECT_CALL(*db_manager_, MatchMalwareIP("193.5.163.8"))
 614       .WillOnce(Return(true));
 615   EXPECT_CALL(*db_manager_, MatchMalwareIP("92.92.92.92"))
 616       .WillOnce(Return(true));
 617   EXPECT_CALL(*db_manager_, MatchMalwareIP("23.94.78.1"))
 618       .WillOnce(Return(false));
 619
 620   ExtractMalwareFeatures(&request);
 621   EXPECT_EQ(4, request.bad_ip_url_info_size());
 622   std::map<std::string, std::set<std::string> > result_urls;
 623   GetMalwareUrls(request, &result_urls);
 624
 625   EXPECT_EQ(2U, result_urls.size());
 626   EXPECT_TRUE(result_urls.count("193.5.163.8"));
 627   std::set<std::string> urls = result_urls["193.5.163.8"];
 628   EXPECT_EQ(2U, urls.size());
 629   EXPECT_TRUE(urls.find("http://bad.com") != urls.end());
 630   EXPECT_TRUE(urls.find("http://evil.com") != urls.end());
 631   EXPECT_TRUE(result_urls.count("92.92.92.92"));
 632   urls = result_urls["92.92.92.92"];
 633   EXPECT_EQ(2U, urls.size());
 634   EXPECT_TRUE(urls.find("http://bad.com") != urls.end());
 635   EXPECT_TRUE(urls.find("http://evil.com") != urls.end());
 636 }
 637
 638 TEST_F(BrowserFeatureExtractorTest, MalwareFeatures_ExceedLimit) {
 639   ClientMalwareRequest request;
 640   request.set_url("http://www.foo.com/");
 641
 642   std::vector<IPUrlInfo> bad_urls;
 643   bad_urls.push_back(
 644       IPUrlInfo("http://bad.com", "GET", "", content::RESOURCE_TYPE_SCRIPT));
 645   std::vector<std::string> ips;
 646   for (int i = 0; i < 7; ++i) {  // Add 7 ips
 647     std::string ip = base::StringPrintf("%d.%d.%d.%d", i, i, i, i);
 648     ips.push_back(ip);
 649     browse_info_->ips.insert(std::make_pair(ip, bad_urls));
 650
 651     // First ip is good but all the others are bad.
 652     EXPECT_CALL(*db_manager_, MatchMalwareIP(ip)).WillOnce(Return(i > 0));
 653   }
 654
 655   ExtractMalwareFeatures(&request);
 656   // The number of IP matched url we store is capped at 5 IPs per request.
 657   EXPECT_EQ(5, request.bad_ip_url_info_size());
 658 }
 659
 660 }  // namespace safe_browsing