chrome/browser/safe_browsing/browser_feature_extractor.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "chrome/browser/safe_browsing/browser_feature_extractor.h"
   6
   7 #include <map>
   8 #include <utility>
   9
  10 #include "base/bind.h"
  11 #include "base/bind_helpers.h"
  12 #include "base/format_macros.h"
  13 #include "base/stl_util.h"
  14 #include "base/strings/stringprintf.h"
  15 #include "base/time/time.h"
  16 #include "chrome/browser/common/cancelable_request.h"
  17 #include "chrome/browser/history/history_service.h"
  18 #include "chrome/browser/history/history_service_factory.h"
  19 #include "chrome/browser/history/history_types.h"
  20 #include "chrome/browser/profiles/profile.h"
  21 #include "chrome/browser/safe_browsing/browser_features.h"
  22 #include "chrome/browser/safe_browsing/client_side_detection_host.h"
  23 #include "chrome/browser/safe_browsing/database_manager.h"
  24 #include "chrome/common/safe_browsing/csd.pb.h"
  25 #include "content/public/browser/browser_thread.h"
  26 #include "content/public/browser/navigation_controller.h"
  27 #include "content/public/browser/navigation_entry.h"
  28 #include "content/public/browser/web_contents.h"
  29 #include "content/public/common/page_transition_types.h"
  30 #include "url/gurl.h"
  31
  32 using content::BrowserThread;
  33 using content::NavigationController;
  34 using content::NavigationEntry;
  35 using content::WebContents;
  36
  37 namespace safe_browsing {
  38
  39 namespace {
  40
  41 const int kMaxMalwareIPPerRequest = 5;
  42
  43 void FilterBenignIpsOnIOThread(
  44     scoped_refptr<SafeBrowsingDatabaseManager> database_manager,
  45     IPUrlMap* ips) {
  46   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
  47   for (IPUrlMap::iterator it = ips->begin(); it != ips->end();) {
  48     if (!database_manager.get() ||
  49         !database_manager->MatchMalwareIP(it->first)) {
  50       // it++ here returns a copy of the old iterator and passes it to erase.
  51       ips->erase(it++);
  52     } else {
  53       ++it;
  54     }
  55   }
  56 }
  57 }  // namespace
  58
  59 IPUrlInfo::IPUrlInfo(const std::string& url,
  60                      const std::string& method,
  61                      const std::string& referrer,
  62                      const ResourceType::Type& resource_type)
  63       : url(url),
  64         method(method),
  65         referrer(referrer),
  66         resource_type(resource_type) {
  67 }
  68
  69 IPUrlInfo::~IPUrlInfo() {}
  70
  71 BrowseInfo::BrowseInfo() : http_status_code(0) {}
  72
  73 BrowseInfo::~BrowseInfo() {}
  74
  75 static void AddFeature(const std::string& feature_name,
  76                        double feature_value,
  77                        ClientPhishingRequest* request) {
  78   DCHECK(request);
  79   ClientPhishingRequest::Feature* feature =
  80       request->add_non_model_feature_map();
  81   feature->set_name(feature_name);
  82   feature->set_value(feature_value);
  83   VLOG(2) << "Browser feature: " << feature->name() << " " << feature->value();
  84 }
  85
  86 static void AddMalwareIpUrlInfo(const std::string& ip,
  87                                 const std::vector<IPUrlInfo>& meta_infos,
  88                                 ClientMalwareRequest* request) {
  89   DCHECK(request);
  90   for (std::vector<IPUrlInfo>::const_iterator it = meta_infos.begin();
  91        it != meta_infos.end(); ++it) {
  92     ClientMalwareRequest::UrlInfo* urlinfo =
  93         request->add_bad_ip_url_info();
  94     // We add the information about url on the bad ip.
  95     urlinfo->set_ip(ip);
  96     urlinfo->set_url(it->url);
  97     urlinfo->set_method(it->method);
  98     urlinfo->set_referrer(it->referrer);
  99     urlinfo->set_resource_type(static_cast<int>(it->resource_type));
 100   }
 101   DVLOG(2) << "Added url info for bad ip: " << ip;
 102 }
 103
 104 static void AddNavigationFeatures(
 105     const std::string& feature_prefix,
 106     const NavigationController& controller,
 107     int index,
 108     const std::vector<GURL>& redirect_chain,
 109     ClientPhishingRequest* request) {
 110   NavigationEntry* entry = controller.GetEntryAtIndex(index);
 111   bool is_secure_referrer = entry->GetReferrer().url.SchemeIsSecure();
 112   if (!is_secure_referrer) {
 113     AddFeature(base::StringPrintf("%s%s=%s",
 114                                   feature_prefix.c_str(),
 115                                   features::kReferrer,
 116                                   entry->GetReferrer().url.spec().c_str()),
 117                1.0,
 118                request);
 119   }
 120   AddFeature(feature_prefix + features::kHasSSLReferrer,
 121              is_secure_referrer ? 1.0 : 0.0,
 122              request);
 123   AddFeature(feature_prefix + features::kPageTransitionType,
 124              static_cast<double>(
 125                  content::PageTransitionStripQualifier(
 126                     entry->GetTransitionType())),
 127              request);
 128   AddFeature(feature_prefix + features::kIsFirstNavigation,
 129              index == 0 ? 1.0 : 0.0,
 130              request);
 131   // Redirect chain should always be at least of size one, as the rendered
 132   // url is the last element in the chain.
 133   if (redirect_chain.empty()) {
 134     NOTREACHED();
 135     return;
 136   }
 137   if (redirect_chain.back() != entry->GetURL()) {
 138     // I originally had this as a DCHECK but I saw a failure once that I
 139     // can't reproduce. It looks like it might be related to the
 140     // navigation controller only keeping a limited number of navigation
 141     // events. For now we'll just attach a feature specifying that this is
 142     // a mismatch and try and figure out what to do with it on the server.
 143     DLOG(WARNING) << "Expected:" << entry->GetURL()
 144                  << " Actual:" << redirect_chain.back();
 145     AddFeature(feature_prefix + features::kRedirectUrlMismatch,
 146                1.0,
 147                request);
 148     return;
 149   }
 150   // We skip the last element since it should just be the current url.
 151   for (size_t i = 0; i < redirect_chain.size() - 1; i++) {
 152     std::string printable_redirect = redirect_chain[i].spec();
 153     if (redirect_chain[i].SchemeIsSecure()) {
 154       printable_redirect = features::kSecureRedirectValue;
 155     }
 156     AddFeature(base::StringPrintf("%s%s[%" PRIuS "]=%s",
 157                                   feature_prefix.c_str(),
 158                                   features::kRedirect,
 159                                   i,
 160                                   printable_redirect.c_str()),
 161                1.0,
 162                request);
 163   }
 164 }
 165
 166 BrowserFeatureExtractor::BrowserFeatureExtractor(
 167     WebContents* tab,
 168     ClientSideDetectionHost* host)
 169     : tab_(tab),
 170       host_(host),
 171       weak_factory_(this) {
 172   DCHECK(tab);
 173 }
 174
 175 BrowserFeatureExtractor::~BrowserFeatureExtractor() {
 176   weak_factory_.InvalidateWeakPtrs();
 177   // Delete all the pending extractions (delete callback and request objects).
 178   STLDeleteContainerPairFirstPointers(pending_extractions_.begin(),
 179                                       pending_extractions_.end());
 180
 181   // Also cancel all the pending history service queries.
 182   HistoryService* history;
 183   bool success = GetHistoryService(&history);
 184   DCHECK(success || pending_queries_.size() == 0);
 185   // Cancel all the pending history lookups and cleanup the memory.
 186   for (PendingQueriesMap::iterator it = pending_queries_.begin();
 187        it != pending_queries_.end(); ++it) {
 188     if (history) {
 189       history->CancelRequest(it->first);
 190     }
 191     ExtractionData& extraction = it->second;
 192     delete extraction.first;  // delete request
 193   }
 194   pending_queries_.clear();
 195 }
 196
 197 void BrowserFeatureExtractor::ExtractFeatures(const BrowseInfo* info,
 198                                               ClientPhishingRequest* request,
 199                                               const DoneCallback& callback) {
 200   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 201   DCHECK(request);
 202   DCHECK(info);
 203   DCHECK_EQ(0U, request->url().find("http:"));
 204   DCHECK(!callback.is_null());
 205   // Extract features pertaining to this navigation.
 206   const NavigationController& controller = tab_->GetController();
 207   int url_index = -1;
 208   int first_host_index = -1;
 209
 210   GURL request_url(request->url());
 211   int index = controller.GetCurrentEntryIndex();
 212   // The url that we are extracting features for should already be commited.
 213   DCHECK_NE(index, -1);
 214   for (; index >= 0; index--) {
 215     NavigationEntry* entry = controller.GetEntryAtIndex(index);
 216     if (url_index == -1 && entry->GetURL() == request_url) {
 217       // It's possible that we've been on the on the possibly phishy url before
 218       // in this tab, so make sure that we use the latest navigation for
 219       // features.
 220       // Note that it's possible that the url_index should always be the
 221       // latest entry, but I'm worried about possible races during a navigation
 222       // and transient entries (i.e. interstiatials) so for now we will just
 223       // be cautious.
 224       url_index = index;
 225     } else if (index < url_index) {
 226       if (entry->GetURL().host() == request_url.host()) {
 227         first_host_index = index;
 228       } else {
 229         // We have found the possibly phishing url, but we are no longer on the
 230         // host. No reason to look back any further.
 231         break;
 232       }
 233     }
 234   }
 235
 236   // Add features pertaining to how we got to
 237   //   1) The candidate url
 238   //   2) The first url on the same host as the candidate url (assuming that
 239   //      it's different from the candidate url).
 240   if (url_index != -1) {
 241     AddNavigationFeatures(
 242         std::string(), controller, url_index, info->url_redirects, request);
 243   }
 244   if (first_host_index != -1) {
 245     AddNavigationFeatures(features::kHostPrefix,
 246                           controller,
 247                           first_host_index,
 248                           info->host_redirects,
 249                           request);
 250   }
 251
 252   ExtractBrowseInfoFeatures(*info, request);
 253   pending_extractions_[request] = callback;
 254   base::MessageLoop::current()->PostTask(
 255       FROM_HERE,
 256       base::Bind(&BrowserFeatureExtractor::StartExtractFeatures,
 257                  weak_factory_.GetWeakPtr(), request, callback));
 258 }
 259
 260 void BrowserFeatureExtractor::ExtractMalwareFeatures(
 261     BrowseInfo* info,
 262     ClientMalwareRequest* request,
 263     const MalwareDoneCallback& callback) {
 264   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 265   DCHECK(!callback.is_null());
 266
 267   // Grab the IPs because they might go away before we're done
 268   // checking them against the IP blacklist on the IO thread.
 269   scoped_ptr<IPUrlMap> ips(new IPUrlMap);
 270   ips->swap(info->ips);
 271
 272   IPUrlMap* ips_ptr = ips.get();
 273
 274   // The API doesn't take a scoped_ptr because the API gets mocked and we
 275   // cannot mock an API that takes scoped_ptr as arguments.
 276   scoped_ptr<ClientMalwareRequest> req(request);
 277
 278   // IP blacklist lookups have to happen on the IO thread.
 279   BrowserThread::PostTaskAndReply(
 280       BrowserThread::IO,
 281       FROM_HERE,
 282       base::Bind(&FilterBenignIpsOnIOThread,
 283                  host_->database_manager(),
 284                  ips_ptr),
 285       base::Bind(&BrowserFeatureExtractor::FinishExtractMalwareFeatures,
 286                  weak_factory_.GetWeakPtr(),
 287                  base::Passed(&ips), callback, base::Passed(&req)));
 288 }
 289
 290 void BrowserFeatureExtractor::ExtractBrowseInfoFeatures(
 291     const BrowseInfo& info,
 292     ClientPhishingRequest* request) {
 293   if (info.unsafe_resource.get()) {
 294     // A SafeBrowsing interstitial was shown for the current URL.
 295     AddFeature(features::kSafeBrowsingMaliciousUrl +
 296                info.unsafe_resource->url.spec(),
 297                1.0,
 298                request);
 299     AddFeature(features::kSafeBrowsingOriginalUrl +
 300                info.unsafe_resource->original_url.spec(),
 301                1.0,
 302                request);
 303     AddFeature(features::kSafeBrowsingIsSubresource,
 304                info.unsafe_resource->is_subresource ? 1.0 : 0.0,
 305                request);
 306     AddFeature(features::kSafeBrowsingThreatType,
 307                static_cast<double>(info.unsafe_resource->threat_type),
 308                request);
 309   }
 310   if (info.http_status_code != 0) {
 311     AddFeature(features::kHttpStatusCode, info.http_status_code, request);
 312   }
 313 }
 314
 315 void BrowserFeatureExtractor::StartExtractFeatures(
 316     ClientPhishingRequest* request,
 317     const DoneCallback& callback) {
 318   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 319   size_t removed = pending_extractions_.erase(request);
 320   DCHECK_EQ(1U, removed);
 321   HistoryService* history;
 322   if (!request || !request->IsInitialized() || !GetHistoryService(&history)) {
 323     callback.Run(false, request);
 324     return;
 325   }
 326   CancelableRequestProvider::Handle handle = history->QueryURL(
 327       GURL(request->url()),
 328       true /* wants_visits */,
 329       &request_consumer_,
 330       base::Bind(&BrowserFeatureExtractor::QueryUrlHistoryDone,
 331                  base::Unretained(this)));
 332
 333   StorePendingQuery(handle, request, callback);
 334 }
 335
 336 void BrowserFeatureExtractor::QueryUrlHistoryDone(
 337     CancelableRequestProvider::Handle handle,
 338     bool success,
 339     const history::URLRow* row,
 340     history::VisitVector* visits) {
 341   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 342   ClientPhishingRequest* request;
 343   DoneCallback callback;
 344   if (!GetPendingQuery(handle, &request, &callback)) {
 345     DLOG(FATAL) << "No pending history query found";
 346     return;
 347   }
 348   DCHECK(request);
 349   DCHECK(!callback.is_null());
 350   if (!success) {
 351     // URL is not found in the history.  In practice this should not
 352     // happen (unless there is a real error) because we just visited
 353     // that URL.
 354     callback.Run(false, request);
 355     return;
 356   }
 357   AddFeature(features::kUrlHistoryVisitCount,
 358              static_cast<double>(row->visit_count()),
 359              request);
 360
 361   base::Time threshold = base::Time::Now() - base::TimeDelta::FromDays(1);
 362   int num_visits_24h_ago = 0;
 363   int num_visits_typed = 0;
 364   int num_visits_link = 0;
 365   for (history::VisitVector::const_iterator it = visits->begin();
 366        it != visits->end(); ++it) {
 367     if (!content::PageTransitionIsMainFrame(it->transition)) {
 368       continue;
 369     }
 370     if (it->visit_time < threshold) {
 371       ++num_visits_24h_ago;
 372     }
 373     content::PageTransition transition = content::PageTransitionStripQualifier(
 374         it->transition);
 375     if (transition == content::PAGE_TRANSITION_TYPED) {
 376       ++num_visits_typed;
 377     } else if (transition == content::PAGE_TRANSITION_LINK) {
 378       ++num_visits_link;
 379     }
 380   }
 381   AddFeature(features::kUrlHistoryVisitCountMoreThan24hAgo,
 382              static_cast<double>(num_visits_24h_ago),
 383              request);
 384   AddFeature(features::kUrlHistoryTypedCount,
 385              static_cast<double>(num_visits_typed),
 386              request);
 387   AddFeature(features::kUrlHistoryLinkCount,
 388              static_cast<double>(num_visits_link),
 389              request);
 390
 391   // Issue next history lookup for host visits.
 392   HistoryService* history;
 393   if (!GetHistoryService(&history)) {
 394     callback.Run(false, request);
 395     return;
 396   }
 397   CancelableRequestProvider::Handle next_handle =
 398       history->GetVisibleVisitCountToHost(
 399           GURL(request->url()),
 400           &request_consumer_,
 401           base::Bind(&BrowserFeatureExtractor::QueryHttpHostVisitsDone,
 402                      base::Unretained(this)));
 403   StorePendingQuery(next_handle, request, callback);
 404 }
 405
 406 void BrowserFeatureExtractor::QueryHttpHostVisitsDone(
 407     CancelableRequestProvider::Handle handle,
 408     bool success,
 409     int num_visits,
 410     base::Time first_visit) {
 411   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 412   ClientPhishingRequest* request;
 413   DoneCallback callback;
 414   if (!GetPendingQuery(handle, &request, &callback)) {
 415     DLOG(FATAL) << "No pending history query found";
 416     return;
 417   }
 418   DCHECK(request);
 419   DCHECK(!callback.is_null());
 420   if (!success) {
 421     callback.Run(false, request);
 422     return;
 423   }
 424   SetHostVisitsFeatures(num_visits, first_visit, true, request);
 425
 426   // Same lookup but for the HTTPS URL.
 427   HistoryService* history;
 428   if (!GetHistoryService(&history)) {
 429     callback.Run(false, request);
 430     return;
 431   }
 432   std::string https_url = request->url();
 433   CancelableRequestProvider::Handle next_handle =
 434       history->GetVisibleVisitCountToHost(
 435           GURL(https_url.replace(0, 5, "https:")),
 436           &request_consumer_,
 437           base::Bind(&BrowserFeatureExtractor::QueryHttpsHostVisitsDone,
 438                      base::Unretained(this)));
 439   StorePendingQuery(next_handle, request, callback);
 440 }
 441
 442 void BrowserFeatureExtractor::QueryHttpsHostVisitsDone(
 443     CancelableRequestProvider::Handle handle,
 444     bool success,
 445     int num_visits,
 446     base::Time first_visit) {
 447   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 448   ClientPhishingRequest* request;
 449   DoneCallback callback;
 450   if (!GetPendingQuery(handle, &request, &callback)) {
 451     DLOG(FATAL) << "No pending history query found";
 452     return;
 453   }
 454   DCHECK(request);
 455   DCHECK(!callback.is_null());
 456   if (!success) {
 457     callback.Run(false, request);
 458     return;
 459   }
 460   SetHostVisitsFeatures(num_visits, first_visit, false, request);
 461   callback.Run(true, request);  // We're done with all the history lookups.
 462 }
 463
 464 void BrowserFeatureExtractor::SetHostVisitsFeatures(
 465     int num_visits,
 466     base::Time first_visit,
 467     bool is_http_query,
 468     ClientPhishingRequest* request) {
 469   DCHECK(request);
 470   AddFeature(is_http_query ?
 471              features::kHttpHostVisitCount : features::kHttpsHostVisitCount,
 472              static_cast<double>(num_visits),
 473              request);
 474   if (num_visits > 0) {
 475     AddFeature(
 476         is_http_query ?
 477         features::kFirstHttpHostVisitMoreThan24hAgo :
 478         features::kFirstHttpsHostVisitMoreThan24hAgo,
 479         (first_visit < (base::Time::Now() - base::TimeDelta::FromDays(1))) ?
 480         1.0 : 0.0,
 481         request);
 482   }
 483 }
 484
 485 void BrowserFeatureExtractor::StorePendingQuery(
 486     CancelableRequestProvider::Handle handle,
 487     ClientPhishingRequest* request,
 488     const DoneCallback& callback) {
 489   DCHECK_EQ(0U, pending_queries_.count(handle));
 490   pending_queries_[handle] = std::make_pair(request, callback);
 491 }
 492
 493 bool BrowserFeatureExtractor::GetPendingQuery(
 494     CancelableRequestProvider::Handle handle,
 495     ClientPhishingRequest** request,
 496     DoneCallback* callback) {
 497   PendingQueriesMap::iterator it = pending_queries_.find(handle);
 498   DCHECK(it != pending_queries_.end());
 499   if (it != pending_queries_.end()) {
 500     *request = it->second.first;
 501     *callback = it->second.second;
 502     pending_queries_.erase(it);
 503     return true;
 504   }
 505   return false;
 506 }
 507
 508 bool BrowserFeatureExtractor::GetHistoryService(HistoryService** history) {
 509   *history = NULL;
 510   if (tab_ && tab_->GetBrowserContext()) {
 511     Profile* profile = Profile::FromBrowserContext(tab_->GetBrowserContext());
 512     *history = HistoryServiceFactory::GetForProfile(profile,
 513                                                     Profile::EXPLICIT_ACCESS);
 514     if (*history) {
 515       return true;
 516     }
 517   }
 518   VLOG(2) << "Unable to query history.  No history service available.";
 519   return false;
 520 }
 521
 522 void BrowserFeatureExtractor::FinishExtractMalwareFeatures(
 523     scoped_ptr<IPUrlMap> bad_ips,
 524     MalwareDoneCallback callback,
 525     scoped_ptr<ClientMalwareRequest> request) {
 526   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 527   int matched_bad_ips = 0;
 528   for (IPUrlMap::const_iterator it = bad_ips->begin();
 529        it != bad_ips->end(); ++it) {
 530     AddMalwareIpUrlInfo(it->first, it->second, request.get());
 531     ++matched_bad_ips;
 532     // Limit the number of matched bad IPs in one request to control
 533     // the request's size
 534     if (matched_bad_ips >= kMaxMalwareIPPerRequest) {
 535       break;
 536     }
 537   }
 538   callback.Run(true, request.Pass());
 539 }
 540
 541 }  // namespace safe_browsing