chrome/browser/safe_browsing/browser_feature_extractor.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "chrome/browser/safe_browsing/browser_feature_extractor.h"
   6
   7 #include <map>
   8 #include <utility>
   9
  10 #include "base/bind.h"
  11 #include "base/bind_helpers.h"
  12 #include "base/format_macros.h"
  13 #include "base/stl_util.h"
  14 #include "base/strings/stringprintf.h"
  15 #include "base/time/time.h"
  16 #include "chrome/browser/common/cancelable_request.h"
  17 #include "chrome/browser/history/history_service.h"
  18 #include "chrome/browser/history/history_service_factory.h"
  19 #include "chrome/browser/history/history_types.h"
  20 #include "chrome/browser/profiles/profile.h"
  21 #include "chrome/browser/safe_browsing/browser_features.h"
  22 #include "chrome/browser/safe_browsing/client_side_detection_host.h"
  23 #include "chrome/browser/safe_browsing/database_manager.h"
  24 #include "chrome/common/safe_browsing/csd.pb.h"
  25 #include "content/public/browser/browser_thread.h"
  26 #include "content/public/browser/navigation_controller.h"
  27 #include "content/public/browser/navigation_entry.h"
  28 #include "content/public/browser/web_contents.h"
  29 #include "content/public/common/page_transition_types.h"
  30 #include "url/gurl.h"
  31
  32 using content::BrowserThread;
  33 using content::NavigationController;
  34 using content::NavigationEntry;
  35 using content::WebContents;
  36
  37 namespace safe_browsing {
  38
  39 namespace {
  40
  41 const int kMaxMalwareIPPerRequest = 5;
  42
  43 void FilterBenignIpsOnIOThread(
  44     scoped_refptr<SafeBrowsingDatabaseManager> database_manager,
  45     IPUrlMap* ips) {
  46   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
  47   for (IPUrlMap::iterator it = ips->begin(); it != ips->end();) {
  48     if (!database_manager.get() ||
  49         !database_manager->MatchMalwareIP(it->first)) {
  50       // it++ here returns a copy of the old iterator and passes it to erase.
  51       ips->erase(it++);
  52     } else {
  53       ++it;
  54     }
  55   }
  56 }
  57 }  // namespace
  58
  59 IPUrlInfo::IPUrlInfo(const std::string& url,
  60                      const std::string& method,
  61                      const std::string& referrer,
  62                      const ResourceType::Type& resource_type)
  63       : url(url),
  64         method(method),
  65         referrer(referrer),
  66         resource_type(resource_type) {
  67 }
  68
  69 IPUrlInfo::~IPUrlInfo() {}
  70
  71 BrowseInfo::BrowseInfo() : http_status_code(0) {}
  72
  73 BrowseInfo::~BrowseInfo() {}
  74
  75 static void AddFeature(const std::string& feature_name,
  76                        double feature_value,
  77                        ClientPhishingRequest* request) {
  78   DCHECK(request);
  79   ClientPhishingRequest::Feature* feature =
  80       request->add_non_model_feature_map();
  81   feature->set_name(feature_name);
  82   feature->set_value(feature_value);
  83   VLOG(2) << "Browser feature: " << feature->name() << " " << feature->value();
  84 }
  85
  86 static void AddMalwareIpUrlInfo(const std::string& ip,
  87                                 const std::vector<IPUrlInfo>& meta_infos,
  88                                 ClientMalwareRequest* request) {
  89   DCHECK(request);
  90   for (std::vector<IPUrlInfo>::const_iterator it = meta_infos.begin();
  91        it != meta_infos.end(); ++it) {
  92     ClientMalwareRequest::UrlInfo* urlinfo =
  93         request->add_bad_ip_url_info();
  94     // We add the information about url on the bad ip.
  95     urlinfo->set_ip(ip);
  96     urlinfo->set_url(it->url);
  97     urlinfo->set_method(it->method);
  98     urlinfo->set_referrer(it->referrer);
  99     urlinfo->set_resource_type(static_cast<int>(it->resource_type));
 100   }
 101   DVLOG(2) << "Added url info for bad ip: " << ip;
 102 }
 103
 104 static void AddNavigationFeatures(
 105     const std::string& feature_prefix,
 106     const NavigationController& controller,
 107     int index,
 108     const std::vector<GURL>& redirect_chain,
 109     ClientPhishingRequest* request) {
 110   NavigationEntry* entry = controller.GetEntryAtIndex(index);
 111   bool is_secure_referrer = entry->GetReferrer().url.SchemeIsSecure();
 112   if (!is_secure_referrer) {
 113     AddFeature(base::StringPrintf("%s%s=%s",
 114                                   feature_prefix.c_str(),
 115                                   features::kReferrer,
 116                                   entry->GetReferrer().url.spec().c_str()),
 117                1.0,
 118                request);
 119   }
 120   AddFeature(feature_prefix + features::kHasSSLReferrer,
 121              is_secure_referrer ? 1.0 : 0.0,
 122              request);
 123   AddFeature(feature_prefix + features::kPageTransitionType,
 124              static_cast<double>(
 125                  content::PageTransitionStripQualifier(
 126                     entry->GetTransitionType())),
 127              request);
 128   AddFeature(feature_prefix + features::kIsFirstNavigation,
 129              index == 0 ? 1.0 : 0.0,
 130              request);
 131   // Redirect chain should always be at least of size one, as the rendered
 132   // url is the last element in the chain.
 133   if (redirect_chain.empty()) {
 134     NOTREACHED();
 135     return;
 136   }
 137   if (redirect_chain.back() != entry->GetURL()) {
 138     // I originally had this as a DCHECK but I saw a failure once that I
 139     // can't reproduce. It looks like it might be related to the
 140     // navigation controller only keeping a limited number of navigation
 141     // events. For now we'll just attach a feature specifying that this is
 142     // a mismatch and try and figure out what to do with it on the server.
 143     DLOG(WARNING) << "Expected:" << entry->GetURL()
 144                  << " Actual:" << redirect_chain.back();
 145     AddFeature(feature_prefix + features::kRedirectUrlMismatch,
 146                1.0,
 147                request);
 148     return;
 149   }
 150   // We skip the last element since it should just be the current url.
 151   for (size_t i = 0; i < redirect_chain.size() - 1; i++) {
 152     std::string printable_redirect = redirect_chain[i].spec();
 153     if (redirect_chain[i].SchemeIsSecure()) {
 154       printable_redirect = features::kSecureRedirectValue;
 155     }
 156     AddFeature(base::StringPrintf("%s%s[%" PRIuS "]=%s",
 157                                   feature_prefix.c_str(),
 158                                   features::kRedirect,
 159                                   i,
 160                                   printable_redirect.c_str()),
 161                1.0,
 162                request);
 163   }
 164 }
 165
 166 BrowserFeatureExtractor::BrowserFeatureExtractor(
 167     WebContents* tab,
 168     ClientSideDetectionHost* host)
 169     : tab_(tab),
 170       host_(host),
 171       weak_factory_(this) {
 172   DCHECK(tab);
 173 }
 174
 175 BrowserFeatureExtractor::~BrowserFeatureExtractor() {
 176   weak_factory_.InvalidateWeakPtrs();
 177   // Delete all the pending extractions (delete callback and request objects).
 178   STLDeleteContainerPairFirstPointers(pending_extractions_.begin(),
 179                                       pending_extractions_.end());
 180
 181   // Also cancel all the pending history service queries.
 182   HistoryService* history;
 183   bool success = GetHistoryService(&history);
 184   DCHECK(success || pending_queries_.size() == 0);
 185   // Cancel all the pending history lookups and cleanup the memory.
 186   for (PendingQueriesMap::iterator it = pending_queries_.begin();
 187        it != pending_queries_.end(); ++it) {
 188     if (history) {
 189       history->CancelRequest(it->first);
 190     }
 191     ExtractionData& extraction = it->second;
 192     delete extraction.first;  // delete request
 193   }
 194   pending_queries_.clear();
 195 }
 196
 197 void BrowserFeatureExtractor::ExtractFeatures(const BrowseInfo* info,
 198                                               ClientPhishingRequest* request,
 199                                               const DoneCallback& callback) {
 200   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 201   DCHECK(request);
 202   DCHECK(info);
 203   DCHECK_EQ(0U, request->url().find("http:"));
 204   DCHECK(!callback.is_null());
 205   // Extract features pertaining to this navigation.
 206   const NavigationController& controller = tab_->GetController();
 207   int url_index = -1;
 208   int first_host_index = -1;
 209
 210   GURL request_url(request->url());
 211   int index = controller.GetCurrentEntryIndex();
 212   // The url that we are extracting features for should already be commited.
 213   DCHECK_NE(index, -1);
 214   for (; index >= 0; index--) {
 215     NavigationEntry* entry = controller.GetEntryAtIndex(index);
 216     if (url_index == -1 && entry->GetURL() == request_url) {
 217       // It's possible that we've been on the on the possibly phishy url before
 218       // in this tab, so make sure that we use the latest navigation for
 219       // features.
 220       // Note that it's possible that the url_index should always be the
 221       // latest entry, but I'm worried about possible races during a navigation
 222       // and transient entries (i.e. interstiatials) so for now we will just
 223       // be cautious.
 224       url_index = index;
 225     } else if (index < url_index) {
 226       if (entry->GetURL().host() == request_url.host()) {
 227         first_host_index = index;
 228       } else {
 229         // We have found the possibly phishing url, but we are no longer on the
 230         // host. No reason to look back any further.
 231         break;
 232       }
 233     }
 234   }
 235
 236   // Add features pertaining to how we got to
 237   //   1) The candidate url
 238   //   2) The first url on the same host as the candidate url (assuming that
 239   //      it's different from the candidate url).
 240   if (url_index != -1) {
 241     AddNavigationFeatures(
 242         std::string(), controller, url_index, info->url_redirects, request);
 243   }
 244   if (first_host_index != -1) {
 245     AddNavigationFeatures(features::kHostPrefix,
 246                           controller,
 247                           first_host_index,
 248                           info->host_redirects,
 249                           request);
 250   }
 251
 252   ExtractBrowseInfoFeatures(*info, request);
 253   pending_extractions_[request] = callback;
 254   base::MessageLoop::current()->PostTask(
 255       FROM_HERE,
 256       base::Bind(&BrowserFeatureExtractor::StartExtractFeatures,
 257                  weak_factory_.GetWeakPtr(), request, callback));
 258 }
 259
 260 void BrowserFeatureExtractor::ExtractMalwareFeatures(
 261     BrowseInfo* info,
 262     ClientMalwareRequest* request,
 263     const MalwareDoneCallback& callback) {
 264   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 265   DCHECK_EQ(0U, request->url().find("http:"));
 266   DCHECK(!callback.is_null());
 267
 268   // Grab the IPs because they might go away before we're done
 269   // checking them against the IP blacklist on the IO thread.
 270   scoped_ptr<IPUrlMap> ips(new IPUrlMap);
 271   ips->swap(info->ips);
 272
 273   IPUrlMap* ips_ptr = ips.get();
 274
 275   // The API doesn't take a scoped_ptr because the API gets mocked and we
 276   // cannot mock an API that takes scoped_ptr as arguments.
 277   scoped_ptr<ClientMalwareRequest> req(request);
 278
 279   // IP blacklist lookups have to happen on the IO thread.
 280   BrowserThread::PostTaskAndReply(
 281       BrowserThread::IO,
 282       FROM_HERE,
 283       base::Bind(&FilterBenignIpsOnIOThread,
 284                  host_->database_manager(),
 285                  ips_ptr),
 286       base::Bind(&BrowserFeatureExtractor::FinishExtractMalwareFeatures,
 287                  weak_factory_.GetWeakPtr(),
 288                  base::Passed(&ips), callback, base::Passed(&req)));
 289 }
 290
 291 void BrowserFeatureExtractor::ExtractBrowseInfoFeatures(
 292     const BrowseInfo& info,
 293     ClientPhishingRequest* request) {
 294   if (info.unsafe_resource.get()) {
 295     // A SafeBrowsing interstitial was shown for the current URL.
 296     AddFeature(features::kSafeBrowsingMaliciousUrl +
 297                info.unsafe_resource->url.spec(),
 298                1.0,
 299                request);
 300     AddFeature(features::kSafeBrowsingOriginalUrl +
 301                info.unsafe_resource->original_url.spec(),
 302                1.0,
 303                request);
 304     AddFeature(features::kSafeBrowsingIsSubresource,
 305                info.unsafe_resource->is_subresource ? 1.0 : 0.0,
 306                request);
 307     AddFeature(features::kSafeBrowsingThreatType,
 308                static_cast<double>(info.unsafe_resource->threat_type),
 309                request);
 310   }
 311   if (info.http_status_code != 0) {
 312     AddFeature(features::kHttpStatusCode, info.http_status_code, request);
 313   }
 314 }
 315
 316 void BrowserFeatureExtractor::StartExtractFeatures(
 317     ClientPhishingRequest* request,
 318     const DoneCallback& callback) {
 319   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 320   size_t removed = pending_extractions_.erase(request);
 321   DCHECK_EQ(1U, removed);
 322   HistoryService* history;
 323   if (!request || !request->IsInitialized() || !GetHistoryService(&history)) {
 324     callback.Run(false, request);
 325     return;
 326   }
 327   CancelableRequestProvider::Handle handle = history->QueryURL(
 328       GURL(request->url()),
 329       true /* wants_visits */,
 330       &request_consumer_,
 331       base::Bind(&BrowserFeatureExtractor::QueryUrlHistoryDone,
 332                  base::Unretained(this)));
 333
 334   StorePendingQuery(handle, request, callback);
 335 }
 336
 337 void BrowserFeatureExtractor::QueryUrlHistoryDone(
 338     CancelableRequestProvider::Handle handle,
 339     bool success,
 340     const history::URLRow* row,
 341     history::VisitVector* visits) {
 342   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 343   ClientPhishingRequest* request;
 344   DoneCallback callback;
 345   if (!GetPendingQuery(handle, &request, &callback)) {
 346     DLOG(FATAL) << "No pending history query found";
 347     return;
 348   }
 349   DCHECK(request);
 350   DCHECK(!callback.is_null());
 351   if (!success) {
 352     // URL is not found in the history.  In practice this should not
 353     // happen (unless there is a real error) because we just visited
 354     // that URL.
 355     callback.Run(false, request);
 356     return;
 357   }
 358   AddFeature(features::kUrlHistoryVisitCount,
 359              static_cast<double>(row->visit_count()),
 360              request);
 361
 362   base::Time threshold = base::Time::Now() - base::TimeDelta::FromDays(1);
 363   int num_visits_24h_ago = 0;
 364   int num_visits_typed = 0;
 365   int num_visits_link = 0;
 366   for (history::VisitVector::const_iterator it = visits->begin();
 367        it != visits->end(); ++it) {
 368     if (!content::PageTransitionIsMainFrame(it->transition)) {
 369       continue;
 370     }
 371     if (it->visit_time < threshold) {
 372       ++num_visits_24h_ago;
 373     }
 374     content::PageTransition transition = content::PageTransitionStripQualifier(
 375         it->transition);
 376     if (transition == content::PAGE_TRANSITION_TYPED) {
 377       ++num_visits_typed;
 378     } else if (transition == content::PAGE_TRANSITION_LINK) {
 379       ++num_visits_link;
 380     }
 381   }
 382   AddFeature(features::kUrlHistoryVisitCountMoreThan24hAgo,
 383              static_cast<double>(num_visits_24h_ago),
 384              request);
 385   AddFeature(features::kUrlHistoryTypedCount,
 386              static_cast<double>(num_visits_typed),
 387              request);
 388   AddFeature(features::kUrlHistoryLinkCount,
 389              static_cast<double>(num_visits_link),
 390              request);
 391
 392   // Issue next history lookup for host visits.
 393   HistoryService* history;
 394   if (!GetHistoryService(&history)) {
 395     callback.Run(false, request);
 396     return;
 397   }
 398   CancelableRequestProvider::Handle next_handle =
 399       history->GetVisibleVisitCountToHost(
 400           GURL(request->url()),
 401           &request_consumer_,
 402           base::Bind(&BrowserFeatureExtractor::QueryHttpHostVisitsDone,
 403                      base::Unretained(this)));
 404   StorePendingQuery(next_handle, request, callback);
 405 }
 406
 407 void BrowserFeatureExtractor::QueryHttpHostVisitsDone(
 408     CancelableRequestProvider::Handle handle,
 409     bool success,
 410     int num_visits,
 411     base::Time first_visit) {
 412   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 413   ClientPhishingRequest* request;
 414   DoneCallback callback;
 415   if (!GetPendingQuery(handle, &request, &callback)) {
 416     DLOG(FATAL) << "No pending history query found";
 417     return;
 418   }
 419   DCHECK(request);
 420   DCHECK(!callback.is_null());
 421   if (!success) {
 422     callback.Run(false, request);
 423     return;
 424   }
 425   SetHostVisitsFeatures(num_visits, first_visit, true, request);
 426
 427   // Same lookup but for the HTTPS URL.
 428   HistoryService* history;
 429   if (!GetHistoryService(&history)) {
 430     callback.Run(false, request);
 431     return;
 432   }
 433   std::string https_url = request->url();
 434   CancelableRequestProvider::Handle next_handle =
 435       history->GetVisibleVisitCountToHost(
 436           GURL(https_url.replace(0, 5, "https:")),
 437           &request_consumer_,
 438           base::Bind(&BrowserFeatureExtractor::QueryHttpsHostVisitsDone,
 439                      base::Unretained(this)));
 440   StorePendingQuery(next_handle, request, callback);
 441 }
 442
 443 void BrowserFeatureExtractor::QueryHttpsHostVisitsDone(
 444     CancelableRequestProvider::Handle handle,
 445     bool success,
 446     int num_visits,
 447     base::Time first_visit) {
 448   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 449   ClientPhishingRequest* request;
 450   DoneCallback callback;
 451   if (!GetPendingQuery(handle, &request, &callback)) {
 452     DLOG(FATAL) << "No pending history query found";
 453     return;
 454   }
 455   DCHECK(request);
 456   DCHECK(!callback.is_null());
 457   if (!success) {
 458     callback.Run(false, request);
 459     return;
 460   }
 461   SetHostVisitsFeatures(num_visits, first_visit, false, request);
 462   callback.Run(true, request);  // We're done with all the history lookups.
 463 }
 464
 465 void BrowserFeatureExtractor::SetHostVisitsFeatures(
 466     int num_visits,
 467     base::Time first_visit,
 468     bool is_http_query,
 469     ClientPhishingRequest* request) {
 470   DCHECK(request);
 471   AddFeature(is_http_query ?
 472              features::kHttpHostVisitCount : features::kHttpsHostVisitCount,
 473              static_cast<double>(num_visits),
 474              request);
 475   if (num_visits > 0) {
 476     AddFeature(
 477         is_http_query ?
 478         features::kFirstHttpHostVisitMoreThan24hAgo :
 479         features::kFirstHttpsHostVisitMoreThan24hAgo,
 480         (first_visit < (base::Time::Now() - base::TimeDelta::FromDays(1))) ?
 481         1.0 : 0.0,
 482         request);
 483   }
 484 }
 485
 486 void BrowserFeatureExtractor::StorePendingQuery(
 487     CancelableRequestProvider::Handle handle,
 488     ClientPhishingRequest* request,
 489     const DoneCallback& callback) {
 490   DCHECK_EQ(0U, pending_queries_.count(handle));
 491   pending_queries_[handle] = std::make_pair(request, callback);
 492 }
 493
 494 bool BrowserFeatureExtractor::GetPendingQuery(
 495     CancelableRequestProvider::Handle handle,
 496     ClientPhishingRequest** request,
 497     DoneCallback* callback) {
 498   PendingQueriesMap::iterator it = pending_queries_.find(handle);
 499   DCHECK(it != pending_queries_.end());
 500   if (it != pending_queries_.end()) {
 501     *request = it->second.first;
 502     *callback = it->second.second;
 503     pending_queries_.erase(it);
 504     return true;
 505   }
 506   return false;
 507 }
 508
 509 bool BrowserFeatureExtractor::GetHistoryService(HistoryService** history) {
 510   *history = NULL;
 511   if (tab_ && tab_->GetBrowserContext()) {
 512     Profile* profile = Profile::FromBrowserContext(tab_->GetBrowserContext());
 513     *history = HistoryServiceFactory::GetForProfile(profile,
 514                                                     Profile::EXPLICIT_ACCESS);
 515     if (*history) {
 516       return true;
 517     }
 518   }
 519   VLOG(2) << "Unable to query history.  No history service available.";
 520   return false;
 521 }
 522
 523 void BrowserFeatureExtractor::FinishExtractMalwareFeatures(
 524     scoped_ptr<IPUrlMap> bad_ips,
 525     MalwareDoneCallback callback,
 526     scoped_ptr<ClientMalwareRequest> request) {
 527   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 528   int matched_bad_ips = 0;
 529   for (IPUrlMap::const_iterator it = bad_ips->begin();
 530        it != bad_ips->end(); ++it) {
 531     AddMalwareIpUrlInfo(it->first, it->second, request.get());
 532     ++matched_bad_ips;
 533     // Limit the number of matched bad IPs in one request to control
 534     // the request's size
 535     if (matched_bad_ips >= kMaxMalwareIPPerRequest) {
 536       break;
 537     }
 538   }
 539   callback.Run(true, request.Pass());
 540 }
 541
 542 }  // namespace safe_browsing