chrome/renderer/safe_browsing/phishing_dom_feature_extractor.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "chrome/renderer/safe_browsing/phishing_dom_feature_extractor.h"
   6
   7 #include "base/bind.h"
   8 #include "base/compiler_specific.h"
   9 #include "base/containers/hash_tables.h"
  10 #include "base/location.h"
  11 #include "base/logging.h"
  12 #include "base/metrics/histogram.h"
  13 #include "base/single_thread_task_runner.h"
  14 #include "base/strings/string_util.h"
  15 #include "base/thread_task_runner_handle.h"
  16 #include "base/time/time.h"
  17 #include "chrome/renderer/safe_browsing/feature_extractor_clock.h"
  18 #include "chrome/renderer/safe_browsing/features.h"
  19 #include "content/public/renderer/render_view.h"
  20 #include "net/base/registry_controlled_domains/registry_controlled_domain.h"
  21 #include "third_party/WebKit/public/platform/WebString.h"
  22 #include "third_party/WebKit/public/web/WebElement.h"
  23 #include "third_party/WebKit/public/web/WebElementCollection.h"
  24 #include "third_party/WebKit/public/web/WebLocalFrame.h"
  25 #include "third_party/WebKit/public/web/WebView.h"
  26
  27 namespace safe_browsing {
  28
  29 // This time should be short enough that it doesn't noticeably disrupt the
  30 // user's interaction with the page.
  31 const int PhishingDOMFeatureExtractor::kMaxTimePerChunkMs = 10;
  32
  33 // Experimenting shows that we get a reasonable gain in performance by
  34 // increasing this up to around 10, but there's not much benefit in
  35 // increasing it past that.
  36 const int PhishingDOMFeatureExtractor::kClockCheckGranularity = 10;
  37
  38 // This should be longer than we expect feature extraction to take on any
  39 // actual phishing page.
  40 const int PhishingDOMFeatureExtractor::kMaxTotalTimeMs = 500;
  41
  42 // Intermediate state used for computing features.  See features.h for
  43 // descriptions of the DOM features that are computed.
  44 struct PhishingDOMFeatureExtractor::PageFeatureState {
  45   // Link related features
  46   int external_links;
  47   base::hash_set<std::string> external_domains;
  48   int secure_links;
  49   int total_links;
  50
  51   // Form related features
  52   int num_forms;
  53   int num_text_inputs;
  54   int num_pswd_inputs;
  55   int num_radio_inputs;
  56   int num_check_inputs;
  57   int action_other_domain;
  58   int total_actions;
  59   base::hash_set<std::string> page_action_urls;
  60
  61   // Image related features
  62   int img_other_domain;
  63   int total_imgs;
  64
  65   // How many script tags
  66   int num_script_tags;
  67
  68   // The time at which we started feature extraction for the current page.
  69   base::TimeTicks start_time;
  70
  71   // The number of iterations we've done for the current extraction.
  72   int num_iterations;
  73
  74   explicit PageFeatureState(base::TimeTicks start_time_ticks)
  75       : external_links(0),
  76         secure_links(0),
  77         total_links(0),
  78         num_forms(0),
  79         num_text_inputs(0),
  80         num_pswd_inputs(0),
  81         num_radio_inputs(0),
  82         num_check_inputs(0),
  83         action_other_domain(0),
  84         total_actions(0),
  85         img_other_domain(0),
  86         total_imgs(0),
  87         num_script_tags(0),
  88         start_time(start_time_ticks),
  89         num_iterations(0) {}
  90
  91   ~PageFeatureState() {}
  92 };
  93
  94 // Per-frame state
  95 struct PhishingDOMFeatureExtractor::FrameData {
  96   // This is our reference to document.all, which is an iterator over all
  97   // of the elements in the document.  It keeps track of our current position.
  98   blink::WebElementCollection elements;
  99   // The domain of the document URL, stored here so that we don't need to
 100   // recompute it every time it's needed.
 101   std::string domain;
 102 };
 103
 104 PhishingDOMFeatureExtractor::PhishingDOMFeatureExtractor(
 105     FeatureExtractorClock* clock)
 106     : clock_(clock), weak_factory_(this) {
 107   Clear();
 108 }
 109
 110 PhishingDOMFeatureExtractor::~PhishingDOMFeatureExtractor() {
 111   // The RenderView should have called CancelPendingExtraction() before
 112   // we are destroyed.
 113   CheckNoPendingExtraction();
 114 }
 115
 116 void PhishingDOMFeatureExtractor::ExtractFeatures(
 117     blink::WebDocument document,
 118     FeatureMap* features,
 119     const DoneCallback& done_callback) {
 120   // The RenderView should have called CancelPendingExtraction() before
 121   // starting a new extraction, so DCHECK this.
 122   CheckNoPendingExtraction();
 123   // However, in an opt build, we will go ahead and clean up the pending
 124   // extraction so that we can start in a known state.
 125   CancelPendingExtraction();
 126
 127   features_ = features;
 128   done_callback_ = done_callback;
 129
 130   page_feature_state_.reset(new PageFeatureState(clock_->Now()));
 131   cur_document_ = document;
 132
 133   base::ThreadTaskRunnerHandle::Get()->PostTask(
 134       FROM_HERE,
 135       base::Bind(&PhishingDOMFeatureExtractor::ExtractFeaturesWithTimeout,
 136                  weak_factory_.GetWeakPtr()));
 137 }
 138
 139 void PhishingDOMFeatureExtractor::CancelPendingExtraction() {
 140   // Cancel any pending callbacks, and clear our state.
 141   weak_factory_.InvalidateWeakPtrs();
 142   Clear();
 143 }
 144
 145 void PhishingDOMFeatureExtractor::ExtractFeaturesWithTimeout() {
 146   DCHECK(page_feature_state_.get());
 147   ++page_feature_state_->num_iterations;
 148   base::TimeTicks current_chunk_start_time = clock_->Now();
 149
 150   if (cur_document_.isNull()) {
 151     // This will only happen if we weren't able to get the document for the
 152     // main frame.  We'll treat this as an extraction failure.
 153     RunCallback(false);
 154     return;
 155   }
 156
 157   int num_elements = 0;
 158   for (; !cur_document_.isNull(); cur_document_ = GetNextDocument()) {
 159     blink::WebElement cur_element;
 160     if (cur_frame_data_.get()) {
 161       // We're resuming traversal of a frame, so just advance to the next
 162       // element.
 163       cur_element = cur_frame_data_->elements.nextItem();
 164       // When we resume the traversal, the first call to nextItem() potentially
 165       // has to walk through the document again from the beginning, if it was
 166       // modified between our chunks of work.  Log how long this takes, so we
 167       // can tell if it's too slow.
 168       UMA_HISTOGRAM_TIMES("SBClientPhishing.DOMFeatureResumeTime",
 169                           clock_->Now() - current_chunk_start_time);
 170     } else {
 171       // We just moved to a new frame, so update our frame state
 172       // and advance to the first element.
 173       ResetFrameData();
 174       cur_element = cur_frame_data_->elements.firstItem();
 175     }
 176
 177     for (; !cur_element.isNull();
 178          cur_element = cur_frame_data_->elements.nextItem()) {
 179       if (cur_element.hasHTMLTagName("a")) {
 180         HandleLink(cur_element);
 181       } else if (cur_element.hasHTMLTagName("form")) {
 182         HandleForm(cur_element);
 183       } else if (cur_element.hasHTMLTagName("img")) {
 184         HandleImage(cur_element);
 185       } else if (cur_element.hasHTMLTagName("input")) {
 186         HandleInput(cur_element);
 187       } else if (cur_element.hasHTMLTagName("script")) {
 188         HandleScript(cur_element);
 189       }
 190
 191       if (++num_elements >= kClockCheckGranularity) {
 192         num_elements = 0;
 193         base::TimeTicks now = clock_->Now();
 194         if (now - page_feature_state_->start_time >=
 195             base::TimeDelta::FromMilliseconds(kMaxTotalTimeMs)) {
 196           DLOG(ERROR) << "Feature extraction took too long, giving up";
 197           // We expect this to happen infrequently, so record when it does.
 198           UMA_HISTOGRAM_COUNTS("SBClientPhishing.DOMFeatureTimeout", 1);
 199           RunCallback(false);
 200           return;
 201         }
 202         base::TimeDelta chunk_elapsed = now - current_chunk_start_time;
 203         if (chunk_elapsed >=
 204             base::TimeDelta::FromMilliseconds(kMaxTimePerChunkMs)) {
 205           // The time limit for the current chunk is up, so post a task to
 206           // continue extraction.
 207           //
 208           // Record how much time we actually spent on the chunk. If this is
 209           // much higher than kMaxTimePerChunkMs, we may need to adjust the
 210           // clock granularity.
 211           UMA_HISTOGRAM_TIMES("SBClientPhishing.DOMFeatureChunkTime",
 212                               chunk_elapsed);
 213           base::ThreadTaskRunnerHandle::Get()->PostTask(
 214               FROM_HERE,
 215               base::Bind(
 216                   &PhishingDOMFeatureExtractor::ExtractFeaturesWithTimeout,
 217                   weak_factory_.GetWeakPtr()));
 218           return;
 219         }
 220         // Otherwise, continue.
 221       }
 222     }
 223
 224     // We're done with this frame, recalculate the FrameData when we
 225     // advance to the next frame.
 226     cur_frame_data_.reset();
 227   }
 228
 229   InsertFeatures();
 230   RunCallback(true);
 231 }
 232
 233 void PhishingDOMFeatureExtractor::HandleLink(
 234     const blink::WebElement& element) {
 235   // Count the number of times we link to a different host.
 236   if (!element.hasAttribute("href")) {
 237     DVLOG(1) << "Skipping anchor tag with no href";
 238     return;
 239   }
 240
 241   // Retrieve the link and resolve the link in case it's relative.
 242   blink::WebURL full_url = element.document().completeURL(
 243       element.getAttribute("href"));
 244
 245   std::string domain;
 246   bool is_external = IsExternalDomain(full_url, &domain);
 247   if (domain.empty()) {
 248     DVLOG(1) << "Could not extract domain from link: " << full_url;
 249     return;
 250   }
 251
 252   if (is_external) {
 253     ++page_feature_state_->external_links;
 254
 255     // Record each unique domain that we link to.
 256     page_feature_state_->external_domains.insert(domain);
 257   }
 258
 259   // Check how many are https links.
 260   if (GURL(full_url).SchemeIs("https")) {
 261     ++page_feature_state_->secure_links;
 262   }
 263
 264   ++page_feature_state_->total_links;
 265 }
 266
 267 void PhishingDOMFeatureExtractor::HandleForm(
 268     const blink::WebElement& element) {
 269   // Increment the number of forms on this page.
 270   ++page_feature_state_->num_forms;
 271
 272   // Record whether the action points to a different domain.
 273   if (!element.hasAttribute("action")) {
 274     return;
 275   }
 276
 277   blink::WebURL full_url = element.document().completeURL(
 278       element.getAttribute("action"));
 279
 280   page_feature_state_->page_action_urls.insert(full_url.string().utf8());
 281
 282   std::string domain;
 283   bool is_external = IsExternalDomain(full_url, &domain);
 284   if (domain.empty()) {
 285     DVLOG(1) << "Could not extract domain from form action: " << full_url;
 286     return;
 287   }
 288
 289   if (is_external) {
 290     ++page_feature_state_->action_other_domain;
 291   }
 292   ++page_feature_state_->total_actions;
 293 }
 294
 295 void PhishingDOMFeatureExtractor::HandleImage(
 296     const blink::WebElement& element) {
 297   if (!element.hasAttribute("src")) {
 298     DVLOG(1) << "Skipping img tag with no src";
 299   }
 300
 301   // Record whether the image points to a different domain.
 302   blink::WebURL full_url = element.document().completeURL(
 303       element.getAttribute("src"));
 304   std::string domain;
 305   bool is_external = IsExternalDomain(full_url, &domain);
 306   if (domain.empty()) {
 307     DVLOG(1) << "Could not extract domain from image src: " << full_url;
 308     return;
 309   }
 310
 311   if (is_external) {
 312     ++page_feature_state_->img_other_domain;
 313   }
 314   ++page_feature_state_->total_imgs;
 315 }
 316
 317 void PhishingDOMFeatureExtractor::HandleInput(
 318     const blink::WebElement& element) {
 319   // The HTML spec says that if the type is unspecified, it defaults to text.
 320   // In addition, any unrecognized type will be treated as a text input.
 321   //
 322   // Note that we use the attribute value rather than
 323   // WebFormControlElement::formControlType() for consistency with the
 324   // way the phishing classification model is created.
 325   std::string type = base::ToLowerASCII(element.getAttribute("type").utf8());
 326   if (type == "password") {
 327     ++page_feature_state_->num_pswd_inputs;
 328   } else if (type == "radio") {
 329     ++page_feature_state_->num_radio_inputs;
 330   } else if (type == "checkbox") {
 331     ++page_feature_state_->num_check_inputs;
 332   } else if (type != "submit" && type != "reset" && type != "file" &&
 333              type != "hidden" && type != "image" && type != "button") {
 334     // Note that there are a number of new input types in HTML5 that are not
 335     // handled above.  For now, we will consider these as text inputs since
 336     // they could be used to capture user input.
 337     ++page_feature_state_->num_text_inputs;
 338   }
 339 }
 340
 341 void PhishingDOMFeatureExtractor::HandleScript(
 342     const blink::WebElement& element) {
 343   ++page_feature_state_->num_script_tags;
 344 }
 345
 346 void PhishingDOMFeatureExtractor::CheckNoPendingExtraction() {
 347   DCHECK(done_callback_.is_null());
 348   DCHECK(!cur_frame_data_.get());
 349   DCHECK(cur_document_.isNull());
 350   if (!done_callback_.is_null() || cur_frame_data_.get() ||
 351       !cur_document_.isNull()) {
 352     LOG(ERROR) << "Extraction in progress, missing call to "
 353                << "CancelPendingExtraction";
 354   }
 355 }
 356
 357 void PhishingDOMFeatureExtractor::RunCallback(bool success) {
 358   // Record some timing stats that we can use to evaluate feature extraction
 359   // performance.  These include both successful and failed extractions.
 360   DCHECK(page_feature_state_.get());
 361   UMA_HISTOGRAM_COUNTS("SBClientPhishing.DOMFeatureIterations",
 362                        page_feature_state_->num_iterations);
 363   UMA_HISTOGRAM_TIMES("SBClientPhishing.DOMFeatureTotalTime",
 364                       clock_->Now() - page_feature_state_->start_time);
 365
 366   DCHECK(!done_callback_.is_null());
 367   done_callback_.Run(success);
 368   Clear();
 369 }
 370
 371 void PhishingDOMFeatureExtractor::Clear() {
 372   features_ = NULL;
 373   done_callback_.Reset();
 374   cur_frame_data_.reset(NULL);
 375   cur_document_.reset();
 376 }
 377
 378 void PhishingDOMFeatureExtractor::ResetFrameData() {
 379   DCHECK(!cur_document_.isNull());
 380   DCHECK(!cur_frame_data_.get());
 381
 382   cur_frame_data_.reset(new FrameData());
 383   cur_frame_data_->elements = cur_document_.all();
 384   cur_frame_data_->domain =
 385       net::registry_controlled_domains::GetDomainAndRegistry(
 386           cur_document_.url(),
 387           net::registry_controlled_domains::EXCLUDE_PRIVATE_REGISTRIES);
 388 }
 389
 390 blink::WebDocument PhishingDOMFeatureExtractor::GetNextDocument() {
 391   DCHECK(!cur_document_.isNull());
 392   blink::WebFrame* frame = cur_document_.frame();
 393   // Advance to the next frame that contains a document, with no wrapping.
 394   if (frame) {
 395     for (frame = frame->traverseNext(false); frame;
 396          frame = frame->traverseNext(false)) {
 397       if (!frame->document().isNull()) {
 398         return frame->document();
 399       }
 400     }
 401   } else {
 402     // Keep track of how often frame traversal got "stuck" due to the
 403     // current subdocument getting removed from the frame tree.
 404     UMA_HISTOGRAM_COUNTS("SBClientPhishing.DOMFeatureFrameRemoved", 1);
 405   }
 406   return blink::WebDocument();
 407 }
 408
 409 bool PhishingDOMFeatureExtractor::IsExternalDomain(const GURL& url,
 410                                                    std::string* domain) const {
 411   DCHECK(domain);
 412   DCHECK(cur_frame_data_.get());
 413
 414   if (cur_frame_data_->domain.empty()) {
 415     return false;
 416   }
 417
 418   // TODO(bryner): Ensure that the url encoding is consistent with the features
 419   // in the model.
 420   if (url.HostIsIPAddress()) {
 421     domain->assign(url.host());
 422   } else {
 423     domain->assign(net::registry_controlled_domains::GetDomainAndRegistry(
 424         url, net::registry_controlled_domains::EXCLUDE_PRIVATE_REGISTRIES));
 425   }
 426
 427   return !domain->empty() && *domain != cur_frame_data_->domain;
 428 }
 429
 430 void PhishingDOMFeatureExtractor::InsertFeatures() {
 431   DCHECK(page_feature_state_.get());
 432
 433   if (page_feature_state_->total_links > 0) {
 434     // Add a feature for the fraction of times the page links to an external
 435     // domain vs. an internal domain.
 436     double link_freq = static_cast<double>(
 437         page_feature_state_->external_links) /
 438         page_feature_state_->total_links;
 439     features_->AddRealFeature(features::kPageExternalLinksFreq, link_freq);
 440
 441     // Add a feature for each unique domain that we're linking to
 442     for (const auto& domain : page_feature_state_->external_domains) {
 443       features_->AddBooleanFeature(features::kPageLinkDomain + domain);
 444     }
 445
 446     // Fraction of links that use https.
 447     double secure_freq = static_cast<double>(
 448         page_feature_state_->secure_links) / page_feature_state_->total_links;
 449     features_->AddRealFeature(features::kPageSecureLinksFreq, secure_freq);
 450   }
 451
 452   // Record whether forms appear and whether various form elements appear.
 453   if (page_feature_state_->num_forms > 0) {
 454     features_->AddBooleanFeature(features::kPageHasForms);
 455   }
 456   if (page_feature_state_->num_text_inputs > 0) {
 457     features_->AddBooleanFeature(features::kPageHasTextInputs);
 458   }
 459   if (page_feature_state_->num_pswd_inputs > 0) {
 460     features_->AddBooleanFeature(features::kPageHasPswdInputs);
 461   }
 462   if (page_feature_state_->num_radio_inputs > 0) {
 463     features_->AddBooleanFeature(features::kPageHasRadioInputs);
 464   }
 465   if (page_feature_state_->num_check_inputs > 0) {
 466     features_->AddBooleanFeature(features::kPageHasCheckInputs);
 467   }
 468
 469   // Record fraction of form actions that point to a different domain.
 470   if (page_feature_state_->total_actions > 0) {
 471     double action_freq = static_cast<double>(
 472         page_feature_state_->action_other_domain) /
 473         page_feature_state_->total_actions;
 474     features_->AddRealFeature(features::kPageActionOtherDomainFreq,
 475                               action_freq);
 476   }
 477
 478   // Add a feature for each unique external action url.
 479   for (const auto& url : page_feature_state_->page_action_urls) {
 480     features_->AddBooleanFeature(features::kPageActionURL + url);
 481   }
 482
 483   // Record how many image src attributes point to a different domain.
 484   if (page_feature_state_->total_imgs > 0) {
 485     double img_freq = static_cast<double>(
 486         page_feature_state_->img_other_domain) /
 487         page_feature_state_->total_imgs;
 488     features_->AddRealFeature(features::kPageImgOtherDomainFreq, img_freq);
 489   }
 490
 491   // Record number of script tags (discretized for numerical stability.)
 492   if (page_feature_state_->num_script_tags > 1) {
 493     features_->AddBooleanFeature(features::kPageNumScriptTagsGTOne);
 494     if (page_feature_state_->num_script_tags > 6) {
 495       features_->AddBooleanFeature(features::kPageNumScriptTagsGTSix);
 496     }
 497   }
 498 }
 499
 500 }  // namespace safe_browsing