chrome/browser/safe_browsing/download_protection_service.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "chrome/browser/safe_browsing/download_protection_service.h"
   6
   7 #include "base/bind.h"
   8 #include "base/compiler_specific.h"
   9 #include "base/format_macros.h"
  10 #include "base/memory/scoped_ptr.h"
  11 #include "base/memory/weak_ptr.h"
  12 #include "base/metrics/histogram.h"
  13 #include "base/metrics/sparse_histogram.h"
  14 #include "base/sequenced_task_runner_helpers.h"
  15 #include "base/stl_util.h"
  16 #include "base/strings/string_number_conversions.h"
  17 #include "base/strings/string_util.h"
  18 #include "base/strings/stringprintf.h"
  19 #include "base/task/cancelable_task_tracker.h"
  20 #include "base/threading/sequenced_worker_pool.h"
  21 #include "base/time/time.h"
  22 #include "chrome/browser/history/history_service.h"
  23 #include "chrome/browser/history/history_service_factory.h"
  24 #include "chrome/browser/safe_browsing/binary_feature_extractor.h"
  25 #include "chrome/browser/safe_browsing/download_feedback_service.h"
  26 #include "chrome/browser/safe_browsing/safe_browsing_service.h"
  27 #include "chrome/browser/safe_browsing/sandboxed_zip_analyzer.h"
  28 #include "chrome/browser/ui/browser.h"
  29 #include "chrome/browser/ui/browser_list.h"
  30 #include "chrome/common/safe_browsing/csd.pb.h"
  31 #include "chrome/common/safe_browsing/download_protection_util.h"
  32 #include "chrome/common/safe_browsing/zip_analyzer.h"
  33 #include "chrome/common/url_constants.h"
  34 #include "content/public/browser/browser_thread.h"
  35 #include "content/public/browser/download_item.h"
  36 #include "content/public/browser/page_navigator.h"
  37 #include "google_apis/google_api_keys.h"
  38 #include "net/base/escape.h"
  39 #include "net/base/load_flags.h"
  40 #include "net/cert/x509_cert_types.h"
  41 #include "net/cert/x509_certificate.h"
  42 #include "net/http/http_status_code.h"
  43 #include "net/url_request/url_fetcher.h"
  44 #include "net/url_request/url_fetcher_delegate.h"
  45 #include "net/url_request/url_request_context_getter.h"
  46 #include "net/url_request/url_request_status.h"
  47
  48 using content::BrowserThread;
  49
  50 namespace {
  51 static const int64 kDownloadRequestTimeoutMs = 7000;
  52 }  // namespace
  53
  54 namespace safe_browsing {
  55
  56 const char DownloadProtectionService::kDownloadRequestUrl[] =
  57     "https://sb-ssl.google.com/safebrowsing/clientreport/download";
  58
  59 namespace {
  60 // List of extensions for which we track some UMA stats.
  61 enum MaliciousExtensionType {
  62   EXTENSION_EXE,
  63   EXTENSION_MSI,
  64   EXTENSION_CAB,
  65   EXTENSION_SYS,
  66   EXTENSION_SCR,
  67   EXTENSION_DRV,
  68   EXTENSION_BAT,
  69   EXTENSION_ZIP,
  70   EXTENSION_RAR,
  71   EXTENSION_DLL,
  72   EXTENSION_PIF,
  73   EXTENSION_COM,
  74   EXTENSION_JAR,
  75   EXTENSION_CLASS,
  76   EXTENSION_PDF,
  77   EXTENSION_VB,
  78   EXTENSION_REG,
  79   EXTENSION_GRP,
  80   EXTENSION_OTHER,  // Groups all other extensions into one bucket.
  81   EXTENSION_CRX,
  82   EXTENSION_APK,
  83   EXTENSION_DMG,
  84   EXTENSION_PKG,
  85   EXTENSION_TORRENT,
  86   EXTENSION_MAX,
  87 };
  88
  89 MaliciousExtensionType GetExtensionType(const base::FilePath& f) {
  90   if (f.MatchesExtension(FILE_PATH_LITERAL(".exe"))) return EXTENSION_EXE;
  91   if (f.MatchesExtension(FILE_PATH_LITERAL(".msi"))) return EXTENSION_MSI;
  92   if (f.MatchesExtension(FILE_PATH_LITERAL(".cab"))) return EXTENSION_CAB;
  93   if (f.MatchesExtension(FILE_PATH_LITERAL(".sys"))) return EXTENSION_SYS;
  94   if (f.MatchesExtension(FILE_PATH_LITERAL(".scr"))) return EXTENSION_SCR;
  95   if (f.MatchesExtension(FILE_PATH_LITERAL(".drv"))) return EXTENSION_DRV;
  96   if (f.MatchesExtension(FILE_PATH_LITERAL(".bat"))) return EXTENSION_BAT;
  97   if (f.MatchesExtension(FILE_PATH_LITERAL(".zip"))) return EXTENSION_ZIP;
  98   if (f.MatchesExtension(FILE_PATH_LITERAL(".rar"))) return EXTENSION_RAR;
  99   if (f.MatchesExtension(FILE_PATH_LITERAL(".dll"))) return EXTENSION_DLL;
 100   if (f.MatchesExtension(FILE_PATH_LITERAL(".pif"))) return EXTENSION_PIF;
 101   if (f.MatchesExtension(FILE_PATH_LITERAL(".com"))) return EXTENSION_COM;
 102   if (f.MatchesExtension(FILE_PATH_LITERAL(".jar"))) return EXTENSION_JAR;
 103   if (f.MatchesExtension(FILE_PATH_LITERAL(".class"))) return EXTENSION_CLASS;
 104   if (f.MatchesExtension(FILE_PATH_LITERAL(".pdf"))) return EXTENSION_PDF;
 105   if (f.MatchesExtension(FILE_PATH_LITERAL(".vb"))) return EXTENSION_VB;
 106   if (f.MatchesExtension(FILE_PATH_LITERAL(".reg"))) return EXTENSION_REG;
 107   if (f.MatchesExtension(FILE_PATH_LITERAL(".grp"))) return EXTENSION_GRP;
 108   if (f.MatchesExtension(FILE_PATH_LITERAL(".crx"))) return EXTENSION_CRX;
 109   if (f.MatchesExtension(FILE_PATH_LITERAL(".apk"))) return EXTENSION_APK;
 110   if (f.MatchesExtension(FILE_PATH_LITERAL(".dmg"))) return EXTENSION_DMG;
 111   if (f.MatchesExtension(FILE_PATH_LITERAL(".pkg"))) return EXTENSION_PKG;
 112   if (f.MatchesExtension(FILE_PATH_LITERAL(".torrent")))
 113     return EXTENSION_TORRENT;
 114   return EXTENSION_OTHER;
 115 }
 116
 117 void RecordFileExtensionType(const base::FilePath& file) {
 118   UMA_HISTOGRAM_ENUMERATION("SBClientDownload.DownloadExtensions",
 119                             GetExtensionType(file),
 120                             EXTENSION_MAX);
 121 }
 122
 123 // Enumerate for histogramming purposes.
 124 // DO NOT CHANGE THE ORDERING OF THESE VALUES (different histogram data will
 125 // be mixed together based on their values).
 126 enum SBStatsType {
 127   DOWNLOAD_URL_CHECKS_TOTAL,
 128   DOWNLOAD_URL_CHECKS_CANCELED,
 129   DOWNLOAD_URL_CHECKS_MALWARE,
 130
 131   DOWNLOAD_HASH_CHECKS_TOTAL,
 132   DOWNLOAD_HASH_CHECKS_MALWARE,
 133
 134   // Memory space for histograms is determined by the max.
 135   // ALWAYS ADD NEW VALUES BEFORE THIS ONE.
 136   DOWNLOAD_CHECKS_MAX
 137 };
 138 }  // namespace
 139
 140 // Parent SafeBrowsing::Client class used to lookup the bad binary
 141 // URL and digest list.  There are two sub-classes (one for each list).
 142 class DownloadSBClient
 143     : public SafeBrowsingDatabaseManager::Client,
 144       public base::RefCountedThreadSafe<DownloadSBClient> {
 145  public:
 146   DownloadSBClient(
 147       const content::DownloadItem& item,
 148       const DownloadProtectionService::CheckDownloadCallback& callback,
 149       const scoped_refptr<SafeBrowsingUIManager>& ui_manager,
 150       SBStatsType total_type,
 151       SBStatsType dangerous_type)
 152       : sha256_hash_(item.GetHash()),
 153         url_chain_(item.GetUrlChain()),
 154         referrer_url_(item.GetReferrerUrl()),
 155         callback_(callback),
 156         ui_manager_(ui_manager),
 157         start_time_(base::TimeTicks::Now()),
 158         total_type_(total_type),
 159         dangerous_type_(dangerous_type) {}
 160
 161   virtual void StartCheck() = 0;
 162   virtual bool IsDangerous(SBThreatType threat_type) const = 0;
 163
 164  protected:
 165   friend class base::RefCountedThreadSafe<DownloadSBClient>;
 166   virtual ~DownloadSBClient() {}
 167
 168   void CheckDone(SBThreatType threat_type) {
 169     DownloadProtectionService::DownloadCheckResult result =
 170         IsDangerous(threat_type) ?
 171         DownloadProtectionService::DANGEROUS :
 172         DownloadProtectionService::SAFE;
 173     BrowserThread::PostTask(BrowserThread::UI,
 174                             FROM_HERE,
 175                             base::Bind(callback_, result));
 176     UpdateDownloadCheckStats(total_type_);
 177     if (threat_type != SB_THREAT_TYPE_SAFE) {
 178       UpdateDownloadCheckStats(dangerous_type_);
 179       BrowserThread::PostTask(
 180           BrowserThread::UI,
 181           FROM_HERE,
 182           base::Bind(&DownloadSBClient::ReportMalware,
 183                      this, threat_type));
 184     }
 185   }
 186
 187   void ReportMalware(SBThreatType threat_type) {
 188     std::string post_data;
 189     if (!sha256_hash_.empty())
 190       post_data += base::HexEncode(sha256_hash_.data(),
 191                                    sha256_hash_.size()) + "\n";
 192     for (size_t i = 0; i < url_chain_.size(); ++i) {
 193       post_data += url_chain_[i].spec() + "\n";
 194     }
 195     ui_manager_->ReportSafeBrowsingHit(
 196         url_chain_.back(),  // malicious_url
 197         url_chain_.front(), // page_url
 198         referrer_url_,
 199         true,  // is_subresource
 200         threat_type,
 201         post_data);
 202   }
 203
 204   void UpdateDownloadCheckStats(SBStatsType stat_type) {
 205     UMA_HISTOGRAM_ENUMERATION("SB2.DownloadChecks",
 206                               stat_type,
 207                               DOWNLOAD_CHECKS_MAX);
 208   }
 209
 210   std::string sha256_hash_;
 211   std::vector<GURL> url_chain_;
 212   GURL referrer_url_;
 213   DownloadProtectionService::CheckDownloadCallback callback_;
 214   scoped_refptr<SafeBrowsingUIManager> ui_manager_;
 215   base::TimeTicks start_time_;
 216
 217  private:
 218   const SBStatsType total_type_;
 219   const SBStatsType dangerous_type_;
 220
 221   DISALLOW_COPY_AND_ASSIGN(DownloadSBClient);
 222 };
 223
 224 class DownloadUrlSBClient : public DownloadSBClient {
 225  public:
 226   DownloadUrlSBClient(
 227       const content::DownloadItem& item,
 228       const DownloadProtectionService::CheckDownloadCallback& callback,
 229       const scoped_refptr<SafeBrowsingUIManager>& ui_manager,
 230       const scoped_refptr<SafeBrowsingDatabaseManager>& database_manager)
 231       : DownloadSBClient(item, callback, ui_manager,
 232                          DOWNLOAD_URL_CHECKS_TOTAL,
 233                          DOWNLOAD_URL_CHECKS_MALWARE),
 234         database_manager_(database_manager) { }
 235
 236   virtual void StartCheck() OVERRIDE {
 237     DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 238     if (!database_manager_.get() ||
 239         database_manager_->CheckDownloadUrl(url_chain_, this)) {
 240       CheckDone(SB_THREAT_TYPE_SAFE);
 241     } else {
 242       AddRef();  // SafeBrowsingService takes a pointer not a scoped_refptr.
 243     }
 244   }
 245
 246   virtual bool IsDangerous(SBThreatType threat_type) const OVERRIDE {
 247     return threat_type == SB_THREAT_TYPE_BINARY_MALWARE_URL;
 248   }
 249
 250   virtual void OnCheckDownloadUrlResult(const std::vector<GURL>& url_chain,
 251                                         SBThreatType threat_type) OVERRIDE {
 252     CheckDone(threat_type);
 253     UMA_HISTOGRAM_TIMES("SB2.DownloadUrlCheckDuration",
 254                         base::TimeTicks::Now() - start_time_);
 255     Release();
 256   }
 257
 258  protected:
 259   virtual ~DownloadUrlSBClient() {}
 260
 261  private:
 262   scoped_refptr<SafeBrowsingDatabaseManager> database_manager_;
 263
 264   DISALLOW_COPY_AND_ASSIGN(DownloadUrlSBClient);
 265 };
 266
 267 class DownloadProtectionService::CheckClientDownloadRequest
 268     : public base::RefCountedThreadSafe<
 269           DownloadProtectionService::CheckClientDownloadRequest,
 270           BrowserThread::DeleteOnUIThread>,
 271       public net::URLFetcherDelegate,
 272       public content::DownloadItem::Observer {
 273  public:
 274   CheckClientDownloadRequest(
 275       content::DownloadItem* item,
 276       const CheckDownloadCallback& callback,
 277       DownloadProtectionService* service,
 278       const scoped_refptr<SafeBrowsingDatabaseManager>& database_manager,
 279       BinaryFeatureExtractor* binary_feature_extractor)
 280       : item_(item),
 281         url_chain_(item->GetUrlChain()),
 282         referrer_url_(item->GetReferrerUrl()),
 283         tab_url_(item->GetTabUrl()),
 284         tab_referrer_url_(item->GetTabReferrerUrl()),
 285         zipped_executable_(false),
 286         callback_(callback),
 287         service_(service),
 288         binary_feature_extractor_(binary_feature_extractor),
 289         database_manager_(database_manager),
 290         pingback_enabled_(service_->enabled()),
 291         finished_(false),
 292         type_(ClientDownloadRequest::WIN_EXECUTABLE),
 293         weakptr_factory_(this),
 294         start_time_(base::TimeTicks::Now()) {
 295     DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 296     item_->AddObserver(this);
 297   }
 298
 299   void Start() {
 300     VLOG(2) << "Starting SafeBrowsing download check for: "
 301             << item_->DebugString(true);
 302     DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 303     // TODO(noelutz): implement some cache to make sure we don't issue the same
 304     // request over and over again if a user downloads the same binary multiple
 305     // times.
 306     DownloadCheckResultReason reason = REASON_MAX;
 307     if (!IsSupportedDownload(
 308         *item_, item_->GetTargetFilePath(), &reason, &type_)) {
 309       switch (reason) {
 310         case REASON_EMPTY_URL_CHAIN:
 311         case REASON_INVALID_URL:
 312           PostFinishTask(SAFE, reason);
 313           return;
 314
 315         case REASON_NOT_BINARY_FILE:
 316           RecordFileExtensionType(item_->GetTargetFilePath());
 317           PostFinishTask(SAFE, reason);
 318           return;
 319
 320         default:
 321           // We only expect the reasons explicitly handled above.
 322           NOTREACHED();
 323       }
 324     }
 325     RecordFileExtensionType(item_->GetTargetFilePath());
 326
 327     // Compute features from the file contents. Note that we record histograms
 328     // based on the result, so this runs regardless of whether the pingbacks
 329     // are enabled.
 330     if (item_->GetTargetFilePath().MatchesExtension(
 331         FILE_PATH_LITERAL(".zip"))) {
 332       StartExtractZipFeatures();
 333     } else {
 334       DCHECK(!download_protection_util::IsArchiveFile(
 335           item_->GetTargetFilePath()));
 336       StartExtractFileFeatures();
 337     }
 338   }
 339
 340   // Start a timeout to cancel the request if it takes too long.
 341   // This should only be called after we have finished accessing the file.
 342   void StartTimeout() {
 343     DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 344     if (!service_) {
 345       // Request has already been cancelled.
 346       return;
 347     }
 348     timeout_start_time_ = base::TimeTicks::Now();
 349     BrowserThread::PostDelayedTask(
 350         BrowserThread::UI,
 351         FROM_HERE,
 352         base::Bind(&CheckClientDownloadRequest::Cancel,
 353                    weakptr_factory_.GetWeakPtr()),
 354         base::TimeDelta::FromMilliseconds(
 355             service_->download_request_timeout_ms()));
 356   }
 357
 358   // Canceling a request will cause us to always report the result as SAFE
 359   // unless a pending request is about to call FinishRequest.
 360   void Cancel() {
 361     DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 362     if (fetcher_.get()) {
 363       // The DownloadProtectionService is going to release its reference, so we
 364       // might be destroyed before the URLFetcher completes.  Cancel the
 365       // fetcher so it does not try to invoke OnURLFetchComplete.
 366       fetcher_.reset();
 367     }
 368     // Note: If there is no fetcher, then some callback is still holding a
 369     // reference to this object.  We'll eventually wind up in some method on
 370     // the UI thread that will call FinishRequest() again.  If FinishRequest()
 371     // is called a second time, it will be a no-op.
 372     FinishRequest(SAFE, REASON_REQUEST_CANCELED);
 373     // Calling FinishRequest might delete this object, we may be deleted by
 374     // this point.
 375   }
 376
 377   // content::DownloadItem::Observer implementation.
 378   virtual void OnDownloadDestroyed(content::DownloadItem* download) OVERRIDE {
 379     Cancel();
 380     DCHECK(item_ == NULL);
 381   }
 382
 383   // From the net::URLFetcherDelegate interface.
 384   virtual void OnURLFetchComplete(const net::URLFetcher* source) OVERRIDE {
 385     DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 386     DCHECK_EQ(source, fetcher_.get());
 387     VLOG(2) << "Received a response for URL: "
 388             << item_->GetUrlChain().back() << ": success="
 389             << source->GetStatus().is_success() << " response_code="
 390             << source->GetResponseCode();
 391     if (source->GetStatus().is_success()) {
 392       UMA_HISTOGRAM_SPARSE_SLOWLY(
 393           "SBClientDownload.DownloadRequestResponseCode",
 394           source->GetResponseCode());
 395     }
 396     UMA_HISTOGRAM_SPARSE_SLOWLY(
 397         "SBClientDownload.DownloadRequestNetError",
 398         -source->GetStatus().error());
 399     DownloadCheckResultReason reason = REASON_SERVER_PING_FAILED;
 400     DownloadCheckResult result = SAFE;
 401     if (source->GetStatus().is_success() &&
 402         net::HTTP_OK == source->GetResponseCode()) {
 403       ClientDownloadResponse response;
 404       std::string data;
 405       bool got_data = source->GetResponseAsString(&data);
 406       DCHECK(got_data);
 407       if (!response.ParseFromString(data)) {
 408         reason = REASON_INVALID_RESPONSE_PROTO;
 409       } else if (response.verdict() == ClientDownloadResponse::SAFE) {
 410         reason = REASON_DOWNLOAD_SAFE;
 411       } else if (service_ && !service_->IsSupportedDownload(
 412           *item_, item_->GetTargetFilePath())) {
 413         // The client of the download protection service assumes that we don't
 414         // support this download so we cannot return any other verdict than
 415         // SAFE even if the server says it's dangerous to download this file.
 416         // Note: if service_ is NULL we already cancelled the request and
 417         // returned SAFE.
 418         reason = REASON_DOWNLOAD_NOT_SUPPORTED;
 419       } else if (response.verdict() == ClientDownloadResponse::DANGEROUS) {
 420         reason = REASON_DOWNLOAD_DANGEROUS;
 421         result = DANGEROUS;
 422       } else if (response.verdict() == ClientDownloadResponse::UNCOMMON) {
 423         reason = REASON_DOWNLOAD_UNCOMMON;
 424         result = UNCOMMON;
 425       } else if (response.verdict() == ClientDownloadResponse::DANGEROUS_HOST) {
 426         reason = REASON_DOWNLOAD_DANGEROUS_HOST;
 427         result = DANGEROUS_HOST;
 428       } else if (
 429           response.verdict() == ClientDownloadResponse::POTENTIALLY_UNWANTED) {
 430         reason = REASON_DOWNLOAD_POTENTIALLY_UNWANTED;
 431         result = POTENTIALLY_UNWANTED;
 432       } else {
 433         LOG(DFATAL) << "Unknown download response verdict: "
 434                     << response.verdict();
 435         reason = REASON_INVALID_RESPONSE_VERDICT;
 436       }
 437       DownloadFeedbackService::MaybeStorePingsForDownload(
 438           result, item_, client_download_request_data_, data);
 439     }
 440     // We don't need the fetcher anymore.
 441     fetcher_.reset();
 442     UMA_HISTOGRAM_TIMES("SBClientDownload.DownloadRequestDuration",
 443                         base::TimeTicks::Now() - start_time_);
 444     UMA_HISTOGRAM_TIMES("SBClientDownload.DownloadRequestNetworkDuration",
 445                         base::TimeTicks::Now() - request_start_time_);
 446     FinishRequest(result, reason);
 447   }
 448
 449   static bool IsSupportedDownload(const content::DownloadItem& item,
 450                                   const base::FilePath& target_path,
 451                                   DownloadCheckResultReason* reason,
 452                                   ClientDownloadRequest::DownloadType* type) {
 453     if (item.GetUrlChain().empty()) {
 454       *reason = REASON_EMPTY_URL_CHAIN;
 455       return false;
 456     }
 457     const GURL& final_url = item.GetUrlChain().back();
 458     if (!final_url.is_valid() || final_url.is_empty() ||
 459         !final_url.IsStandard() || final_url.SchemeIsFile()) {
 460       *reason = REASON_INVALID_URL;
 461       return false;
 462     }
 463     if (!download_protection_util::IsBinaryFile(target_path)) {
 464       *reason = REASON_NOT_BINARY_FILE;
 465       return false;
 466     }
 467     *type = download_protection_util::GetDownloadType(target_path);
 468     return true;
 469   }
 470
 471  private:
 472   friend struct BrowserThread::DeleteOnThread<BrowserThread::UI>;
 473   friend class base::DeleteHelper<CheckClientDownloadRequest>;
 474
 475   virtual ~CheckClientDownloadRequest() {
 476     DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 477     DCHECK(item_ == NULL);
 478   }
 479
 480   void OnFileFeatureExtractionDone() {
 481     // This can run in any thread, since it just posts more messages.
 482
 483     // TODO(noelutz): DownloadInfo should also contain the IP address of
 484     // every URL in the redirect chain.  We also should check whether the
 485     // download URL is hosted on the internal network.
 486     BrowserThread::PostTask(
 487         BrowserThread::IO,
 488         FROM_HERE,
 489         base::Bind(&CheckClientDownloadRequest::CheckWhitelists, this));
 490
 491     // We wait until after the file checks finish to start the timeout, as
 492     // windows can cause permissions errors if the timeout fired while we were
 493     // checking the file signature and we tried to complete the download.
 494     BrowserThread::PostTask(
 495         BrowserThread::UI,
 496         FROM_HERE,
 497         base::Bind(&CheckClientDownloadRequest::StartTimeout, this));
 498   }
 499
 500   void StartExtractFileFeatures() {
 501     DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 502     DCHECK(item_);  // Called directly from Start(), item should still exist.
 503     // Since we do blocking I/O, offload this to a worker thread.
 504     // The task does not need to block shutdown.
 505     BrowserThread::GetBlockingPool()->PostWorkerTaskWithShutdownBehavior(
 506         FROM_HERE,
 507         base::Bind(&CheckClientDownloadRequest::ExtractFileFeatures,
 508                    this, item_->GetFullPath()),
 509         base::SequencedWorkerPool::CONTINUE_ON_SHUTDOWN);
 510   }
 511
 512   void ExtractFileFeatures(const base::FilePath& file_path) {
 513     base::TimeTicks start_time = base::TimeTicks::Now();
 514     binary_feature_extractor_->CheckSignature(file_path, &signature_info_);
 515     bool is_signed = (signature_info_.certificate_chain_size() > 0);
 516     if (is_signed) {
 517       VLOG(2) << "Downloaded a signed binary: " << file_path.value();
 518     } else {
 519       VLOG(2) << "Downloaded an unsigned binary: "
 520               << file_path.value();
 521     }
 522     UMA_HISTOGRAM_BOOLEAN("SBClientDownload.SignedBinaryDownload", is_signed);
 523     UMA_HISTOGRAM_TIMES("SBClientDownload.ExtractSignatureFeaturesTime",
 524                         base::TimeTicks::Now() - start_time);
 525
 526     start_time = base::TimeTicks::Now();
 527     binary_feature_extractor_->ExtractImageHeaders(file_path, &image_headers_);
 528     UMA_HISTOGRAM_TIMES("SBClientDownload.ExtractImageHeadersTime",
 529                         base::TimeTicks::Now() - start_time);
 530
 531     OnFileFeatureExtractionDone();
 532   }
 533
 534   void StartExtractZipFeatures() {
 535     DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 536     DCHECK(item_);  // Called directly from Start(), item should still exist.
 537     zip_analysis_start_time_ = base::TimeTicks::Now();
 538     // We give the zip analyzer a weak pointer to this object.  Since the
 539     // analyzer is refcounted, it might outlive the request.
 540     analyzer_ = new SandboxedZipAnalyzer(
 541         item_->GetFullPath(),
 542         base::Bind(&CheckClientDownloadRequest::OnZipAnalysisFinished,
 543                    weakptr_factory_.GetWeakPtr()));
 544     analyzer_->Start();
 545   }
 546
 547   void OnZipAnalysisFinished(const zip_analyzer::Results& results) {
 548     DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 549     if (!service_)
 550       return;
 551     if (results.success) {
 552       zipped_executable_ = results.has_executable;
 553       VLOG(1) << "Zip analysis finished for " << item_->GetFullPath().value()
 554               << ", has_executable=" << results.has_executable
 555               << " has_archive=" << results.has_archive;
 556     } else {
 557       VLOG(1) << "Zip analysis failed for " << item_->GetFullPath().value();
 558     }
 559     UMA_HISTOGRAM_BOOLEAN("SBClientDownload.ZipFileHasExecutable",
 560                           zipped_executable_);
 561     UMA_HISTOGRAM_BOOLEAN("SBClientDownload.ZipFileHasArchiveButNoExecutable",
 562                           results.has_archive && !zipped_executable_);
 563     UMA_HISTOGRAM_TIMES("SBClientDownload.ExtractZipFeaturesTime",
 564                         base::TimeTicks::Now() - zip_analysis_start_time_);
 565
 566     if (!zipped_executable_) {
 567       PostFinishTask(SAFE, REASON_ARCHIVE_WITHOUT_BINARIES);
 568       return;
 569     }
 570     OnFileFeatureExtractionDone();
 571   }
 572
 573   void CheckWhitelists() {
 574     DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 575     DownloadCheckResultReason reason = REASON_MAX;
 576     if (!database_manager_.get()) {
 577       reason = REASON_SB_DISABLED;
 578     } else {
 579       const GURL& url = url_chain_.back();
 580       if (url.is_valid() && database_manager_->MatchDownloadWhitelistUrl(url)) {
 581         VLOG(2) << url << " is on the download whitelist.";
 582         reason = REASON_WHITELISTED_URL;
 583       }
 584       if (reason != REASON_MAX || signature_info_.trusted()) {
 585         UMA_HISTOGRAM_COUNTS("SBClientDownload.SignedOrWhitelistedDownload", 1);
 586       }
 587     }
 588     if (reason == REASON_MAX && signature_info_.trusted()) {
 589       for (int i = 0; i < signature_info_.certificate_chain_size(); ++i) {
 590         if (CertificateChainIsWhitelisted(
 591                 signature_info_.certificate_chain(i))) {
 592           reason = REASON_TRUSTED_EXECUTABLE;
 593           break;
 594         }
 595       }
 596     }
 597     if (reason != REASON_MAX) {
 598       PostFinishTask(SAFE, reason);
 599     } else if (!pingback_enabled_) {
 600       PostFinishTask(SAFE, REASON_PING_DISABLED);
 601     } else {
 602       // Currently, the UI only works on Windows so we don't even bother
 603       // with pinging the server if we're not on Windows.  TODO(noelutz):
 604       // change this code once the UI is done for Linux and Mac.
 605 #if defined(OS_WIN)
 606       // The URLFetcher is owned by the UI thread, so post a message to
 607       // start the pingback.
 608       BrowserThread::PostTask(
 609           BrowserThread::UI,
 610           FROM_HERE,
 611           base::Bind(&CheckClientDownloadRequest::GetTabRedirects, this));
 612 #else
 613       PostFinishTask(SAFE, REASON_OS_NOT_SUPPORTED);
 614 #endif
 615     }
 616   }
 617
 618   void GetTabRedirects() {
 619     DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 620     if (!tab_url_.is_valid()) {
 621       SendRequest();
 622       return;
 623     }
 624
 625     Profile* profile = Profile::FromBrowserContext(item_->GetBrowserContext());
 626     HistoryService* history =
 627         HistoryServiceFactory::GetForProfile(profile, Profile::EXPLICIT_ACCESS);
 628     if (!history) {
 629       SendRequest();
 630       return;
 631     }
 632
 633     history->QueryRedirectsTo(
 634         tab_url_,
 635         base::Bind(&CheckClientDownloadRequest::OnGotTabRedirects,
 636                    base::Unretained(this),
 637                    tab_url_),
 638         &request_tracker_);
 639   }
 640
 641   void OnGotTabRedirects(const GURL& url,
 642                          const history::RedirectList* redirect_list) {
 643     DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 644     DCHECK_EQ(url, tab_url_);
 645
 646     if (!redirect_list->empty()) {
 647       tab_redirects_.insert(
 648           tab_redirects_.end(), redirect_list->rbegin(), redirect_list->rend());
 649     }
 650
 651     SendRequest();
 652   }
 653
 654   void SendRequest() {
 655     DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 656
 657     // This is our last chance to check whether the request has been canceled
 658     // before sending it.
 659     if (!service_)
 660       return;
 661
 662     ClientDownloadRequest request;
 663     request.set_url(item_->GetUrlChain().back().spec());
 664     request.mutable_digests()->set_sha256(item_->GetHash());
 665     request.set_length(item_->GetReceivedBytes());
 666     for (size_t i = 0; i < item_->GetUrlChain().size(); ++i) {
 667       ClientDownloadRequest::Resource* resource = request.add_resources();
 668       resource->set_url(item_->GetUrlChain()[i].spec());
 669       if (i == item_->GetUrlChain().size() - 1) {
 670         // The last URL in the chain is the download URL.
 671         resource->set_type(ClientDownloadRequest::DOWNLOAD_URL);
 672         resource->set_referrer(item_->GetReferrerUrl().spec());
 673         DVLOG(2) << "dl url " << resource->url();
 674         if (!item_->GetRemoteAddress().empty()) {
 675           resource->set_remote_ip(item_->GetRemoteAddress());
 676           DVLOG(2) << "  dl url remote addr: " << resource->remote_ip();
 677         }
 678         DVLOG(2) << "dl referrer " << resource->referrer();
 679       } else {
 680         DVLOG(2) << "dl redirect " << i << " " << resource->url();
 681         resource->set_type(ClientDownloadRequest::DOWNLOAD_REDIRECT);
 682       }
 683       // TODO(noelutz): fill out the remote IP addresses.
 684     }
 685     // TODO(mattm): fill out the remote IP addresses for tab resources.
 686     for (size_t i = 0; i < tab_redirects_.size(); ++i) {
 687       ClientDownloadRequest::Resource* resource = request.add_resources();
 688       DVLOG(2) << "tab redirect " << i << " " << tab_redirects_[i].spec();
 689       resource->set_url(tab_redirects_[i].spec());
 690       resource->set_type(ClientDownloadRequest::TAB_REDIRECT);
 691     }
 692     if (tab_url_.is_valid()) {
 693       ClientDownloadRequest::Resource* resource = request.add_resources();
 694       resource->set_url(tab_url_.spec());
 695       DVLOG(2) << "tab url " << resource->url();
 696       resource->set_type(ClientDownloadRequest::TAB_URL);
 697       if (tab_referrer_url_.is_valid()) {
 698         resource->set_referrer(tab_referrer_url_.spec());
 699         DVLOG(2) << "tab referrer " << resource->referrer();
 700       }
 701     }
 702
 703     request.set_user_initiated(item_->HasUserGesture());
 704     request.set_file_basename(
 705         item_->GetTargetFilePath().BaseName().AsUTF8Unsafe());
 706     request.set_download_type(type_);
 707     request.mutable_signature()->CopyFrom(signature_info_);
 708     request.mutable_image_headers()->CopyFrom(image_headers_);
 709     if (!request.SerializeToString(&client_download_request_data_)) {
 710       FinishRequest(SAFE, REASON_INVALID_REQUEST_PROTO);
 711       return;
 712     }
 713
 714     VLOG(2) << "Sending a request for URL: "
 715             << item_->GetUrlChain().back();
 716     fetcher_.reset(net::URLFetcher::Create(0 /* ID used for testing */,
 717                                            GetDownloadRequestUrl(),
 718                                            net::URLFetcher::POST,
 719                                            this));
 720     fetcher_->SetLoadFlags(net::LOAD_DISABLE_CACHE);
 721     fetcher_->SetAutomaticallyRetryOn5xx(false);  // Don't retry on error.
 722     fetcher_->SetRequestContext(service_->request_context_getter_.get());
 723     fetcher_->SetUploadData("application/octet-stream",
 724                             client_download_request_data_);
 725     request_start_time_ = base::TimeTicks::Now();
 726     UMA_HISTOGRAM_COUNTS("SBClientDownload.DownloadRequestPayloadSize",
 727                          client_download_request_data_.size());
 728     fetcher_->Start();
 729   }
 730
 731   void PostFinishTask(DownloadCheckResult result,
 732                       DownloadCheckResultReason reason) {
 733     BrowserThread::PostTask(
 734         BrowserThread::UI,
 735         FROM_HERE,
 736         base::Bind(&CheckClientDownloadRequest::FinishRequest, this, result,
 737                    reason));
 738   }
 739
 740   void FinishRequest(DownloadCheckResult result,
 741                      DownloadCheckResultReason reason) {
 742     DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 743     if (finished_) {
 744       return;
 745     }
 746     finished_ = true;
 747     // Ensure the timeout task is cancelled while we still have a non-zero
 748     // refcount. (crbug.com/240449)
 749     weakptr_factory_.InvalidateWeakPtrs();
 750     if (!request_start_time_.is_null()) {
 751       UMA_HISTOGRAM_ENUMERATION("SBClientDownload.DownloadRequestNetworkStats",
 752                                 reason,
 753                                 REASON_MAX);
 754     }
 755     if (!timeout_start_time_.is_null()) {
 756       UMA_HISTOGRAM_ENUMERATION("SBClientDownload.DownloadRequestTimeoutStats",
 757                                 reason,
 758                                 REASON_MAX);
 759       if (reason != REASON_REQUEST_CANCELED) {
 760         UMA_HISTOGRAM_TIMES("SBClientDownload.DownloadRequestTimeoutDuration",
 761                             base::TimeTicks::Now() - timeout_start_time_);
 762       }
 763     }
 764     if (service_) {
 765       VLOG(2) << "SafeBrowsing download verdict for: "
 766               << item_->DebugString(true) << " verdict:" << reason;
 767       UMA_HISTOGRAM_ENUMERATION("SBClientDownload.CheckDownloadStats",
 768                                 reason,
 769                                 REASON_MAX);
 770       callback_.Run(result);
 771       item_->RemoveObserver(this);
 772       item_ = NULL;
 773       DownloadProtectionService* service = service_;
 774       service_ = NULL;
 775       service->RequestFinished(this);
 776       // DownloadProtectionService::RequestFinished will decrement our refcount,
 777       // so we may be deleted now.
 778     } else {
 779       callback_.Run(SAFE);
 780     }
 781   }
 782
 783   bool CertificateChainIsWhitelisted(
 784       const ClientDownloadRequest_CertificateChain& chain) {
 785     DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 786     if (chain.element_size() < 2) {
 787       // We need to have both a signing certificate and its issuer certificate
 788       // present to construct a whitelist entry.
 789       return false;
 790     }
 791     scoped_refptr<net::X509Certificate> cert =
 792         net::X509Certificate::CreateFromBytes(
 793             chain.element(0).certificate().data(),
 794             chain.element(0).certificate().size());
 795     if (!cert.get()) {
 796       return false;
 797     }
 798
 799     for (int i = 1; i < chain.element_size(); ++i) {
 800       scoped_refptr<net::X509Certificate> issuer =
 801           net::X509Certificate::CreateFromBytes(
 802               chain.element(i).certificate().data(),
 803               chain.element(i).certificate().size());
 804       if (!issuer.get()) {
 805         return false;
 806       }
 807       std::vector<std::string> whitelist_strings;
 808       DownloadProtectionService::GetCertificateWhitelistStrings(
 809           *cert.get(), *issuer.get(), &whitelist_strings);
 810       for (size_t j = 0; j < whitelist_strings.size(); ++j) {
 811         if (database_manager_->MatchDownloadWhitelistString(
 812                 whitelist_strings[j])) {
 813           VLOG(2) << "Certificate matched whitelist, cert="
 814                   << cert->subject().GetDisplayName()
 815                   << " issuer=" << issuer->subject().GetDisplayName();
 816           return true;
 817         }
 818       }
 819       cert = issuer;
 820     }
 821     return false;
 822   }
 823
 824   // The DownloadItem we are checking. Will be NULL if the request has been
 825   // canceled. Must be accessed only on UI thread.
 826   content::DownloadItem* item_;
 827   // Copies of data from |item_| for access on other threads.
 828   std::vector<GURL> url_chain_;
 829   GURL referrer_url_;
 830   // URL chain of redirects leading to (but not including) |tab_url|.
 831   std::vector<GURL> tab_redirects_;
 832   // URL and referrer of the window the download was started from.
 833   GURL tab_url_;
 834   GURL tab_referrer_url_;
 835
 836   bool zipped_executable_;
 837   ClientDownloadRequest_SignatureInfo signature_info_;
 838   ClientDownloadRequest_ImageHeaders image_headers_;
 839   CheckDownloadCallback callback_;
 840   // Will be NULL if the request has been canceled.
 841   DownloadProtectionService* service_;
 842   scoped_refptr<BinaryFeatureExtractor> binary_feature_extractor_;
 843   scoped_refptr<SafeBrowsingDatabaseManager> database_manager_;
 844   const bool pingback_enabled_;
 845   scoped_ptr<net::URLFetcher> fetcher_;
 846   scoped_refptr<SandboxedZipAnalyzer> analyzer_;
 847   base::TimeTicks zip_analysis_start_time_;
 848   bool finished_;
 849   ClientDownloadRequest::DownloadType type_;
 850   std::string client_download_request_data_;
 851   base::CancelableTaskTracker request_tracker_;  // For HistoryService lookup.
 852   base::WeakPtrFactory<CheckClientDownloadRequest> weakptr_factory_;
 853   base::TimeTicks start_time_;  // Used for stats.
 854   base::TimeTicks timeout_start_time_;
 855   base::TimeTicks request_start_time_;
 856
 857   DISALLOW_COPY_AND_ASSIGN(CheckClientDownloadRequest);
 858 };
 859
 860 DownloadProtectionService::DownloadProtectionService(
 861     SafeBrowsingService* sb_service,
 862     net::URLRequestContextGetter* request_context_getter)
 863     : request_context_getter_(request_context_getter),
 864       enabled_(false),
 865       binary_feature_extractor_(new BinaryFeatureExtractor()),
 866       download_request_timeout_ms_(kDownloadRequestTimeoutMs),
 867       feedback_service_(new DownloadFeedbackService(
 868           request_context_getter, BrowserThread::GetBlockingPool())) {
 869
 870   if (sb_service) {
 871     ui_manager_ = sb_service->ui_manager();
 872     database_manager_ = sb_service->database_manager();
 873   }
 874 }
 875
 876 DownloadProtectionService::~DownloadProtectionService() {
 877   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 878   CancelPendingRequests();
 879 }
 880
 881 void DownloadProtectionService::SetEnabled(bool enabled) {
 882   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 883   if (enabled == enabled_) {
 884     return;
 885   }
 886   enabled_ = enabled;
 887   if (!enabled_) {
 888     CancelPendingRequests();
 889   }
 890 }
 891
 892 void DownloadProtectionService::CheckClientDownload(
 893     content::DownloadItem* item,
 894     const CheckDownloadCallback& callback) {
 895   scoped_refptr<CheckClientDownloadRequest> request(
 896       new CheckClientDownloadRequest(item, callback, this,
 897                                      database_manager_,
 898                                      binary_feature_extractor_.get()));
 899   download_requests_.insert(request);
 900   request->Start();
 901 }
 902
 903 void DownloadProtectionService::CheckDownloadUrl(
 904     const content::DownloadItem& item,
 905     const CheckDownloadCallback& callback) {
 906   DCHECK(!item.GetUrlChain().empty());
 907   scoped_refptr<DownloadUrlSBClient> client(
 908       new DownloadUrlSBClient(item, callback, ui_manager_, database_manager_));
 909   // The client will release itself once it is done.
 910   BrowserThread::PostTask(
 911         BrowserThread::IO,
 912         FROM_HERE,
 913         base::Bind(&DownloadUrlSBClient::StartCheck, client));
 914 }
 915
 916 bool DownloadProtectionService::IsSupportedDownload(
 917     const content::DownloadItem& item,
 918     const base::FilePath& target_path) const {
 919   // Currently, the UI only works on Windows.  On Linux and Mac we still
 920   // want to show the dangerous file type warning if the file is possibly
 921   // dangerous which means we have to always return false here.
 922 #if defined(OS_WIN)
 923   DownloadCheckResultReason reason = REASON_MAX;
 924   ClientDownloadRequest::DownloadType type =
 925       ClientDownloadRequest::WIN_EXECUTABLE;
 926   return (CheckClientDownloadRequest::IsSupportedDownload(item, target_path,
 927                                                           &reason, &type) &&
 928           (ClientDownloadRequest::ANDROID_APK == type ||
 929            ClientDownloadRequest::WIN_EXECUTABLE == type ||
 930            ClientDownloadRequest::ZIPPED_EXECUTABLE == type));
 931 #else
 932   return false;
 933 #endif
 934 }
 935
 936 void DownloadProtectionService::CancelPendingRequests() {
 937   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 938   for (std::set<scoped_refptr<CheckClientDownloadRequest> >::iterator it =
 939            download_requests_.begin();
 940        it != download_requests_.end();) {
 941     // We need to advance the iterator before we cancel because canceling
 942     // the request will invalidate it when RequestFinished is called below.
 943     scoped_refptr<CheckClientDownloadRequest> tmp = *it++;
 944     tmp->Cancel();
 945   }
 946   DCHECK(download_requests_.empty());
 947 }
 948
 949 void DownloadProtectionService::RequestFinished(
 950     CheckClientDownloadRequest* request) {
 951   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 952   std::set<scoped_refptr<CheckClientDownloadRequest> >::iterator it =
 953       download_requests_.find(request);
 954   DCHECK(it != download_requests_.end());
 955   download_requests_.erase(*it);
 956 }
 957
 958 void DownloadProtectionService::ShowDetailsForDownload(
 959     const content::DownloadItem& item,
 960     content::PageNavigator* navigator) {
 961   GURL learn_more_url(chrome::kDownloadScanningLearnMoreURL);
 962   navigator->OpenURL(
 963       content::OpenURLParams(learn_more_url,
 964                              content::Referrer(),
 965                              NEW_FOREGROUND_TAB,
 966                              content::PAGE_TRANSITION_LINK,
 967                              false));
 968 }
 969
 970 namespace {
 971 // Escapes a certificate attribute so that it can be used in a whitelist
 972 // entry.  Currently, we only escape slashes, since they are used as a
 973 // separator between attributes.
 974 std::string EscapeCertAttribute(const std::string& attribute) {
 975   std::string escaped;
 976   for (size_t i = 0; i < attribute.size(); ++i) {
 977     if (attribute[i] == '%') {
 978       escaped.append("%25");
 979     } else if (attribute[i] == '/') {
 980       escaped.append("%2F");
 981     } else {
 982       escaped.push_back(attribute[i]);
 983     }
 984   }
 985   return escaped;
 986 }
 987 }  // namespace
 988
 989 // static
 990 void DownloadProtectionService::GetCertificateWhitelistStrings(
 991     const net::X509Certificate& certificate,
 992     const net::X509Certificate& issuer,
 993     std::vector<std::string>* whitelist_strings) {
 994   // The whitelist paths are in the format:
 995   // cert/<ascii issuer fingerprint>[/CN=common_name][/O=org][/OU=unit]
 996   //
 997   // Any of CN, O, or OU may be omitted from the whitelist entry, in which
 998   // case they match anything.  However, the attributes that do appear will
 999   // always be in the order shown above.  At least one attribute will always
1000   // be present.
1001
1002   const net::CertPrincipal& subject = certificate.subject();
1003   std::vector<std::string> ou_tokens;
1004   for (size_t i = 0; i < subject.organization_unit_names.size(); ++i) {
1005     ou_tokens.push_back(
1006         "/OU=" + EscapeCertAttribute(subject.organization_unit_names[i]));
1007   }
1008
1009   std::vector<std::string> o_tokens;
1010   for (size_t i = 0; i < subject.organization_names.size(); ++i) {
1011     o_tokens.push_back(
1012         "/O=" + EscapeCertAttribute(subject.organization_names[i]));
1013   }
1014
1015   std::string cn_token;
1016   if (!subject.common_name.empty()) {
1017     cn_token = "/CN=" + EscapeCertAttribute(subject.common_name);
1018   }
1019
1020   std::set<std::string> paths_to_check;
1021   if (!cn_token.empty()) {
1022     paths_to_check.insert(cn_token);
1023   }
1024   for (size_t i = 0; i < o_tokens.size(); ++i) {
1025     paths_to_check.insert(cn_token + o_tokens[i]);
1026     paths_to_check.insert(o_tokens[i]);
1027     for (size_t j = 0; j < ou_tokens.size(); ++j) {
1028       paths_to_check.insert(cn_token + o_tokens[i] + ou_tokens[j]);
1029       paths_to_check.insert(o_tokens[i] + ou_tokens[j]);
1030     }
1031   }
1032   for (size_t i = 0; i < ou_tokens.size(); ++i) {
1033     paths_to_check.insert(cn_token + ou_tokens[i]);
1034     paths_to_check.insert(ou_tokens[i]);
1035   }
1036
1037   std::string issuer_fp = base::HexEncode(issuer.fingerprint().data,
1038                                           sizeof(issuer.fingerprint().data));
1039   for (std::set<std::string>::iterator it = paths_to_check.begin();
1040        it != paths_to_check.end(); ++it) {
1041     whitelist_strings->push_back("cert/" + issuer_fp + *it);
1042   }
1043 }
1044
1045 // static
1046 GURL DownloadProtectionService::GetDownloadRequestUrl() {
1047   GURL url(kDownloadRequestUrl);
1048   std::string api_key = google_apis::GetAPIKey();
1049   if (!api_key.empty())
1050     url = url.Resolve("?key=" + net::EscapeQueryParamValue(api_key, true));
1051
1052   return url;
1053 }
1054
1055 }  // namespace safe_browsing