chrome/browser/safe_browsing/download_protection_service.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "chrome/browser/safe_browsing/download_protection_service.h"
   6
   7 #include "base/bind.h"
   8 #include "base/compiler_specific.h"
   9 #include "base/format_macros.h"
  10 #include "base/memory/scoped_ptr.h"
  11 #include "base/memory/weak_ptr.h"
  12 #include "base/metrics/histogram.h"
  13 #include "base/metrics/sparse_histogram.h"
  14 #include "base/sequenced_task_runner_helpers.h"
  15 #include "base/stl_util.h"
  16 #include "base/strings/string_number_conversions.h"
  17 #include "base/strings/string_util.h"
  18 #include "base/strings/stringprintf.h"
  19 #include "base/threading/sequenced_worker_pool.h"
  20 #include "base/time/time.h"
  21 #include "chrome/browser/history/history_service.h"
  22 #include "chrome/browser/history/history_service_factory.h"
  23 #include "chrome/browser/safe_browsing/binary_feature_extractor.h"
  24 #include "chrome/browser/safe_browsing/download_feedback_service.h"
  25 #include "chrome/browser/safe_browsing/safe_browsing_service.h"
  26 #include "chrome/browser/safe_browsing/sandboxed_zip_analyzer.h"
  27 #include "chrome/browser/ui/browser.h"
  28 #include "chrome/browser/ui/browser_list.h"
  29 #include "chrome/common/safe_browsing/csd.pb.h"
  30 #include "chrome/common/safe_browsing/download_protection_util.h"
  31 #include "chrome/common/safe_browsing/zip_analyzer.h"
  32 #include "chrome/common/url_constants.h"
  33 #include "content/public/browser/browser_thread.h"
  34 #include "content/public/browser/download_item.h"
  35 #include "content/public/browser/page_navigator.h"
  36 #include "google_apis/google_api_keys.h"
  37 #include "net/base/escape.h"
  38 #include "net/base/load_flags.h"
  39 #include "net/cert/x509_cert_types.h"
  40 #include "net/cert/x509_certificate.h"
  41 #include "net/http/http_status_code.h"
  42 #include "net/url_request/url_fetcher.h"
  43 #include "net/url_request/url_fetcher_delegate.h"
  44 #include "net/url_request/url_request_context_getter.h"
  45 #include "net/url_request/url_request_status.h"
  46
  47 using content::BrowserThread;
  48
  49 namespace {
  50 static const int64 kDownloadRequestTimeoutMs = 3000;
  51 }  // namespace
  52
  53 namespace safe_browsing {
  54
  55 const char DownloadProtectionService::kDownloadRequestUrl[] =
  56     "https://sb-ssl.google.com/safebrowsing/clientreport/download";
  57
  58 namespace {
  59 ClientDownloadRequest::DownloadType GetDownloadType(
  60     const base::FilePath& file) {
  61   DCHECK(download_protection_util::IsBinaryFile(file));
  62   if (file.MatchesExtension(FILE_PATH_LITERAL(".apk")))
  63     return ClientDownloadRequest::ANDROID_APK;
  64   else if (file.MatchesExtension(FILE_PATH_LITERAL(".crx")))
  65     return ClientDownloadRequest::CHROME_EXTENSION;
  66   // For zip files, we use the ZIPPED_EXECUTABLE type since we will only send
  67   // the pingback if we find an executable inside the zip archive.
  68   else if (file.MatchesExtension(FILE_PATH_LITERAL(".zip")))
  69     return ClientDownloadRequest::ZIPPED_EXECUTABLE;
  70   return ClientDownloadRequest::WIN_EXECUTABLE;
  71 }
  72
  73 // List of extensions for which we track some UMA stats.
  74 enum MaliciousExtensionType {
  75   EXTENSION_EXE,
  76   EXTENSION_MSI,
  77   EXTENSION_CAB,
  78   EXTENSION_SYS,
  79   EXTENSION_SCR,
  80   EXTENSION_DRV,
  81   EXTENSION_BAT,
  82   EXTENSION_ZIP,
  83   EXTENSION_RAR,
  84   EXTENSION_DLL,
  85   EXTENSION_PIF,
  86   EXTENSION_COM,
  87   EXTENSION_JAR,
  88   EXTENSION_CLASS,
  89   EXTENSION_PDF,
  90   EXTENSION_VB,
  91   EXTENSION_REG,
  92   EXTENSION_GRP,
  93   EXTENSION_OTHER,  // Groups all other extensions into one bucket.
  94   EXTENSION_CRX,
  95   EXTENSION_APK,
  96   EXTENSION_DMG,
  97   EXTENSION_PKG,
  98   EXTENSION_MAX,
  99 };
 100
 101 MaliciousExtensionType GetExtensionType(const base::FilePath& f) {
 102   if (f.MatchesExtension(FILE_PATH_LITERAL(".exe"))) return EXTENSION_EXE;
 103   if (f.MatchesExtension(FILE_PATH_LITERAL(".msi"))) return EXTENSION_MSI;
 104   if (f.MatchesExtension(FILE_PATH_LITERAL(".cab"))) return EXTENSION_CAB;
 105   if (f.MatchesExtension(FILE_PATH_LITERAL(".sys"))) return EXTENSION_SYS;
 106   if (f.MatchesExtension(FILE_PATH_LITERAL(".scr"))) return EXTENSION_SCR;
 107   if (f.MatchesExtension(FILE_PATH_LITERAL(".drv"))) return EXTENSION_DRV;
 108   if (f.MatchesExtension(FILE_PATH_LITERAL(".bat"))) return EXTENSION_BAT;
 109   if (f.MatchesExtension(FILE_PATH_LITERAL(".zip"))) return EXTENSION_ZIP;
 110   if (f.MatchesExtension(FILE_PATH_LITERAL(".rar"))) return EXTENSION_RAR;
 111   if (f.MatchesExtension(FILE_PATH_LITERAL(".dll"))) return EXTENSION_DLL;
 112   if (f.MatchesExtension(FILE_PATH_LITERAL(".pif"))) return EXTENSION_PIF;
 113   if (f.MatchesExtension(FILE_PATH_LITERAL(".com"))) return EXTENSION_COM;
 114   if (f.MatchesExtension(FILE_PATH_LITERAL(".jar"))) return EXTENSION_JAR;
 115   if (f.MatchesExtension(FILE_PATH_LITERAL(".class"))) return EXTENSION_CLASS;
 116   if (f.MatchesExtension(FILE_PATH_LITERAL(".pdf"))) return EXTENSION_PDF;
 117   if (f.MatchesExtension(FILE_PATH_LITERAL(".vb"))) return EXTENSION_VB;
 118   if (f.MatchesExtension(FILE_PATH_LITERAL(".reg"))) return EXTENSION_REG;
 119   if (f.MatchesExtension(FILE_PATH_LITERAL(".grp"))) return EXTENSION_GRP;
 120   if (f.MatchesExtension(FILE_PATH_LITERAL(".crx"))) return EXTENSION_CRX;
 121   if (f.MatchesExtension(FILE_PATH_LITERAL(".apk"))) return EXTENSION_APK;
 122   if (f.MatchesExtension(FILE_PATH_LITERAL(".dmg"))) return EXTENSION_DMG;
 123   if (f.MatchesExtension(FILE_PATH_LITERAL(".pkg"))) return EXTENSION_PKG;
 124   return EXTENSION_OTHER;
 125 }
 126
 127 void RecordFileExtensionType(const base::FilePath& file) {
 128   UMA_HISTOGRAM_ENUMERATION("SBClientDownload.DownloadExtensions",
 129                             GetExtensionType(file),
 130                             EXTENSION_MAX);
 131 }
 132
 133 // Enumerate for histogramming purposes.
 134 // DO NOT CHANGE THE ORDERING OF THESE VALUES (different histogram data will
 135 // be mixed together based on their values).
 136 enum SBStatsType {
 137   DOWNLOAD_URL_CHECKS_TOTAL,
 138   DOWNLOAD_URL_CHECKS_CANCELED,
 139   DOWNLOAD_URL_CHECKS_MALWARE,
 140
 141   DOWNLOAD_HASH_CHECKS_TOTAL,
 142   DOWNLOAD_HASH_CHECKS_MALWARE,
 143
 144   // Memory space for histograms is determined by the max.
 145   // ALWAYS ADD NEW VALUES BEFORE THIS ONE.
 146   DOWNLOAD_CHECKS_MAX
 147 };
 148 }  // namespace
 149
 150 // Parent SafeBrowsing::Client class used to lookup the bad binary
 151 // URL and digest list.  There are two sub-classes (one for each list).
 152 class DownloadSBClient
 153     : public SafeBrowsingDatabaseManager::Client,
 154       public base::RefCountedThreadSafe<DownloadSBClient> {
 155  public:
 156   DownloadSBClient(
 157       const content::DownloadItem& item,
 158       const DownloadProtectionService::CheckDownloadCallback& callback,
 159       const scoped_refptr<SafeBrowsingUIManager>& ui_manager,
 160       SBStatsType total_type,
 161       SBStatsType dangerous_type)
 162       : sha256_hash_(item.GetHash()),
 163         url_chain_(item.GetUrlChain()),
 164         referrer_url_(item.GetReferrerUrl()),
 165         callback_(callback),
 166         ui_manager_(ui_manager),
 167         start_time_(base::TimeTicks::Now()),
 168         total_type_(total_type),
 169         dangerous_type_(dangerous_type) {}
 170
 171   virtual void StartCheck() = 0;
 172   virtual bool IsDangerous(SBThreatType threat_type) const = 0;
 173
 174  protected:
 175   friend class base::RefCountedThreadSafe<DownloadSBClient>;
 176   virtual ~DownloadSBClient() {}
 177
 178   void CheckDone(SBThreatType threat_type) {
 179     DownloadProtectionService::DownloadCheckResult result =
 180         IsDangerous(threat_type) ?
 181         DownloadProtectionService::DANGEROUS :
 182         DownloadProtectionService::SAFE;
 183     BrowserThread::PostTask(BrowserThread::UI,
 184                             FROM_HERE,
 185                             base::Bind(callback_, result));
 186     UpdateDownloadCheckStats(total_type_);
 187     if (threat_type != SB_THREAT_TYPE_SAFE) {
 188       UpdateDownloadCheckStats(dangerous_type_);
 189       BrowserThread::PostTask(
 190           BrowserThread::UI,
 191           FROM_HERE,
 192           base::Bind(&DownloadSBClient::ReportMalware,
 193                      this, threat_type));
 194     }
 195   }
 196
 197   void ReportMalware(SBThreatType threat_type) {
 198     std::string post_data;
 199     if (!sha256_hash_.empty())
 200       post_data += base::HexEncode(sha256_hash_.data(),
 201                                    sha256_hash_.size()) + "\n";
 202     for (size_t i = 0; i < url_chain_.size(); ++i) {
 203       post_data += url_chain_[i].spec() + "\n";
 204     }
 205     ui_manager_->ReportSafeBrowsingHit(
 206         url_chain_.back(),  // malicious_url
 207         url_chain_.front(), // page_url
 208         referrer_url_,
 209         true,  // is_subresource
 210         threat_type,
 211         post_data);
 212   }
 213
 214   void UpdateDownloadCheckStats(SBStatsType stat_type) {
 215     UMA_HISTOGRAM_ENUMERATION("SB2.DownloadChecks",
 216                               stat_type,
 217                               DOWNLOAD_CHECKS_MAX);
 218   }
 219
 220   std::string sha256_hash_;
 221   std::vector<GURL> url_chain_;
 222   GURL referrer_url_;
 223   DownloadProtectionService::CheckDownloadCallback callback_;
 224   scoped_refptr<SafeBrowsingUIManager> ui_manager_;
 225   base::TimeTicks start_time_;
 226
 227  private:
 228   const SBStatsType total_type_;
 229   const SBStatsType dangerous_type_;
 230
 231   DISALLOW_COPY_AND_ASSIGN(DownloadSBClient);
 232 };
 233
 234 class DownloadUrlSBClient : public DownloadSBClient {
 235  public:
 236   DownloadUrlSBClient(
 237       const content::DownloadItem& item,
 238       const DownloadProtectionService::CheckDownloadCallback& callback,
 239       const scoped_refptr<SafeBrowsingUIManager>& ui_manager,
 240       const scoped_refptr<SafeBrowsingDatabaseManager>& database_manager)
 241       : DownloadSBClient(item, callback, ui_manager,
 242                          DOWNLOAD_URL_CHECKS_TOTAL,
 243                          DOWNLOAD_URL_CHECKS_MALWARE),
 244         database_manager_(database_manager) { }
 245
 246   virtual void StartCheck() OVERRIDE {
 247     DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 248     if (!database_manager_.get() ||
 249         database_manager_->CheckDownloadUrl(url_chain_, this)) {
 250       CheckDone(SB_THREAT_TYPE_SAFE);
 251     } else {
 252       AddRef();  // SafeBrowsingService takes a pointer not a scoped_refptr.
 253     }
 254   }
 255
 256   virtual bool IsDangerous(SBThreatType threat_type) const OVERRIDE {
 257     return threat_type == SB_THREAT_TYPE_BINARY_MALWARE_URL;
 258   }
 259
 260   virtual void OnCheckDownloadUrlResult(const std::vector<GURL>& url_chain,
 261                                         SBThreatType threat_type) OVERRIDE {
 262     CheckDone(threat_type);
 263     UMA_HISTOGRAM_TIMES("SB2.DownloadUrlCheckDuration",
 264                         base::TimeTicks::Now() - start_time_);
 265     Release();
 266   }
 267
 268  protected:
 269   virtual ~DownloadUrlSBClient() {}
 270
 271  private:
 272   scoped_refptr<SafeBrowsingDatabaseManager> database_manager_;
 273
 274   DISALLOW_COPY_AND_ASSIGN(DownloadUrlSBClient);
 275 };
 276
 277 class DownloadProtectionService::CheckClientDownloadRequest
 278     : public base::RefCountedThreadSafe<
 279           DownloadProtectionService::CheckClientDownloadRequest,
 280           BrowserThread::DeleteOnUIThread>,
 281       public net::URLFetcherDelegate,
 282       public content::DownloadItem::Observer {
 283  public:
 284   CheckClientDownloadRequest(
 285       content::DownloadItem* item,
 286       const CheckDownloadCallback& callback,
 287       DownloadProtectionService* service,
 288       const scoped_refptr<SafeBrowsingDatabaseManager>& database_manager,
 289       BinaryFeatureExtractor* binary_feature_extractor)
 290       : item_(item),
 291         url_chain_(item->GetUrlChain()),
 292         referrer_url_(item->GetReferrerUrl()),
 293         tab_url_(item->GetTabUrl()),
 294         tab_referrer_url_(item->GetTabReferrerUrl()),
 295         zipped_executable_(false),
 296         callback_(callback),
 297         service_(service),
 298         binary_feature_extractor_(binary_feature_extractor),
 299         database_manager_(database_manager),
 300         pingback_enabled_(service_->enabled()),
 301         finished_(false),
 302         type_(ClientDownloadRequest::WIN_EXECUTABLE),
 303         weakptr_factory_(this),
 304         start_time_(base::TimeTicks::Now()) {
 305     DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 306     item_->AddObserver(this);
 307   }
 308
 309   void Start() {
 310     VLOG(2) << "Starting SafeBrowsing download check for: "
 311             << item_->DebugString(true);
 312     DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 313     // TODO(noelutz): implement some cache to make sure we don't issue the same
 314     // request over and over again if a user downloads the same binary multiple
 315     // times.
 316     DownloadCheckResultReason reason = REASON_MAX;
 317     if (!IsSupportedDownload(
 318         *item_, item_->GetTargetFilePath(), &reason, &type_)) {
 319       switch (reason) {
 320         case REASON_EMPTY_URL_CHAIN:
 321         case REASON_INVALID_URL:
 322           PostFinishTask(SAFE, reason);
 323           return;
 324
 325         case REASON_NOT_BINARY_FILE:
 326           RecordFileExtensionType(item_->GetTargetFilePath());
 327           PostFinishTask(SAFE, reason);
 328           return;
 329
 330         default:
 331           // We only expect the reasons explicitly handled above.
 332           NOTREACHED();
 333       }
 334     }
 335     RecordFileExtensionType(item_->GetTargetFilePath());
 336
 337     // Compute features from the file contents. Note that we record histograms
 338     // based on the result, so this runs regardless of whether the pingbacks
 339     // are enabled.
 340     if (item_->GetTargetFilePath().MatchesExtension(
 341         FILE_PATH_LITERAL(".zip"))) {
 342       StartExtractZipFeatures();
 343     } else {
 344       DCHECK(!download_protection_util::IsArchiveFile(
 345           item_->GetTargetFilePath()));
 346       StartExtractFileFeatures();
 347     }
 348   }
 349
 350   // Start a timeout to cancel the request if it takes too long.
 351   // This should only be called after we have finished accessing the file.
 352   void StartTimeout() {
 353     DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 354     if (!service_) {
 355       // Request has already been cancelled.
 356       return;
 357     }
 358     BrowserThread::PostDelayedTask(
 359         BrowserThread::UI,
 360         FROM_HERE,
 361         base::Bind(&CheckClientDownloadRequest::Cancel,
 362                    weakptr_factory_.GetWeakPtr()),
 363         base::TimeDelta::FromMilliseconds(
 364             service_->download_request_timeout_ms()));
 365   }
 366
 367   // Canceling a request will cause us to always report the result as SAFE
 368   // unless a pending request is about to call FinishRequest.
 369   void Cancel() {
 370     DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 371     if (fetcher_.get()) {
 372       // The DownloadProtectionService is going to release its reference, so we
 373       // might be destroyed before the URLFetcher completes.  Cancel the
 374       // fetcher so it does not try to invoke OnURLFetchComplete.
 375       fetcher_.reset();
 376     }
 377     // Note: If there is no fetcher, then some callback is still holding a
 378     // reference to this object.  We'll eventually wind up in some method on
 379     // the UI thread that will call FinishRequest() again.  If FinishRequest()
 380     // is called a second time, it will be a no-op.
 381     FinishRequest(SAFE, REASON_REQUEST_CANCELED);
 382     // Calling FinishRequest might delete this object, we may be deleted by
 383     // this point.
 384   }
 385
 386   // content::DownloadItem::Observer implementation.
 387   virtual void OnDownloadDestroyed(content::DownloadItem* download) OVERRIDE {
 388     Cancel();
 389     DCHECK(item_ == NULL);
 390   }
 391
 392   // From the net::URLFetcherDelegate interface.
 393   virtual void OnURLFetchComplete(const net::URLFetcher* source) OVERRIDE {
 394     DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 395     DCHECK_EQ(source, fetcher_.get());
 396     VLOG(2) << "Received a response for URL: "
 397             << item_->GetUrlChain().back() << ": success="
 398             << source->GetStatus().is_success() << " response_code="
 399             << source->GetResponseCode();
 400     if (source->GetStatus().is_success()) {
 401       UMA_HISTOGRAM_SPARSE_SLOWLY(
 402           "SBClientDownload.DownloadRequestResponseCode",
 403           source->GetResponseCode());
 404     }
 405     UMA_HISTOGRAM_SPARSE_SLOWLY(
 406         "SBClientDownload.DownloadRequestNetError",
 407         -source->GetStatus().error());
 408     DownloadCheckResultReason reason = REASON_SERVER_PING_FAILED;
 409     DownloadCheckResult result = SAFE;
 410     if (source->GetStatus().is_success() &&
 411         net::HTTP_OK == source->GetResponseCode()) {
 412       ClientDownloadResponse response;
 413       std::string data;
 414       bool got_data = source->GetResponseAsString(&data);
 415       DCHECK(got_data);
 416       if (!response.ParseFromString(data)) {
 417         reason = REASON_INVALID_RESPONSE_PROTO;
 418       } else if (response.verdict() == ClientDownloadResponse::SAFE) {
 419         reason = REASON_DOWNLOAD_SAFE;
 420       } else if (service_ && !service_->IsSupportedDownload(
 421           *item_, item_->GetTargetFilePath())) {
 422         // The client of the download protection service assumes that we don't
 423         // support this download so we cannot return any other verdict than
 424         // SAFE even if the server says it's dangerous to download this file.
 425         // Note: if service_ is NULL we already cancelled the request and
 426         // returned SAFE.
 427         reason = REASON_DOWNLOAD_NOT_SUPPORTED;
 428       } else if (response.verdict() == ClientDownloadResponse::DANGEROUS) {
 429         reason = REASON_DOWNLOAD_DANGEROUS;
 430         result = DANGEROUS;
 431       } else if (response.verdict() == ClientDownloadResponse::UNCOMMON) {
 432         reason = REASON_DOWNLOAD_UNCOMMON;
 433         result = UNCOMMON;
 434       } else if (response.verdict() == ClientDownloadResponse::DANGEROUS_HOST) {
 435         reason = REASON_DOWNLOAD_DANGEROUS_HOST;
 436         result = DANGEROUS_HOST;
 437       } else if (
 438           response.verdict() == ClientDownloadResponse::POTENTIALLY_UNWANTED) {
 439         reason = REASON_DOWNLOAD_POTENTIALLY_UNWANTED;
 440         result = POTENTIALLY_UNWANTED;
 441       } else {
 442         LOG(DFATAL) << "Unknown download response verdict: "
 443                     << response.verdict();
 444         reason = REASON_INVALID_RESPONSE_VERDICT;
 445       }
 446       DownloadFeedbackService::MaybeStorePingsForDownload(
 447           result, item_, client_download_request_data_, data);
 448     }
 449     // We don't need the fetcher anymore.
 450     fetcher_.reset();
 451     UMA_HISTOGRAM_TIMES("SBClientDownload.DownloadRequestDuration",
 452                         base::TimeTicks::Now() - start_time_);
 453     FinishRequest(result, reason);
 454   }
 455
 456   static bool IsSupportedDownload(const content::DownloadItem& item,
 457                                   const base::FilePath& target_path,
 458                                   DownloadCheckResultReason* reason,
 459                                   ClientDownloadRequest::DownloadType* type) {
 460     if (item.GetUrlChain().empty()) {
 461       *reason = REASON_EMPTY_URL_CHAIN;
 462       return false;
 463     }
 464     const GURL& final_url = item.GetUrlChain().back();
 465     if (!final_url.is_valid() || final_url.is_empty() ||
 466         !final_url.IsStandard() || final_url.SchemeIsFile()) {
 467       *reason = REASON_INVALID_URL;
 468       return false;
 469     }
 470     if (!download_protection_util::IsBinaryFile(target_path)) {
 471       *reason = REASON_NOT_BINARY_FILE;
 472       return false;
 473     }
 474     *type = GetDownloadType(target_path);
 475     return true;
 476   }
 477
 478  private:
 479   friend struct BrowserThread::DeleteOnThread<BrowserThread::UI>;
 480   friend class base::DeleteHelper<CheckClientDownloadRequest>;
 481
 482   virtual ~CheckClientDownloadRequest() {
 483     DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 484     DCHECK(item_ == NULL);
 485   }
 486
 487   void OnFileFeatureExtractionDone() {
 488     // This can run in any thread, since it just posts more messages.
 489
 490     // TODO(noelutz): DownloadInfo should also contain the IP address of
 491     // every URL in the redirect chain.  We also should check whether the
 492     // download URL is hosted on the internal network.
 493     BrowserThread::PostTask(
 494         BrowserThread::IO,
 495         FROM_HERE,
 496         base::Bind(&CheckClientDownloadRequest::CheckWhitelists, this));
 497
 498     // We wait until after the file checks finish to start the timeout, as
 499     // windows can cause permissions errors if the timeout fired while we were
 500     // checking the file signature and we tried to complete the download.
 501     BrowserThread::PostTask(
 502         BrowserThread::UI,
 503         FROM_HERE,
 504         base::Bind(&CheckClientDownloadRequest::StartTimeout, this));
 505   }
 506
 507   void StartExtractFileFeatures() {
 508     DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 509     DCHECK(item_);  // Called directly from Start(), item should still exist.
 510     // Since we do blocking I/O, offload this to a worker thread.
 511     // The task does not need to block shutdown.
 512     BrowserThread::GetBlockingPool()->PostWorkerTaskWithShutdownBehavior(
 513         FROM_HERE,
 514         base::Bind(&CheckClientDownloadRequest::ExtractFileFeatures,
 515                    this, item_->GetFullPath()),
 516         base::SequencedWorkerPool::CONTINUE_ON_SHUTDOWN);
 517   }
 518
 519   void ExtractFileFeatures(const base::FilePath& file_path) {
 520     base::TimeTicks start_time = base::TimeTicks::Now();
 521     binary_feature_extractor_->CheckSignature(file_path, &signature_info_);
 522     bool is_signed = (signature_info_.certificate_chain_size() > 0);
 523     if (is_signed) {
 524       VLOG(2) << "Downloaded a signed binary: " << file_path.value();
 525     } else {
 526       VLOG(2) << "Downloaded an unsigned binary: "
 527               << file_path.value();
 528     }
 529     UMA_HISTOGRAM_BOOLEAN("SBClientDownload.SignedBinaryDownload", is_signed);
 530     UMA_HISTOGRAM_TIMES("SBClientDownload.ExtractSignatureFeaturesTime",
 531                         base::TimeTicks::Now() - start_time);
 532
 533     start_time = base::TimeTicks::Now();
 534     binary_feature_extractor_->ExtractImageHeaders(file_path, &image_headers_);
 535     UMA_HISTOGRAM_TIMES("SBClientDownload.ExtractImageHeadersTime",
 536                         base::TimeTicks::Now() - start_time);
 537
 538     OnFileFeatureExtractionDone();
 539   }
 540
 541   void StartExtractZipFeatures() {
 542     DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 543     DCHECK(item_);  // Called directly from Start(), item should still exist.
 544     zip_analysis_start_time_ = base::TimeTicks::Now();
 545     // We give the zip analyzer a weak pointer to this object.  Since the
 546     // analyzer is refcounted, it might outlive the request.
 547     analyzer_ = new SandboxedZipAnalyzer(
 548         item_->GetFullPath(),
 549         base::Bind(&CheckClientDownloadRequest::OnZipAnalysisFinished,
 550                    weakptr_factory_.GetWeakPtr()));
 551     analyzer_->Start();
 552   }
 553
 554   void OnZipAnalysisFinished(const zip_analyzer::Results& results) {
 555     DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 556     if (!service_)
 557       return;
 558     if (results.success) {
 559       zipped_executable_ = results.has_executable;
 560       VLOG(1) << "Zip analysis finished for " << item_->GetFullPath().value()
 561               << ", has_executable=" << results.has_executable
 562               << " has_archive=" << results.has_archive;
 563     } else {
 564       VLOG(1) << "Zip analysis failed for " << item_->GetFullPath().value();
 565     }
 566     UMA_HISTOGRAM_BOOLEAN("SBClientDownload.ZipFileHasExecutable",
 567                           zipped_executable_);
 568     UMA_HISTOGRAM_BOOLEAN("SBClientDownload.ZipFileHasArchiveButNoExecutable",
 569                           results.has_archive && !zipped_executable_);
 570     UMA_HISTOGRAM_TIMES("SBClientDownload.ExtractZipFeaturesTime",
 571                         base::TimeTicks::Now() - zip_analysis_start_time_);
 572
 573     if (!zipped_executable_) {
 574       PostFinishTask(SAFE, REASON_ARCHIVE_WITHOUT_BINARIES);
 575       return;
 576     }
 577     OnFileFeatureExtractionDone();
 578   }
 579
 580   void CheckWhitelists() {
 581     DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 582     DownloadCheckResultReason reason = REASON_MAX;
 583     if (!database_manager_.get()) {
 584       reason = REASON_SB_DISABLED;
 585     } else {
 586       const GURL& url = url_chain_.back();
 587       if (url.is_valid() && database_manager_->MatchDownloadWhitelistUrl(url)) {
 588         VLOG(2) << url << " is on the download whitelist.";
 589         reason = REASON_WHITELISTED_URL;
 590       }
 591       if (reason != REASON_MAX || signature_info_.trusted()) {
 592         UMA_HISTOGRAM_COUNTS("SBClientDownload.SignedOrWhitelistedDownload", 1);
 593       }
 594     }
 595     if (reason == REASON_MAX && signature_info_.trusted()) {
 596       for (int i = 0; i < signature_info_.certificate_chain_size(); ++i) {
 597         if (CertificateChainIsWhitelisted(
 598                 signature_info_.certificate_chain(i))) {
 599           reason = REASON_TRUSTED_EXECUTABLE;
 600           break;
 601         }
 602       }
 603     }
 604     if (reason != REASON_MAX) {
 605       PostFinishTask(SAFE, reason);
 606     } else if (!pingback_enabled_) {
 607       PostFinishTask(SAFE, REASON_PING_DISABLED);
 608     } else {
 609       // Currently, the UI only works on Windows so we don't even bother
 610       // with pinging the server if we're not on Windows.  TODO(noelutz):
 611       // change this code once the UI is done for Linux and Mac.
 612 #if defined(OS_WIN)
 613       // The URLFetcher is owned by the UI thread, so post a message to
 614       // start the pingback.
 615       BrowserThread::PostTask(
 616           BrowserThread::UI,
 617           FROM_HERE,
 618           base::Bind(&CheckClientDownloadRequest::GetTabRedirects, this));
 619 #else
 620       PostFinishTask(SAFE, REASON_OS_NOT_SUPPORTED);
 621 #endif
 622     }
 623   }
 624
 625   void GetTabRedirects() {
 626     DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 627     if (!tab_url_.is_valid()) {
 628       SendRequest();
 629       return;
 630     }
 631
 632     Profile* profile = Profile::FromBrowserContext(item_->GetBrowserContext());
 633     HistoryService* history =
 634         HistoryServiceFactory::GetForProfile(profile, Profile::EXPLICIT_ACCESS);
 635     if (!history) {
 636       SendRequest();
 637       return;
 638     }
 639
 640     history->QueryRedirectsTo(
 641         tab_url_,
 642         &request_consumer_,
 643         base::Bind(&CheckClientDownloadRequest::OnGotTabRedirects,
 644                    base::Unretained(this)));
 645   }
 646
 647   void OnGotTabRedirects(HistoryService::Handle handle,
 648                          GURL url,
 649                          bool success,
 650                          history::RedirectList* redirect_list) {
 651     DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 652     DCHECK_EQ(url, tab_url_);
 653
 654     if (success && redirect_list->size() > 0) {
 655       for (history::RedirectList::reverse_iterator i = redirect_list->rbegin();
 656            i != redirect_list->rend();
 657            ++i) {
 658         tab_redirects_.push_back(*i);
 659       }
 660     }
 661
 662     SendRequest();
 663   }
 664
 665   void SendRequest() {
 666     DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 667
 668     // This is our last chance to check whether the request has been canceled
 669     // before sending it.
 670     if (!service_)
 671       return;
 672
 673     ClientDownloadRequest request;
 674     request.set_url(item_->GetUrlChain().back().spec());
 675     request.mutable_digests()->set_sha256(item_->GetHash());
 676     request.set_length(item_->GetReceivedBytes());
 677     for (size_t i = 0; i < item_->GetUrlChain().size(); ++i) {
 678       ClientDownloadRequest::Resource* resource = request.add_resources();
 679       resource->set_url(item_->GetUrlChain()[i].spec());
 680       if (i == item_->GetUrlChain().size() - 1) {
 681         // The last URL in the chain is the download URL.
 682         resource->set_type(ClientDownloadRequest::DOWNLOAD_URL);
 683         resource->set_referrer(item_->GetReferrerUrl().spec());
 684         DVLOG(2) << "dl url " << resource->url();
 685         if (!item_->GetRemoteAddress().empty()) {
 686           resource->set_remote_ip(item_->GetRemoteAddress());
 687           DVLOG(2) << "  dl url remote addr: " << resource->remote_ip();
 688         }
 689         DVLOG(2) << "dl referrer " << resource->referrer();
 690       } else {
 691         DVLOG(2) << "dl redirect " << i << " " << resource->url();
 692         resource->set_type(ClientDownloadRequest::DOWNLOAD_REDIRECT);
 693       }
 694       // TODO(noelutz): fill out the remote IP addresses.
 695     }
 696     // TODO(mattm): fill out the remote IP addresses for tab resources.
 697     for (size_t i = 0; i < tab_redirects_.size(); ++i) {
 698       ClientDownloadRequest::Resource* resource = request.add_resources();
 699       DVLOG(2) << "tab redirect " << i << " " << tab_redirects_[i].spec();
 700       resource->set_url(tab_redirects_[i].spec());
 701       resource->set_type(ClientDownloadRequest::TAB_REDIRECT);
 702     }
 703     if (tab_url_.is_valid()) {
 704       ClientDownloadRequest::Resource* resource = request.add_resources();
 705       resource->set_url(tab_url_.spec());
 706       DVLOG(2) << "tab url " << resource->url();
 707       resource->set_type(ClientDownloadRequest::TAB_URL);
 708       if (tab_referrer_url_.is_valid()) {
 709         resource->set_referrer(tab_referrer_url_.spec());
 710         DVLOG(2) << "tab referrer " << resource->referrer();
 711       }
 712     }
 713
 714     request.set_user_initiated(item_->HasUserGesture());
 715     request.set_file_basename(
 716         item_->GetTargetFilePath().BaseName().AsUTF8Unsafe());
 717     request.set_download_type(type_);
 718     request.mutable_signature()->CopyFrom(signature_info_);
 719     request.mutable_image_headers()->CopyFrom(image_headers_);
 720     if (!request.SerializeToString(&client_download_request_data_)) {
 721       FinishRequest(SAFE, REASON_INVALID_REQUEST_PROTO);
 722       return;
 723     }
 724
 725     VLOG(2) << "Sending a request for URL: "
 726             << item_->GetUrlChain().back();
 727     fetcher_.reset(net::URLFetcher::Create(0 /* ID used for testing */,
 728                                            GetDownloadRequestUrl(),
 729                                            net::URLFetcher::POST,
 730                                            this));
 731     fetcher_->SetLoadFlags(net::LOAD_DISABLE_CACHE);
 732     fetcher_->SetAutomaticallyRetryOn5xx(false);  // Don't retry on error.
 733     fetcher_->SetRequestContext(service_->request_context_getter_.get());
 734     fetcher_->SetUploadData("application/octet-stream",
 735                             client_download_request_data_);
 736     UMA_HISTOGRAM_COUNTS("SBClientDownload.DownloadRequestPayloadSize",
 737                          client_download_request_data_.size());
 738     fetcher_->Start();
 739   }
 740
 741   void PostFinishTask(DownloadCheckResult result,
 742                       DownloadCheckResultReason reason) {
 743     BrowserThread::PostTask(
 744         BrowserThread::UI,
 745         FROM_HERE,
 746         base::Bind(&CheckClientDownloadRequest::FinishRequest, this, result,
 747                    reason));
 748   }
 749
 750   void FinishRequest(DownloadCheckResult result,
 751                      DownloadCheckResultReason reason) {
 752     DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 753     if (finished_) {
 754       return;
 755     }
 756     finished_ = true;
 757     // Ensure the timeout task is cancelled while we still have a non-zero
 758     // refcount. (crbug.com/240449)
 759     weakptr_factory_.InvalidateWeakPtrs();
 760     if (service_) {
 761       VLOG(2) << "SafeBrowsing download verdict for: "
 762               << item_->DebugString(true) << " verdict:" << reason;
 763       UMA_HISTOGRAM_ENUMERATION("SBClientDownload.CheckDownloadStats",
 764                                 reason,
 765                                 REASON_MAX);
 766       callback_.Run(result);
 767       item_->RemoveObserver(this);
 768       item_ = NULL;
 769       DownloadProtectionService* service = service_;
 770       service_ = NULL;
 771       service->RequestFinished(this);
 772       // DownloadProtectionService::RequestFinished will decrement our refcount,
 773       // so we may be deleted now.
 774     } else {
 775       callback_.Run(SAFE);
 776     }
 777   }
 778
 779   bool CertificateChainIsWhitelisted(
 780       const ClientDownloadRequest_CertificateChain& chain) {
 781     DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 782     if (chain.element_size() < 2) {
 783       // We need to have both a signing certificate and its issuer certificate
 784       // present to construct a whitelist entry.
 785       return false;
 786     }
 787     scoped_refptr<net::X509Certificate> cert =
 788         net::X509Certificate::CreateFromBytes(
 789             chain.element(0).certificate().data(),
 790             chain.element(0).certificate().size());
 791     if (!cert.get()) {
 792       return false;
 793     }
 794
 795     for (int i = 1; i < chain.element_size(); ++i) {
 796       scoped_refptr<net::X509Certificate> issuer =
 797           net::X509Certificate::CreateFromBytes(
 798               chain.element(i).certificate().data(),
 799               chain.element(i).certificate().size());
 800       if (!issuer.get()) {
 801         return false;
 802       }
 803       std::vector<std::string> whitelist_strings;
 804       DownloadProtectionService::GetCertificateWhitelistStrings(
 805           *cert.get(), *issuer.get(), &whitelist_strings);
 806       for (size_t j = 0; j < whitelist_strings.size(); ++j) {
 807         if (database_manager_->MatchDownloadWhitelistString(
 808                 whitelist_strings[j])) {
 809           VLOG(2) << "Certificate matched whitelist, cert="
 810                   << cert->subject().GetDisplayName()
 811                   << " issuer=" << issuer->subject().GetDisplayName();
 812           return true;
 813         }
 814       }
 815       cert = issuer;
 816     }
 817     return false;
 818   }
 819
 820   // The DownloadItem we are checking. Will be NULL if the request has been
 821   // canceled. Must be accessed only on UI thread.
 822   content::DownloadItem* item_;
 823   // Copies of data from |item_| for access on other threads.
 824   std::vector<GURL> url_chain_;
 825   GURL referrer_url_;
 826   // URL chain of redirects leading to (but not including) |tab_url|.
 827   std::vector<GURL> tab_redirects_;
 828   // URL and referrer of the window the download was started from.
 829   GURL tab_url_;
 830   GURL tab_referrer_url_;
 831
 832   bool zipped_executable_;
 833   ClientDownloadRequest_SignatureInfo signature_info_;
 834   ClientDownloadRequest_ImageHeaders image_headers_;
 835   CheckDownloadCallback callback_;
 836   // Will be NULL if the request has been canceled.
 837   DownloadProtectionService* service_;
 838   scoped_refptr<BinaryFeatureExtractor> binary_feature_extractor_;
 839   scoped_refptr<SafeBrowsingDatabaseManager> database_manager_;
 840   const bool pingback_enabled_;
 841   scoped_ptr<net::URLFetcher> fetcher_;
 842   scoped_refptr<SandboxedZipAnalyzer> analyzer_;
 843   base::TimeTicks zip_analysis_start_time_;
 844   bool finished_;
 845   ClientDownloadRequest::DownloadType type_;
 846   std::string client_download_request_data_;
 847   CancelableRequestConsumer request_consumer_;  // For HistoryService lookup.
 848   base::WeakPtrFactory<CheckClientDownloadRequest> weakptr_factory_;
 849   base::TimeTicks start_time_;  // Used for stats.
 850
 851   DISALLOW_COPY_AND_ASSIGN(CheckClientDownloadRequest);
 852 };
 853
 854 DownloadProtectionService::DownloadProtectionService(
 855     SafeBrowsingService* sb_service,
 856     net::URLRequestContextGetter* request_context_getter)
 857     : request_context_getter_(request_context_getter),
 858       enabled_(false),
 859       binary_feature_extractor_(new BinaryFeatureExtractor()),
 860       download_request_timeout_ms_(kDownloadRequestTimeoutMs),
 861       feedback_service_(new DownloadFeedbackService(
 862           request_context_getter, BrowserThread::GetBlockingPool())) {
 863
 864   if (sb_service) {
 865     ui_manager_ = sb_service->ui_manager();
 866     database_manager_ = sb_service->database_manager();
 867   }
 868 }
 869
 870 DownloadProtectionService::~DownloadProtectionService() {
 871   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 872   CancelPendingRequests();
 873 }
 874
 875 void DownloadProtectionService::SetEnabled(bool enabled) {
 876   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 877   if (enabled == enabled_) {
 878     return;
 879   }
 880   enabled_ = enabled;
 881   if (!enabled_) {
 882     CancelPendingRequests();
 883   }
 884 }
 885
 886 void DownloadProtectionService::CheckClientDownload(
 887     content::DownloadItem* item,
 888     const CheckDownloadCallback& callback) {
 889   scoped_refptr<CheckClientDownloadRequest> request(
 890       new CheckClientDownloadRequest(item, callback, this,
 891                                      database_manager_,
 892                                      binary_feature_extractor_.get()));
 893   download_requests_.insert(request);
 894   request->Start();
 895 }
 896
 897 void DownloadProtectionService::CheckDownloadUrl(
 898     const content::DownloadItem& item,
 899     const CheckDownloadCallback& callback) {
 900   DCHECK(!item.GetUrlChain().empty());
 901   scoped_refptr<DownloadUrlSBClient> client(
 902       new DownloadUrlSBClient(item, callback, ui_manager_, database_manager_));
 903   // The client will release itself once it is done.
 904   BrowserThread::PostTask(
 905         BrowserThread::IO,
 906         FROM_HERE,
 907         base::Bind(&DownloadUrlSBClient::StartCheck, client));
 908 }
 909
 910 bool DownloadProtectionService::IsSupportedDownload(
 911     const content::DownloadItem& item,
 912     const base::FilePath& target_path) const {
 913   // Currently, the UI only works on Windows.  On Linux and Mac we still
 914   // want to show the dangerous file type warning if the file is possibly
 915   // dangerous which means we have to always return false here.
 916 #if defined(OS_WIN)
 917   DownloadCheckResultReason reason = REASON_MAX;
 918   ClientDownloadRequest::DownloadType type =
 919       ClientDownloadRequest::WIN_EXECUTABLE;
 920   return (CheckClientDownloadRequest::IsSupportedDownload(item, target_path,
 921                                                           &reason, &type) &&
 922           (ClientDownloadRequest::ANDROID_APK == type ||
 923            ClientDownloadRequest::WIN_EXECUTABLE == type ||
 924            ClientDownloadRequest::ZIPPED_EXECUTABLE == type));
 925 #else
 926   return false;
 927 #endif
 928 }
 929
 930 void DownloadProtectionService::CancelPendingRequests() {
 931   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 932   for (std::set<scoped_refptr<CheckClientDownloadRequest> >::iterator it =
 933            download_requests_.begin();
 934        it != download_requests_.end();) {
 935     // We need to advance the iterator before we cancel because canceling
 936     // the request will invalidate it when RequestFinished is called below.
 937     scoped_refptr<CheckClientDownloadRequest> tmp = *it++;
 938     tmp->Cancel();
 939   }
 940   DCHECK(download_requests_.empty());
 941 }
 942
 943 void DownloadProtectionService::RequestFinished(
 944     CheckClientDownloadRequest* request) {
 945   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 946   std::set<scoped_refptr<CheckClientDownloadRequest> >::iterator it =
 947       download_requests_.find(request);
 948   DCHECK(it != download_requests_.end());
 949   download_requests_.erase(*it);
 950 }
 951
 952 void DownloadProtectionService::ShowDetailsForDownload(
 953     const content::DownloadItem& item,
 954     content::PageNavigator* navigator) {
 955   GURL learn_more_url(chrome::kDownloadScanningLearnMoreURL);
 956   navigator->OpenURL(
 957       content::OpenURLParams(learn_more_url,
 958                              content::Referrer(),
 959                              NEW_FOREGROUND_TAB,
 960                              content::PAGE_TRANSITION_LINK,
 961                              false));
 962 }
 963
 964 namespace {
 965 // Escapes a certificate attribute so that it can be used in a whitelist
 966 // entry.  Currently, we only escape slashes, since they are used as a
 967 // separator between attributes.
 968 std::string EscapeCertAttribute(const std::string& attribute) {
 969   std::string escaped;
 970   for (size_t i = 0; i < attribute.size(); ++i) {
 971     if (attribute[i] == '%') {
 972       escaped.append("%25");
 973     } else if (attribute[i] == '/') {
 974       escaped.append("%2F");
 975     } else {
 976       escaped.push_back(attribute[i]);
 977     }
 978   }
 979   return escaped;
 980 }
 981 }  // namespace
 982
 983 // static
 984 void DownloadProtectionService::GetCertificateWhitelistStrings(
 985     const net::X509Certificate& certificate,
 986     const net::X509Certificate& issuer,
 987     std::vector<std::string>* whitelist_strings) {
 988   // The whitelist paths are in the format:
 989   // cert/<ascii issuer fingerprint>[/CN=common_name][/O=org][/OU=unit]
 990   //
 991   // Any of CN, O, or OU may be omitted from the whitelist entry, in which
 992   // case they match anything.  However, the attributes that do appear will
 993   // always be in the order shown above.  At least one attribute will always
 994   // be present.
 995
 996   const net::CertPrincipal& subject = certificate.subject();
 997   std::vector<std::string> ou_tokens;
 998   for (size_t i = 0; i < subject.organization_unit_names.size(); ++i) {
 999     ou_tokens.push_back(
1000         "/OU=" + EscapeCertAttribute(subject.organization_unit_names[i]));
1001   }
1002
1003   std::vector<std::string> o_tokens;
1004   for (size_t i = 0; i < subject.organization_names.size(); ++i) {
1005     o_tokens.push_back(
1006         "/O=" + EscapeCertAttribute(subject.organization_names[i]));
1007   }
1008
1009   std::string cn_token;
1010   if (!subject.common_name.empty()) {
1011     cn_token = "/CN=" + EscapeCertAttribute(subject.common_name);
1012   }
1013
1014   std::set<std::string> paths_to_check;
1015   if (!cn_token.empty()) {
1016     paths_to_check.insert(cn_token);
1017   }
1018   for (size_t i = 0; i < o_tokens.size(); ++i) {
1019     paths_to_check.insert(cn_token + o_tokens[i]);
1020     paths_to_check.insert(o_tokens[i]);
1021     for (size_t j = 0; j < ou_tokens.size(); ++j) {
1022       paths_to_check.insert(cn_token + o_tokens[i] + ou_tokens[j]);
1023       paths_to_check.insert(o_tokens[i] + ou_tokens[j]);
1024     }
1025   }
1026   for (size_t i = 0; i < ou_tokens.size(); ++i) {
1027     paths_to_check.insert(cn_token + ou_tokens[i]);
1028     paths_to_check.insert(ou_tokens[i]);
1029   }
1030
1031   std::string issuer_fp = base::HexEncode(issuer.fingerprint().data,
1032                                           sizeof(issuer.fingerprint().data));
1033   for (std::set<std::string>::iterator it = paths_to_check.begin();
1034        it != paths_to_check.end(); ++it) {
1035     whitelist_strings->push_back("cert/" + issuer_fp + *it);
1036   }
1037 }
1038
1039 // static
1040 GURL DownloadProtectionService::GetDownloadRequestUrl() {
1041   GURL url(kDownloadRequestUrl);
1042   std::string api_key = google_apis::GetAPIKey();
1043   if (!api_key.empty())
1044     url = url.Resolve("?key=" + net::EscapeQueryParamValue(api_key, true));
1045
1046   return url;
1047 }
1048
1049 }  // namespace safe_browsing