chrome/browser/safe_browsing/download_protection_service.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "chrome/browser/safe_browsing/download_protection_service.h"
   6
   7 #include "base/bind.h"
   8 #include "base/compiler_specific.h"
   9 #include "base/format_macros.h"
  10 #include "base/memory/scoped_ptr.h"
  11 #include "base/memory/weak_ptr.h"
  12 #include "base/metrics/field_trial.h"
  13 #include "base/metrics/histogram.h"
  14 #include "base/metrics/sparse_histogram.h"
  15 #include "base/sequenced_task_runner_helpers.h"
  16 #include "base/stl_util.h"
  17 #include "base/strings/string_number_conversions.h"
  18 #include "base/strings/string_util.h"
  19 #include "base/strings/stringprintf.h"
  20 #include "base/task/cancelable_task_tracker.h"
  21 #include "base/threading/sequenced_worker_pool.h"
  22 #include "base/time/time.h"
  23 #include "chrome/browser/browser_process.h"
  24 #include "chrome/browser/history/history_service_factory.h"
  25 #include "chrome/browser/profiles/profile.h"
  26 #include "chrome/browser/safe_browsing/download_feedback_service.h"
  27 #include "chrome/browser/safe_browsing/safe_browsing_service.h"
  28 #include "chrome/browser/safe_browsing/sandboxed_zip_analyzer.h"
  29 #include "chrome/browser/ui/browser.h"
  30 #include "chrome/browser/ui/browser_list.h"
  31 #include "chrome/common/safe_browsing/binary_feature_extractor.h"
  32 #include "chrome/common/safe_browsing/csd.pb.h"
  33 #include "chrome/common/safe_browsing/download_protection_util.h"
  34 #include "chrome/common/safe_browsing/zip_analyzer_results.h"
  35 #include "chrome/common/url_constants.h"
  36 #include "components/google/core/browser/google_util.h"
  37 #include "components/history/core/browser/history_service.h"
  38 #include "content/public/browser/browser_thread.h"
  39 #include "content/public/browser/download_item.h"
  40 #include "content/public/browser/page_navigator.h"
  41 #include "crypto/sha2.h"
  42 #include "google_apis/google_api_keys.h"
  43 #include "net/base/escape.h"
  44 #include "net/base/load_flags.h"
  45 #include "net/base/url_util.h"
  46 #include "net/cert/x509_cert_types.h"
  47 #include "net/cert/x509_certificate.h"
  48 #include "net/http/http_status_code.h"
  49 #include "net/url_request/url_fetcher.h"
  50 #include "net/url_request/url_fetcher_delegate.h"
  51 #include "net/url_request/url_request_context_getter.h"
  52 #include "net/url_request/url_request_status.h"
  53
  54 using content::BrowserThread;
  55
  56 namespace {
  57 static const int64 kDownloadRequestTimeoutMs = 7000;
  58 }  // namespace
  59
  60 namespace safe_browsing {
  61
  62 const char DownloadProtectionService::kDownloadRequestUrl[] =
  63     "https://sb-ssl.google.com/safebrowsing/clientreport/download";
  64
  65 namespace {
  66 void RecordFileExtensionType(const base::FilePath& file) {
  67   UMA_HISTOGRAM_ENUMERATION(
  68       "SBClientDownload.DownloadExtensions",
  69       download_protection_util::GetSBClientDownloadExtensionValueForUMA(file),
  70       download_protection_util::kSBClientDownloadExtensionsMax);
  71 }
  72
  73 void RecordArchivedArchiveFileExtensionType(
  74     const base::FilePath::StringType& extension) {
  75   UMA_HISTOGRAM_ENUMERATION(
  76       "SBClientDownload.ArchivedArchiveExtensions",
  77       download_protection_util::GetSBClientDownloadExtensionValueForUMA(
  78           base::FilePath(extension)),
  79       download_protection_util::kSBClientDownloadExtensionsMax);
  80 }
  81
  82 // Enumerate for histogramming purposes.
  83 // DO NOT CHANGE THE ORDERING OF THESE VALUES (different histogram data will
  84 // be mixed together based on their values).
  85 enum SBStatsType {
  86   DOWNLOAD_URL_CHECKS_TOTAL,
  87   DOWNLOAD_URL_CHECKS_CANCELED,
  88   DOWNLOAD_URL_CHECKS_MALWARE,
  89
  90   DOWNLOAD_HASH_CHECKS_TOTAL,
  91   DOWNLOAD_HASH_CHECKS_MALWARE,
  92
  93   // Memory space for histograms is determined by the max.
  94   // ALWAYS ADD NEW VALUES BEFORE THIS ONE.
  95   DOWNLOAD_CHECKS_MAX
  96 };
  97
  98 // Prepares URLs to be put into a ping message. Currently this just shortens
  99 // data: URIs, other URLs are included verbatim.
 100 std::string SanitizeUrl(const GURL& url) {
 101   std::string spec = url.spec();
 102   if (url.SchemeIs(url::kDataScheme)) {
 103     size_t comma_pos = spec.find(',');
 104     if (comma_pos != std::string::npos && comma_pos != spec.size() - 1) {
 105       std::string hash_value = crypto::SHA256HashString(spec);
 106       spec.erase(comma_pos + 1);
 107       spec += base::HexEncode(hash_value.data(), hash_value.size());
 108     }
 109   }
 110   return spec;
 111 }
 112
 113 }  // namespace
 114
 115 // Parent SafeBrowsing::Client class used to lookup the bad binary
 116 // URL and digest list.  There are two sub-classes (one for each list).
 117 class DownloadSBClient
 118     : public SafeBrowsingDatabaseManager::Client,
 119       public base::RefCountedThreadSafe<DownloadSBClient> {
 120  public:
 121   DownloadSBClient(
 122       const content::DownloadItem& item,
 123       const DownloadProtectionService::CheckDownloadCallback& callback,
 124       const scoped_refptr<SafeBrowsingUIManager>& ui_manager,
 125       SBStatsType total_type,
 126       SBStatsType dangerous_type)
 127       : sha256_hash_(item.GetHash()),
 128         url_chain_(item.GetUrlChain()),
 129         referrer_url_(item.GetReferrerUrl()),
 130         callback_(callback),
 131         ui_manager_(ui_manager),
 132         start_time_(base::TimeTicks::Now()),
 133         total_type_(total_type),
 134         dangerous_type_(dangerous_type) {}
 135
 136   virtual void StartCheck() = 0;
 137   virtual bool IsDangerous(SBThreatType threat_type) const = 0;
 138
 139  protected:
 140   friend class base::RefCountedThreadSafe<DownloadSBClient>;
 141   ~DownloadSBClient() override {}
 142
 143   void CheckDone(SBThreatType threat_type) {
 144     DownloadProtectionService::DownloadCheckResult result =
 145         IsDangerous(threat_type) ?
 146         DownloadProtectionService::DANGEROUS :
 147         DownloadProtectionService::SAFE;
 148     BrowserThread::PostTask(BrowserThread::UI,
 149                             FROM_HERE,
 150                             base::Bind(callback_, result));
 151     UpdateDownloadCheckStats(total_type_);
 152     if (threat_type != SB_THREAT_TYPE_SAFE) {
 153       UpdateDownloadCheckStats(dangerous_type_);
 154       BrowserThread::PostTask(
 155           BrowserThread::UI,
 156           FROM_HERE,
 157           base::Bind(&DownloadSBClient::ReportMalware,
 158                      this, threat_type));
 159     }
 160   }
 161
 162   void ReportMalware(SBThreatType threat_type) {
 163     std::string post_data;
 164     if (!sha256_hash_.empty())
 165       post_data += base::HexEncode(sha256_hash_.data(),
 166                                    sha256_hash_.size()) + "\n";
 167     for (size_t i = 0; i < url_chain_.size(); ++i) {
 168       post_data += url_chain_[i].spec() + "\n";
 169     }
 170     ui_manager_->ReportSafeBrowsingHit(
 171         url_chain_.back(),  // malicious_url
 172         url_chain_.front(), // page_url
 173         referrer_url_,
 174         true,  // is_subresource
 175         threat_type,
 176         post_data);
 177   }
 178
 179   void UpdateDownloadCheckStats(SBStatsType stat_type) {
 180     UMA_HISTOGRAM_ENUMERATION("SB2.DownloadChecks",
 181                               stat_type,
 182                               DOWNLOAD_CHECKS_MAX);
 183   }
 184
 185   std::string sha256_hash_;
 186   std::vector<GURL> url_chain_;
 187   GURL referrer_url_;
 188   DownloadProtectionService::CheckDownloadCallback callback_;
 189   scoped_refptr<SafeBrowsingUIManager> ui_manager_;
 190   base::TimeTicks start_time_;
 191
 192  private:
 193   const SBStatsType total_type_;
 194   const SBStatsType dangerous_type_;
 195
 196   DISALLOW_COPY_AND_ASSIGN(DownloadSBClient);
 197 };
 198
 199 class DownloadUrlSBClient : public DownloadSBClient {
 200  public:
 201   DownloadUrlSBClient(
 202       const content::DownloadItem& item,
 203       const DownloadProtectionService::CheckDownloadCallback& callback,
 204       const scoped_refptr<SafeBrowsingUIManager>& ui_manager,
 205       const scoped_refptr<SafeBrowsingDatabaseManager>& database_manager)
 206       : DownloadSBClient(item, callback, ui_manager,
 207                          DOWNLOAD_URL_CHECKS_TOTAL,
 208                          DOWNLOAD_URL_CHECKS_MALWARE),
 209         database_manager_(database_manager) { }
 210
 211   void StartCheck() override {
 212     DCHECK_CURRENTLY_ON(BrowserThread::IO);
 213     if (!database_manager_.get() ||
 214         database_manager_->CheckDownloadUrl(url_chain_, this)) {
 215       CheckDone(SB_THREAT_TYPE_SAFE);
 216     } else {
 217       AddRef();  // SafeBrowsingService takes a pointer not a scoped_refptr.
 218     }
 219   }
 220
 221   bool IsDangerous(SBThreatType threat_type) const override {
 222     return threat_type == SB_THREAT_TYPE_BINARY_MALWARE_URL;
 223   }
 224
 225   void OnCheckDownloadUrlResult(const std::vector<GURL>& url_chain,
 226                                 SBThreatType threat_type) override {
 227     CheckDone(threat_type);
 228     UMA_HISTOGRAM_TIMES("SB2.DownloadUrlCheckDuration",
 229                         base::TimeTicks::Now() - start_time_);
 230     Release();
 231   }
 232
 233  protected:
 234   ~DownloadUrlSBClient() override {}
 235
 236  private:
 237   scoped_refptr<SafeBrowsingDatabaseManager> database_manager_;
 238
 239   DISALLOW_COPY_AND_ASSIGN(DownloadUrlSBClient);
 240 };
 241
 242 class DownloadProtectionService::CheckClientDownloadRequest
 243     : public base::RefCountedThreadSafe<
 244           DownloadProtectionService::CheckClientDownloadRequest,
 245           BrowserThread::DeleteOnUIThread>,
 246       public net::URLFetcherDelegate,
 247       public content::DownloadItem::Observer {
 248  public:
 249   CheckClientDownloadRequest(
 250       content::DownloadItem* item,
 251       const CheckDownloadCallback& callback,
 252       DownloadProtectionService* service,
 253       const scoped_refptr<SafeBrowsingDatabaseManager>& database_manager,
 254       BinaryFeatureExtractor* binary_feature_extractor)
 255       : item_(item),
 256         url_chain_(item->GetUrlChain()),
 257         referrer_url_(item->GetReferrerUrl()),
 258         tab_url_(item->GetTabUrl()),
 259         tab_referrer_url_(item->GetTabReferrerUrl()),
 260         zipped_executable_(false),
 261         callback_(callback),
 262         service_(service),
 263         binary_feature_extractor_(binary_feature_extractor),
 264         database_manager_(database_manager),
 265         pingback_enabled_(service_->enabled()),
 266         finished_(false),
 267         type_(ClientDownloadRequest::WIN_EXECUTABLE),
 268         start_time_(base::TimeTicks::Now()),
 269         weakptr_factory_(this) {
 270     DCHECK_CURRENTLY_ON(BrowserThread::UI);
 271     item_->AddObserver(this);
 272   }
 273
 274   void Start() {
 275     DVLOG(2) << "Starting SafeBrowsing download check for: "
 276              << item_->DebugString(true);
 277     DCHECK_CURRENTLY_ON(BrowserThread::UI);
 278     // TODO(noelutz): implement some cache to make sure we don't issue the same
 279     // request over and over again if a user downloads the same binary multiple
 280     // times.
 281     DownloadCheckResultReason reason = REASON_MAX;
 282     if (!IsSupportedDownload(
 283         *item_, item_->GetTargetFilePath(), &reason, &type_)) {
 284       switch (reason) {
 285         case REASON_EMPTY_URL_CHAIN:
 286         case REASON_INVALID_URL:
 287         case REASON_UNSUPPORTED_URL_SCHEME:
 288           PostFinishTask(UNKNOWN, reason);
 289           return;
 290
 291         case REASON_NOT_BINARY_FILE:
 292           RecordFileExtensionType(item_->GetTargetFilePath());
 293           PostFinishTask(UNKNOWN, reason);
 294           return;
 295
 296         default:
 297           // We only expect the reasons explicitly handled above.
 298           NOTREACHED();
 299       }
 300     }
 301     RecordFileExtensionType(item_->GetTargetFilePath());
 302
 303     // Compute features from the file contents. Note that we record histograms
 304     // based on the result, so this runs regardless of whether the pingbacks
 305     // are enabled.
 306     if (item_->GetTargetFilePath().MatchesExtension(
 307         FILE_PATH_LITERAL(".zip"))) {
 308       StartExtractZipFeatures();
 309     } else {
 310       DCHECK(!download_protection_util::IsArchiveFile(
 311           item_->GetTargetFilePath()));
 312       StartExtractFileFeatures();
 313     }
 314   }
 315
 316   // Start a timeout to cancel the request if it takes too long.
 317   // This should only be called after we have finished accessing the file.
 318   void StartTimeout() {
 319     DCHECK_CURRENTLY_ON(BrowserThread::UI);
 320     if (!service_) {
 321       // Request has already been cancelled.
 322       return;
 323     }
 324     timeout_start_time_ = base::TimeTicks::Now();
 325     BrowserThread::PostDelayedTask(
 326         BrowserThread::UI,
 327         FROM_HERE,
 328         base::Bind(&CheckClientDownloadRequest::Cancel,
 329                    weakptr_factory_.GetWeakPtr()),
 330         base::TimeDelta::FromMilliseconds(
 331             service_->download_request_timeout_ms()));
 332   }
 333
 334   // Canceling a request will cause us to always report the result as UNKNOWN
 335   // unless a pending request is about to call FinishRequest.
 336   void Cancel() {
 337     DCHECK_CURRENTLY_ON(BrowserThread::UI);
 338     if (fetcher_.get()) {
 339       // The DownloadProtectionService is going to release its reference, so we
 340       // might be destroyed before the URLFetcher completes.  Cancel the
 341       // fetcher so it does not try to invoke OnURLFetchComplete.
 342       fetcher_.reset();
 343     }
 344     // Note: If there is no fetcher, then some callback is still holding a
 345     // reference to this object.  We'll eventually wind up in some method on
 346     // the UI thread that will call FinishRequest() again.  If FinishRequest()
 347     // is called a second time, it will be a no-op.
 348     FinishRequest(UNKNOWN, REASON_REQUEST_CANCELED);
 349     // Calling FinishRequest might delete this object, we may be deleted by
 350     // this point.
 351   }
 352
 353   // content::DownloadItem::Observer implementation.
 354   void OnDownloadDestroyed(content::DownloadItem* download) override {
 355     Cancel();
 356     DCHECK(item_ == NULL);
 357   }
 358
 359   // From the net::URLFetcherDelegate interface.
 360   void OnURLFetchComplete(const net::URLFetcher* source) override {
 361     DCHECK_CURRENTLY_ON(BrowserThread::UI);
 362     DCHECK_EQ(source, fetcher_.get());
 363     DVLOG(2) << "Received a response for URL: "
 364              << item_->GetUrlChain().back() << ": success="
 365              << source->GetStatus().is_success() << " response_code="
 366              << source->GetResponseCode();
 367     if (source->GetStatus().is_success()) {
 368       UMA_HISTOGRAM_SPARSE_SLOWLY(
 369           "SBClientDownload.DownloadRequestResponseCode",
 370           source->GetResponseCode());
 371     }
 372     UMA_HISTOGRAM_SPARSE_SLOWLY(
 373         "SBClientDownload.DownloadRequestNetError",
 374         -source->GetStatus().error());
 375     DownloadCheckResultReason reason = REASON_SERVER_PING_FAILED;
 376     DownloadCheckResult result = UNKNOWN;
 377     if (source->GetStatus().is_success() &&
 378         net::HTTP_OK == source->GetResponseCode()) {
 379       ClientDownloadResponse response;
 380       std::string data;
 381       bool got_data = source->GetResponseAsString(&data);
 382       DCHECK(got_data);
 383       if (!response.ParseFromString(data)) {
 384         reason = REASON_INVALID_RESPONSE_PROTO;
 385         result = UNKNOWN;
 386       } else if (response.verdict() == ClientDownloadResponse::SAFE) {
 387         reason = REASON_DOWNLOAD_SAFE;
 388         result = SAFE;
 389       } else if (service_ && !service_->IsSupportedDownload(
 390           *item_, item_->GetTargetFilePath())) {
 391         // The client of the download protection service assumes that we don't
 392         // support this download so we cannot return any other verdict than
 393         // UNKNOWN even if the server says it's dangerous to download this file.
 394         // Note: if service_ is NULL we already cancelled the request and
 395         // returned UNKNOWN.
 396         reason = REASON_DOWNLOAD_NOT_SUPPORTED;
 397         result = UNKNOWN;
 398       } else if (response.verdict() == ClientDownloadResponse::DANGEROUS) {
 399         reason = REASON_DOWNLOAD_DANGEROUS;
 400         result = DANGEROUS;
 401       } else if (response.verdict() == ClientDownloadResponse::UNCOMMON) {
 402         reason = REASON_DOWNLOAD_UNCOMMON;
 403         result = UNCOMMON;
 404       } else if (response.verdict() == ClientDownloadResponse::DANGEROUS_HOST) {
 405         reason = REASON_DOWNLOAD_DANGEROUS_HOST;
 406         result = DANGEROUS_HOST;
 407       } else if (
 408           response.verdict() == ClientDownloadResponse::POTENTIALLY_UNWANTED) {
 409         reason = REASON_DOWNLOAD_POTENTIALLY_UNWANTED;
 410         result = POTENTIALLY_UNWANTED;
 411       } else {
 412         LOG(DFATAL) << "Unknown download response verdict: "
 413                     << response.verdict();
 414         reason = REASON_INVALID_RESPONSE_VERDICT;
 415         result = UNKNOWN;
 416       }
 417       DownloadFeedbackService::MaybeStorePingsForDownload(
 418           result, item_, client_download_request_data_, data);
 419     }
 420     // We don't need the fetcher anymore.
 421     fetcher_.reset();
 422     UMA_HISTOGRAM_TIMES("SBClientDownload.DownloadRequestDuration",
 423                         base::TimeTicks::Now() - start_time_);
 424     UMA_HISTOGRAM_TIMES("SBClientDownload.DownloadRequestNetworkDuration",
 425                         base::TimeTicks::Now() - request_start_time_);
 426     FinishRequest(result, reason);
 427   }
 428
 429   static bool IsSupportedDownload(const content::DownloadItem& item,
 430                                   const base::FilePath& target_path,
 431                                   DownloadCheckResultReason* reason,
 432                                   ClientDownloadRequest::DownloadType* type) {
 433     if (item.GetUrlChain().empty()) {
 434       *reason = REASON_EMPTY_URL_CHAIN;
 435       return false;
 436     }
 437     const GURL& final_url = item.GetUrlChain().back();
 438     if (!final_url.is_valid() || final_url.is_empty()) {
 439       *reason = REASON_INVALID_URL;
 440       return false;
 441     }
 442     if (!download_protection_util::IsSupportedBinaryFile(target_path)) {
 443       *reason = REASON_NOT_BINARY_FILE;
 444       return false;
 445     }
 446     if ((!final_url.IsStandard() && !final_url.SchemeIsBlob() &&
 447          !final_url.SchemeIs(url::kDataScheme)) ||
 448         final_url.SchemeIsFile()) {
 449       *reason = REASON_UNSUPPORTED_URL_SCHEME;
 450       return false;
 451     }
 452     *type = download_protection_util::GetDownloadType(target_path);
 453     return true;
 454   }
 455
 456  private:
 457   friend struct BrowserThread::DeleteOnThread<BrowserThread::UI>;
 458   friend class base::DeleteHelper<CheckClientDownloadRequest>;
 459
 460   ~CheckClientDownloadRequest() override {
 461     DCHECK_CURRENTLY_ON(BrowserThread::UI);
 462     DCHECK(item_ == NULL);
 463   }
 464
 465   void OnFileFeatureExtractionDone() {
 466     // This can run in any thread, since it just posts more messages.
 467
 468     // TODO(noelutz): DownloadInfo should also contain the IP address of
 469     // every URL in the redirect chain.  We also should check whether the
 470     // download URL is hosted on the internal network.
 471     BrowserThread::PostTask(
 472         BrowserThread::IO,
 473         FROM_HERE,
 474         base::Bind(&CheckClientDownloadRequest::CheckWhitelists, this));
 475
 476     // We wait until after the file checks finish to start the timeout, as
 477     // windows can cause permissions errors if the timeout fired while we were
 478     // checking the file signature and we tried to complete the download.
 479     BrowserThread::PostTask(
 480         BrowserThread::UI,
 481         FROM_HERE,
 482         base::Bind(&CheckClientDownloadRequest::StartTimeout, this));
 483   }
 484
 485   void StartExtractFileFeatures() {
 486     DCHECK_CURRENTLY_ON(BrowserThread::UI);
 487     DCHECK(item_);  // Called directly from Start(), item should still exist.
 488     // Since we do blocking I/O, offload this to a worker thread.
 489     // The task does not need to block shutdown.
 490     BrowserThread::GetBlockingPool()->PostWorkerTaskWithShutdownBehavior(
 491         FROM_HERE,
 492         base::Bind(&CheckClientDownloadRequest::ExtractFileFeatures,
 493                    this, item_->GetFullPath()),
 494         base::SequencedWorkerPool::CONTINUE_ON_SHUTDOWN);
 495   }
 496
 497   void ExtractFileFeatures(const base::FilePath& file_path) {
 498     base::TimeTicks start_time = base::TimeTicks::Now();
 499     binary_feature_extractor_->CheckSignature(file_path, &signature_info_);
 500     bool is_signed = (signature_info_.certificate_chain_size() > 0);
 501     if (is_signed) {
 502       DVLOG(2) << "Downloaded a signed binary: " << file_path.value();
 503     } else {
 504       DVLOG(2) << "Downloaded an unsigned binary: "
 505                << file_path.value();
 506     }
 507     UMA_HISTOGRAM_BOOLEAN("SBClientDownload.SignedBinaryDownload", is_signed);
 508     UMA_HISTOGRAM_TIMES("SBClientDownload.ExtractSignatureFeaturesTime",
 509                         base::TimeTicks::Now() - start_time);
 510
 511     start_time = base::TimeTicks::Now();
 512     image_headers_.reset(new ClientDownloadRequest_ImageHeaders());
 513     if (!binary_feature_extractor_->ExtractImageFeatures(
 514             file_path,
 515             BinaryFeatureExtractor::kDefaultOptions,
 516             image_headers_.get(),
 517             nullptr /* signed_data */)) {
 518       image_headers_.reset();
 519     }
 520     UMA_HISTOGRAM_TIMES("SBClientDownload.ExtractImageHeadersTime",
 521                         base::TimeTicks::Now() - start_time);
 522
 523     OnFileFeatureExtractionDone();
 524   }
 525
 526   void StartExtractZipFeatures() {
 527     DCHECK_CURRENTLY_ON(BrowserThread::UI);
 528     DCHECK(item_);  // Called directly from Start(), item should still exist.
 529     zip_analysis_start_time_ = base::TimeTicks::Now();
 530     // We give the zip analyzer a weak pointer to this object.  Since the
 531     // analyzer is refcounted, it might outlive the request.
 532     analyzer_ = new SandboxedZipAnalyzer(
 533         item_->GetFullPath(),
 534         base::Bind(&CheckClientDownloadRequest::OnZipAnalysisFinished,
 535                    weakptr_factory_.GetWeakPtr()));
 536     analyzer_->Start();
 537   }
 538
 539   void OnZipAnalysisFinished(const zip_analyzer::Results& results) {
 540     DCHECK_CURRENTLY_ON(BrowserThread::UI);
 541     DCHECK_EQ(ClientDownloadRequest::ZIPPED_EXECUTABLE, type_);
 542     if (!service_)
 543       return;
 544     if (results.success) {
 545       zipped_executable_ = results.has_executable;
 546       archived_binary_.CopyFrom(results.archived_binary);
 547       DVLOG(1) << "Zip analysis finished for " << item_->GetFullPath().value()
 548                << ", has_executable=" << results.has_executable
 549                << " has_archive=" << results.has_archive;
 550     } else {
 551       DVLOG(1) << "Zip analysis failed for " << item_->GetFullPath().value();
 552     }
 553     UMA_HISTOGRAM_BOOLEAN("SBClientDownload.ZipFileHasExecutable",
 554                           zipped_executable_);
 555     UMA_HISTOGRAM_BOOLEAN("SBClientDownload.ZipFileHasArchiveButNoExecutable",
 556                           results.has_archive && !zipped_executable_);
 557     UMA_HISTOGRAM_TIMES("SBClientDownload.ExtractZipFeaturesTime",
 558                         base::TimeTicks::Now() - zip_analysis_start_time_);
 559     for (const auto& file_extension : results.archived_archive_filetypes)
 560       RecordArchivedArchiveFileExtensionType(file_extension);
 561
 562     if (!zipped_executable_ && !results.has_archive) {
 563       PostFinishTask(UNKNOWN, REASON_ARCHIVE_WITHOUT_BINARIES);
 564       return;
 565     }
 566
 567     if (!zipped_executable_ && results.has_archive)
 568       type_ = ClientDownloadRequest::ZIPPED_ARCHIVE;
 569     OnFileFeatureExtractionDone();
 570   }
 571
 572   static void RecordCountOfSignedOrWhitelistedDownload() {
 573     UMA_HISTOGRAM_COUNTS("SBClientDownload.SignedOrWhitelistedDownload", 1);
 574   }
 575
 576   void CheckWhitelists() {
 577     DCHECK_CURRENTLY_ON(BrowserThread::IO);
 578
 579     if (!database_manager_.get()) {
 580       PostFinishTask(UNKNOWN, REASON_SB_DISABLED);
 581       return;
 582     }
 583
 584     const GURL& url = url_chain_.back();
 585     // TODO(asanka): This may acquire a lock on the SB DB on the IO thread.
 586     if (url.is_valid() && database_manager_->MatchDownloadWhitelistUrl(url)) {
 587       DVLOG(2) << url << " is on the download whitelist.";
 588       RecordCountOfSignedOrWhitelistedDownload();
 589       // TODO(grt): Continue processing without uploading so that
 590       // ClientDownloadRequest callbacks can be run even for this type of safe
 591       // download.
 592       PostFinishTask(SAFE, REASON_WHITELISTED_URL);
 593       return;
 594     }
 595
 596     if (signature_info_.trusted()) {
 597       RecordCountOfSignedOrWhitelistedDownload();
 598       for (int i = 0; i < signature_info_.certificate_chain_size(); ++i) {
 599         if (CertificateChainIsWhitelisted(
 600                 signature_info_.certificate_chain(i))) {
 601           // TODO(grt): Continue processing without uploading so that
 602           // ClientDownloadRequest callbacks can be run even for this type of
 603           // safe download.
 604           PostFinishTask(SAFE, REASON_TRUSTED_EXECUTABLE);
 605           return;
 606         }
 607       }
 608     }
 609
 610     if (!pingback_enabled_) {
 611       PostFinishTask(UNKNOWN, REASON_PING_DISABLED);
 612       return;
 613     }
 614
 615     // Currently, the UI is only enabled on Windows and OSX so we don't even
 616     // bother with pinging the server if we're not on one of those platforms.
 617     // TODO(noelutz): change this code once the UI is done for Linux.
 618 #if defined(OS_WIN) || defined(OS_MACOSX)
 619     // The URLFetcher is owned by the UI thread, so post a message to
 620     // start the pingback.
 621     BrowserThread::PostTask(
 622         BrowserThread::UI,
 623         FROM_HERE,
 624         base::Bind(&CheckClientDownloadRequest::GetTabRedirects, this));
 625 #else
 626     PostFinishTask(UNKNOWN, REASON_OS_NOT_SUPPORTED);
 627 #endif
 628   }
 629
 630   void GetTabRedirects() {
 631     DCHECK_CURRENTLY_ON(BrowserThread::UI);
 632     if (!service_)
 633       return;
 634
 635     if (!tab_url_.is_valid()) {
 636       SendRequest();
 637       return;
 638     }
 639
 640     Profile* profile = Profile::FromBrowserContext(item_->GetBrowserContext());
 641     history::HistoryService* history = HistoryServiceFactory::GetForProfile(
 642         profile, ServiceAccessType::EXPLICIT_ACCESS);
 643     if (!history) {
 644       SendRequest();
 645       return;
 646     }
 647
 648     history->QueryRedirectsTo(
 649         tab_url_,
 650         base::Bind(&CheckClientDownloadRequest::OnGotTabRedirects,
 651                    base::Unretained(this),
 652                    tab_url_),
 653         &request_tracker_);
 654   }
 655
 656   void OnGotTabRedirects(const GURL& url,
 657                          const history::RedirectList* redirect_list) {
 658     DCHECK_CURRENTLY_ON(BrowserThread::UI);
 659     DCHECK_EQ(url, tab_url_);
 660     if (!service_)
 661       return;
 662
 663     if (!redirect_list->empty()) {
 664       tab_redirects_.insert(
 665           tab_redirects_.end(), redirect_list->rbegin(), redirect_list->rend());
 666     }
 667
 668     SendRequest();
 669   }
 670
 671   void SendRequest() {
 672     DCHECK_CURRENTLY_ON(BrowserThread::UI);
 673
 674     // This is our last chance to check whether the request has been canceled
 675     // before sending it.
 676     if (!service_)
 677       return;
 678
 679     ClientDownloadRequest request;
 680     request.set_url(SanitizeUrl(item_->GetUrlChain().back()));
 681     request.mutable_digests()->set_sha256(item_->GetHash());
 682     request.set_length(item_->GetReceivedBytes());
 683     for (size_t i = 0; i < item_->GetUrlChain().size(); ++i) {
 684       ClientDownloadRequest::Resource* resource = request.add_resources();
 685       resource->set_url(SanitizeUrl(item_->GetUrlChain()[i]));
 686       if (i == item_->GetUrlChain().size() - 1) {
 687         // The last URL in the chain is the download URL.
 688         resource->set_type(ClientDownloadRequest::DOWNLOAD_URL);
 689         resource->set_referrer(SanitizeUrl(item_->GetReferrerUrl()));
 690         DVLOG(2) << "dl url " << resource->url();
 691         if (!item_->GetRemoteAddress().empty()) {
 692           resource->set_remote_ip(item_->GetRemoteAddress());
 693           DVLOG(2) << "  dl url remote addr: " << resource->remote_ip();
 694         }
 695         DVLOG(2) << "dl referrer " << resource->referrer();
 696       } else {
 697         DVLOG(2) << "dl redirect " << i << " " << resource->url();
 698         resource->set_type(ClientDownloadRequest::DOWNLOAD_REDIRECT);
 699       }
 700       // TODO(noelutz): fill out the remote IP addresses.
 701     }
 702     // TODO(mattm): fill out the remote IP addresses for tab resources.
 703     for (size_t i = 0; i < tab_redirects_.size(); ++i) {
 704       ClientDownloadRequest::Resource* resource = request.add_resources();
 705       DVLOG(2) << "tab redirect " << i << " " << tab_redirects_[i].spec();
 706       resource->set_url(SanitizeUrl(tab_redirects_[i]));
 707       resource->set_type(ClientDownloadRequest::TAB_REDIRECT);
 708     }
 709     if (tab_url_.is_valid()) {
 710       ClientDownloadRequest::Resource* resource = request.add_resources();
 711       resource->set_url(SanitizeUrl(tab_url_));
 712       DVLOG(2) << "tab url " << resource->url();
 713       resource->set_type(ClientDownloadRequest::TAB_URL);
 714       if (tab_referrer_url_.is_valid()) {
 715         resource->set_referrer(SanitizeUrl(tab_referrer_url_));
 716         DVLOG(2) << "tab referrer " << resource->referrer();
 717       }
 718     }
 719
 720     request.set_user_initiated(item_->HasUserGesture());
 721     request.set_file_basename(
 722         item_->GetTargetFilePath().BaseName().AsUTF8Unsafe());
 723     request.set_download_type(type_);
 724     request.mutable_signature()->CopyFrom(signature_info_);
 725     if (image_headers_)
 726       request.set_allocated_image_headers(image_headers_.release());
 727     if (zipped_executable_)
 728       request.mutable_archived_binary()->Swap(&archived_binary_);
 729     if (!request.SerializeToString(&client_download_request_data_)) {
 730       FinishRequest(UNKNOWN, REASON_INVALID_REQUEST_PROTO);
 731       return;
 732     }
 733     service_->client_download_request_callbacks_.Notify(item_, &request);
 734
 735     DVLOG(2) << "Sending a request for URL: "
 736              << item_->GetUrlChain().back();
 737     fetcher_ = net::URLFetcher::Create(0 /* ID used for testing */,
 738                                        GetDownloadRequestUrl(),
 739                                        net::URLFetcher::POST, this);
 740     fetcher_->SetLoadFlags(net::LOAD_DISABLE_CACHE);
 741     fetcher_->SetAutomaticallyRetryOn5xx(false);  // Don't retry on error.
 742     fetcher_->SetRequestContext(service_->request_context_getter_.get());
 743     fetcher_->SetUploadData("application/octet-stream",
 744                             client_download_request_data_);
 745     request_start_time_ = base::TimeTicks::Now();
 746     UMA_HISTOGRAM_COUNTS("SBClientDownload.DownloadRequestPayloadSize",
 747                          client_download_request_data_.size());
 748     fetcher_->Start();
 749   }
 750
 751   void PostFinishTask(DownloadCheckResult result,
 752                       DownloadCheckResultReason reason) {
 753     BrowserThread::PostTask(
 754         BrowserThread::UI,
 755         FROM_HERE,
 756         base::Bind(&CheckClientDownloadRequest::FinishRequest, this, result,
 757                    reason));
 758   }
 759
 760   void FinishRequest(DownloadCheckResult result,
 761                      DownloadCheckResultReason reason) {
 762     DCHECK_CURRENTLY_ON(BrowserThread::UI);
 763     if (finished_) {
 764       return;
 765     }
 766     finished_ = true;
 767     // Ensure the timeout task is cancelled while we still have a non-zero
 768     // refcount. (crbug.com/240449)
 769     weakptr_factory_.InvalidateWeakPtrs();
 770     if (!request_start_time_.is_null()) {
 771       UMA_HISTOGRAM_ENUMERATION("SBClientDownload.DownloadRequestNetworkStats",
 772                                 reason,
 773                                 REASON_MAX);
 774     }
 775     if (!timeout_start_time_.is_null()) {
 776       UMA_HISTOGRAM_ENUMERATION("SBClientDownload.DownloadRequestTimeoutStats",
 777                                 reason,
 778                                 REASON_MAX);
 779       if (reason != REASON_REQUEST_CANCELED) {
 780         UMA_HISTOGRAM_TIMES("SBClientDownload.DownloadRequestTimeoutDuration",
 781                             base::TimeTicks::Now() - timeout_start_time_);
 782       }
 783     }
 784     if (service_) {
 785       DVLOG(2) << "SafeBrowsing download verdict for: "
 786                << item_->DebugString(true) << " verdict:" << reason
 787                << " result:" << result;
 788       UMA_HISTOGRAM_ENUMERATION("SBClientDownload.CheckDownloadStats",
 789                                 reason,
 790                                 REASON_MAX);
 791       callback_.Run(result);
 792       item_->RemoveObserver(this);
 793       item_ = NULL;
 794       DownloadProtectionService* service = service_;
 795       service_ = NULL;
 796       service->RequestFinished(this);
 797       // DownloadProtectionService::RequestFinished will decrement our refcount,
 798       // so we may be deleted now.
 799     } else {
 800       callback_.Run(UNKNOWN);
 801     }
 802   }
 803
 804   bool CertificateChainIsWhitelisted(
 805       const ClientDownloadRequest_CertificateChain& chain) {
 806     DCHECK_CURRENTLY_ON(BrowserThread::IO);
 807     if (chain.element_size() < 2) {
 808       // We need to have both a signing certificate and its issuer certificate
 809       // present to construct a whitelist entry.
 810       return false;
 811     }
 812     scoped_refptr<net::X509Certificate> cert =
 813         net::X509Certificate::CreateFromBytes(
 814             chain.element(0).certificate().data(),
 815             chain.element(0).certificate().size());
 816     if (!cert.get()) {
 817       return false;
 818     }
 819
 820     for (int i = 1; i < chain.element_size(); ++i) {
 821       scoped_refptr<net::X509Certificate> issuer =
 822           net::X509Certificate::CreateFromBytes(
 823               chain.element(i).certificate().data(),
 824               chain.element(i).certificate().size());
 825       if (!issuer.get()) {
 826         return false;
 827       }
 828       std::vector<std::string> whitelist_strings;
 829       DownloadProtectionService::GetCertificateWhitelistStrings(
 830           *cert.get(), *issuer.get(), &whitelist_strings);
 831       for (size_t j = 0; j < whitelist_strings.size(); ++j) {
 832         if (database_manager_->MatchDownloadWhitelistString(
 833                 whitelist_strings[j])) {
 834           DVLOG(2) << "Certificate matched whitelist, cert="
 835                    << cert->subject().GetDisplayName()
 836                    << " issuer=" << issuer->subject().GetDisplayName();
 837           return true;
 838         }
 839       }
 840       cert = issuer;
 841     }
 842     return false;
 843   }
 844
 845   // The DownloadItem we are checking. Will be NULL if the request has been
 846   // canceled. Must be accessed only on UI thread.
 847   content::DownloadItem* item_;
 848   // Copies of data from |item_| for access on other threads.
 849   std::vector<GURL> url_chain_;
 850   GURL referrer_url_;
 851   // URL chain of redirects leading to (but not including) |tab_url|.
 852   std::vector<GURL> tab_redirects_;
 853   // URL and referrer of the window the download was started from.
 854   GURL tab_url_;
 855   GURL tab_referrer_url_;
 856
 857   bool zipped_executable_;
 858   ClientDownloadRequest_SignatureInfo signature_info_;
 859   scoped_ptr<ClientDownloadRequest_ImageHeaders> image_headers_;
 860   google::protobuf::RepeatedPtrField<ClientDownloadRequest_ArchivedBinary>
 861       archived_binary_;
 862   CheckDownloadCallback callback_;
 863   // Will be NULL if the request has been canceled.
 864   DownloadProtectionService* service_;
 865   scoped_refptr<BinaryFeatureExtractor> binary_feature_extractor_;
 866   scoped_refptr<SafeBrowsingDatabaseManager> database_manager_;
 867   const bool pingback_enabled_;
 868   scoped_ptr<net::URLFetcher> fetcher_;
 869   scoped_refptr<SandboxedZipAnalyzer> analyzer_;
 870   base::TimeTicks zip_analysis_start_time_;
 871   bool finished_;
 872   ClientDownloadRequest::DownloadType type_;
 873   std::string client_download_request_data_;
 874   base::CancelableTaskTracker request_tracker_;  // For HistoryService lookup.
 875   base::TimeTicks start_time_;  // Used for stats.
 876   base::TimeTicks timeout_start_time_;
 877   base::TimeTicks request_start_time_;
 878   base::WeakPtrFactory<CheckClientDownloadRequest> weakptr_factory_;
 879
 880   DISALLOW_COPY_AND_ASSIGN(CheckClientDownloadRequest);
 881 };
 882
 883 DownloadProtectionService::DownloadProtectionService(
 884     SafeBrowsingService* sb_service,
 885     net::URLRequestContextGetter* request_context_getter)
 886     : request_context_getter_(request_context_getter),
 887       enabled_(false),
 888       binary_feature_extractor_(new BinaryFeatureExtractor()),
 889       download_request_timeout_ms_(kDownloadRequestTimeoutMs),
 890       feedback_service_(new DownloadFeedbackService(
 891           request_context_getter, BrowserThread::GetBlockingPool())) {
 892
 893   if (sb_service) {
 894     ui_manager_ = sb_service->ui_manager();
 895     database_manager_ = sb_service->database_manager();
 896   }
 897 }
 898
 899 DownloadProtectionService::~DownloadProtectionService() {
 900   DCHECK_CURRENTLY_ON(BrowserThread::UI);
 901   CancelPendingRequests();
 902 }
 903
 904 void DownloadProtectionService::SetEnabled(bool enabled) {
 905   DCHECK_CURRENTLY_ON(BrowserThread::UI);
 906   if (enabled == enabled_) {
 907     return;
 908   }
 909   enabled_ = enabled;
 910   if (!enabled_) {
 911     CancelPendingRequests();
 912   }
 913 }
 914
 915 void DownloadProtectionService::CheckClientDownload(
 916     content::DownloadItem* item,
 917     const CheckDownloadCallback& callback) {
 918   scoped_refptr<CheckClientDownloadRequest> request(
 919       new CheckClientDownloadRequest(item, callback, this,
 920                                      database_manager_,
 921                                      binary_feature_extractor_.get()));
 922   download_requests_.insert(request);
 923   request->Start();
 924 }
 925
 926 void DownloadProtectionService::CheckDownloadUrl(
 927     const content::DownloadItem& item,
 928     const CheckDownloadCallback& callback) {
 929   DCHECK(!item.GetUrlChain().empty());
 930   scoped_refptr<DownloadUrlSBClient> client(
 931       new DownloadUrlSBClient(item, callback, ui_manager_, database_manager_));
 932   // The client will release itself once it is done.
 933   BrowserThread::PostTask(
 934         BrowserThread::IO,
 935         FROM_HERE,
 936         base::Bind(&DownloadUrlSBClient::StartCheck, client));
 937 }
 938
 939 bool DownloadProtectionService::IsSupportedDownload(
 940     const content::DownloadItem& item,
 941     const base::FilePath& target_path) const {
 942   // Currently, the UI is only enabled on Windows and OSX.  On Linux we still
 943   // want to show the dangerous file type warning if the file is possibly
 944   // dangerous which means we have to always return false here.
 945 #if defined(OS_WIN) || defined(OS_MACOSX)
 946   DownloadCheckResultReason reason = REASON_MAX;
 947   ClientDownloadRequest::DownloadType type =
 948       ClientDownloadRequest::WIN_EXECUTABLE;
 949   return (CheckClientDownloadRequest::IsSupportedDownload(
 950               item, target_path, &reason, &type) &&
 951           (ClientDownloadRequest::CHROME_EXTENSION != type));
 952 #else
 953   return false;
 954 #endif
 955 }
 956
 957 DownloadProtectionService::ClientDownloadRequestSubscription
 958 DownloadProtectionService::RegisterClientDownloadRequestCallback(
 959     const ClientDownloadRequestCallback& callback) {
 960   DCHECK_CURRENTLY_ON(BrowserThread::UI);
 961   return client_download_request_callbacks_.Add(callback);
 962 }
 963
 964 void DownloadProtectionService::CancelPendingRequests() {
 965   DCHECK_CURRENTLY_ON(BrowserThread::UI);
 966   for (std::set<scoped_refptr<CheckClientDownloadRequest> >::iterator it =
 967            download_requests_.begin();
 968        it != download_requests_.end();) {
 969     // We need to advance the iterator before we cancel because canceling
 970     // the request will invalidate it when RequestFinished is called below.
 971     scoped_refptr<CheckClientDownloadRequest> tmp = *it++;
 972     tmp->Cancel();
 973   }
 974   DCHECK(download_requests_.empty());
 975 }
 976
 977 void DownloadProtectionService::RequestFinished(
 978     CheckClientDownloadRequest* request) {
 979   DCHECK_CURRENTLY_ON(BrowserThread::UI);
 980   std::set<scoped_refptr<CheckClientDownloadRequest> >::iterator it =
 981       download_requests_.find(request);
 982   DCHECK(it != download_requests_.end());
 983   download_requests_.erase(*it);
 984 }
 985
 986 void DownloadProtectionService::ShowDetailsForDownload(
 987     const content::DownloadItem& item,
 988     content::PageNavigator* navigator) {
 989   GURL learn_more_url(chrome::kDownloadScanningLearnMoreURL);
 990   learn_more_url = google_util::AppendGoogleLocaleParam(
 991       learn_more_url, g_browser_process->GetApplicationLocale());
 992   learn_more_url = net::AppendQueryParameter(
 993       learn_more_url, "ctx",
 994       base::IntToString(static_cast<int>(item.GetDangerType())));
 995   navigator->OpenURL(
 996       content::OpenURLParams(learn_more_url,
 997                              content::Referrer(),
 998                              NEW_FOREGROUND_TAB,
 999                              ui::PAGE_TRANSITION_LINK,
1000                              false));
1001 }
1002
1003 namespace {
1004 // Escapes a certificate attribute so that it can be used in a whitelist
1005 // entry.  Currently, we only escape slashes, since they are used as a
1006 // separator between attributes.
1007 std::string EscapeCertAttribute(const std::string& attribute) {
1008   std::string escaped;
1009   for (size_t i = 0; i < attribute.size(); ++i) {
1010     if (attribute[i] == '%') {
1011       escaped.append("%25");
1012     } else if (attribute[i] == '/') {
1013       escaped.append("%2F");
1014     } else {
1015       escaped.push_back(attribute[i]);
1016     }
1017   }
1018   return escaped;
1019 }
1020 }  // namespace
1021
1022 // static
1023 void DownloadProtectionService::GetCertificateWhitelistStrings(
1024     const net::X509Certificate& certificate,
1025     const net::X509Certificate& issuer,
1026     std::vector<std::string>* whitelist_strings) {
1027   // The whitelist paths are in the format:
1028   // cert/<ascii issuer fingerprint>[/CN=common_name][/O=org][/OU=unit]
1029   //
1030   // Any of CN, O, or OU may be omitted from the whitelist entry, in which
1031   // case they match anything.  However, the attributes that do appear will
1032   // always be in the order shown above.  At least one attribute will always
1033   // be present.
1034
1035   const net::CertPrincipal& subject = certificate.subject();
1036   std::vector<std::string> ou_tokens;
1037   for (size_t i = 0; i < subject.organization_unit_names.size(); ++i) {
1038     ou_tokens.push_back(
1039         "/OU=" + EscapeCertAttribute(subject.organization_unit_names[i]));
1040   }
1041
1042   std::vector<std::string> o_tokens;
1043   for (size_t i = 0; i < subject.organization_names.size(); ++i) {
1044     o_tokens.push_back(
1045         "/O=" + EscapeCertAttribute(subject.organization_names[i]));
1046   }
1047
1048   std::string cn_token;
1049   if (!subject.common_name.empty()) {
1050     cn_token = "/CN=" + EscapeCertAttribute(subject.common_name);
1051   }
1052
1053   std::set<std::string> paths_to_check;
1054   if (!cn_token.empty()) {
1055     paths_to_check.insert(cn_token);
1056   }
1057   for (size_t i = 0; i < o_tokens.size(); ++i) {
1058     paths_to_check.insert(cn_token + o_tokens[i]);
1059     paths_to_check.insert(o_tokens[i]);
1060     for (size_t j = 0; j < ou_tokens.size(); ++j) {
1061       paths_to_check.insert(cn_token + o_tokens[i] + ou_tokens[j]);
1062       paths_to_check.insert(o_tokens[i] + ou_tokens[j]);
1063     }
1064   }
1065   for (size_t i = 0; i < ou_tokens.size(); ++i) {
1066     paths_to_check.insert(cn_token + ou_tokens[i]);
1067     paths_to_check.insert(ou_tokens[i]);
1068   }
1069
1070   std::string issuer_fp = base::HexEncode(issuer.fingerprint().data,
1071                                           sizeof(issuer.fingerprint().data));
1072   for (std::set<std::string>::iterator it = paths_to_check.begin();
1073        it != paths_to_check.end(); ++it) {
1074     whitelist_strings->push_back("cert/" + issuer_fp + *it);
1075   }
1076 }
1077
1078 // static
1079 GURL DownloadProtectionService::GetDownloadRequestUrl() {
1080   GURL url(kDownloadRequestUrl);
1081   std::string api_key = google_apis::GetAPIKey();
1082   if (!api_key.empty())
1083     url = url.Resolve("?key=" + net::EscapeQueryParamValue(api_key, true));
1084
1085   return url;
1086 }
1087
1088 }  // namespace safe_browsing