1 // Copyright 2014 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "components/component_updater/background_downloader_win.h"
14 #include "base/file_util.h"
15 #include "base/message_loop/message_loop_proxy.h"
16 #include "base/single_thread_task_runner.h"
17 #include "base/strings/sys_string_conversions.h"
18 #include "base/win/scoped_co_mem.h"
19 #include "components/component_updater/component_updater_utils.h"
20 #include "ui/base/win/atl_module.h"
23 using base::win::ScopedCoMem
;
24 using base::win::ScopedComPtr
;
26 // The class BackgroundDownloader in this module is an adapter between
27 // the CrxDownloader interface and the BITS service interfaces.
28 // The interface exposed on the CrxDownloader code runs on the main thread,
29 // while the BITS specific code runs on a separate thread passed in by the
30 // client. For every url to download, a BITS job is created, unless there is
31 // already an existing job for that url, in which case, the downloader
32 // connects to it. Once a job is associated with the url, the code looks for
33 // changes in the BITS job state. The checks are triggered by a timer.
34 // The BITS job contains just one file to download. There could only be one
35 // download in progress at a time. If Chrome closes down before the download is
36 // complete, the BITS job remains active and finishes in the background, without
37 // any intervention. The job can be completed next time the code runs, if the
38 // file is still needed, otherwise it will be cleaned up on a periodic basis.
40 // To list the BITS jobs for a user, use the |bitsadmin| tool. The command line
41 // to do that is: "bitsadmin /list /verbose". Another useful command is
42 // "bitsadmin /info" and provide the job id returned by the previous /list
45 // Ignoring the suspend/resume issues since this code is not using them, the
46 // job state machine implemented by BITS is something like this:
48 // Suspended--->Queued--->Connecting---->Transferring--->Transferred
50 // | | V V | (complete)
51 // +----------|---------+-----------------+-----+ V
52 // | | | | Acknowledged
54 // | Transient Error------->Error |
56 // | +-------+---------+--->-+
59 // +------<----------+ +---->Cancelled
61 // The job is created in the "suspended" state. Once |Resume| is called,
62 // BITS queues up the job, then tries to connect, begins transferring the
63 // job bytes, and moves the job to the "transferred state, after the job files
64 // have been transferred. When calling |Complete| for a job, the job files are
65 // made available to the caller, and the job is moved to the "acknowledged"
67 // At any point, the job can be cancelled, in which case, the job is moved
68 // to the "cancelled state" and the job object is removed from the BITS queue.
69 // Along the way, the job can encounter recoverable and non-recoverable errors.
70 // BITS moves the job to "transient error" or "error", depending on which kind
71 // of error has occured.
72 // If the job has reached the "transient error" state, BITS retries the
73 // job after a certain programmable delay. If the job can't be completed in a
74 // certain time interval, BITS stops retrying and errors the job out. This time
75 // interval is also programmable.
76 // If the job is in either of the error states, the job parameters can be
77 // adjusted to handle the error, after which the job can be resumed, and the
78 // whole cycle starts again.
79 // Jobs that are not touched in 90 days (or a value set by group policy) are
80 // automatically disposed off by BITS. This concludes the brief description of
81 // a job lifetime, according to BITS.
83 // In addition to how BITS is managing the life time of the job, there are a
84 // couple of special cases defined by the BackgroundDownloader.
85 // First, if the job encounters any of the 5xx HTTP responses, the job is
86 // not retried, in order to avoid DDOS-ing the servers.
87 // Second, there is a simple mechanism to detect stuck jobs, and allow the rest
88 // of the code to move on to trying other urls or trying other components.
89 // Last, after completing a job, irrespective of the outcome, the jobs older
90 // than a week are proactively cleaned up.
92 namespace component_updater
{
96 // All jobs created by this module have a specific description so they can
97 // be found at run-time or by using system administration tools.
98 const base::char16 kJobDescription
[] = L
"Chrome Component Updater";
100 // How often the code looks for changes in the BITS job state.
101 const int kJobPollingIntervalSec
= 4;
103 // How long BITS waits before retrying a job after the job encountered
104 // a transient error. If this value is not set, the BITS default is 10 minutes.
105 const int kMinimumRetryDelayMin
= 1;
107 // How long to wait for stuck jobs. Stuck jobs could be queued for too long,
108 // have trouble connecting, could be suspended for any reason, or they have
109 // encountered some transient error.
110 const int kJobStuckTimeoutMin
= 15;
112 // How long BITS waits before giving up on a job that could not be completed
113 // since the job has encountered its first transient error. If this value is
114 // not set, the BITS default is 14 days.
115 const int kSetNoProgressTimeoutDays
= 1;
117 // How often the jobs which were started but not completed for any reason
118 // are cleaned up. Reasons for jobs to be left behind include browser restarts,
119 // system restarts, etc. Also, the check to purge stale jobs only happens
120 // at most once a day. If the job clean up code is not running, the BITS
121 // default policy is to cancel jobs after 90 days of inactivity.
122 const int kPurgeStaleJobsAfterDays
= 7;
123 const int kPurgeStaleJobsIntervalBetweenChecksDays
= 1;
125 // Returns the status code from a given BITS error.
126 int GetHttpStatusFromBitsError(HRESULT error
) {
127 // BITS errors are defined in bitsmsg.h. Although not documented, it is
128 // clear that all errors corresponding to http status code have the high
129 // word equal to 0x8019 and the low word equal to the http status code.
130 const int kHttpStatusFirst
= 100; // Continue.
131 const int kHttpStatusLast
= 505; // Version not supported.
132 bool is_valid
= HIWORD(error
) == 0x8019 &&
133 LOWORD(error
) >= kHttpStatusFirst
&&
134 LOWORD(error
) <= kHttpStatusLast
;
135 return is_valid
? LOWORD(error
) : 0;
138 // Returns the files in a BITS job.
139 HRESULT
GetFilesInJob(IBackgroundCopyJob
* job
,
140 std::vector
<ScopedComPtr
<IBackgroundCopyFile
> >* files
) {
141 ScopedComPtr
<IEnumBackgroundCopyFiles
> enum_files
;
142 HRESULT hr
= job
->EnumFiles(enum_files
.Receive());
147 hr
= enum_files
->GetCount(&num_files
);
151 for (ULONG i
= 0; i
!= num_files
; ++i
) {
152 ScopedComPtr
<IBackgroundCopyFile
> file
;
153 if (enum_files
->Next(1, file
.Receive(), NULL
) == S_OK
&& file
)
154 files
->push_back(file
);
160 // Returns the file name, the url, and some per-file progress information.
161 // The function out parameters can be NULL if that data is not requested.
162 HRESULT
GetJobFileProperties(IBackgroundCopyFile
* file
,
163 base::string16
* local_name
,
164 base::string16
* remote_name
,
165 BG_FILE_PROGRESS
* progress
) {
172 ScopedCoMem
<base::char16
> name
;
173 hr
= file
->GetLocalName(&name
);
176 local_name
->assign(name
);
180 ScopedCoMem
<base::char16
> name
;
181 hr
= file
->GetRemoteName(&name
);
184 remote_name
->assign(name
);
188 BG_FILE_PROGRESS bg_file_progress
= {};
189 hr
= file
->GetProgress(&bg_file_progress
);
192 *progress
= bg_file_progress
;
198 // Returns the number of bytes downloaded and bytes to download for all files
199 // in the job. If the values are not known or if an error has occurred,
200 // a value of -1 is reported.
201 HRESULT
GetJobByteCount(IBackgroundCopyJob
* job
,
202 int64
* downloaded_bytes
,
203 int64
* total_bytes
) {
204 *downloaded_bytes
= -1;
210 BG_JOB_PROGRESS job_progress
= {0};
211 HRESULT hr
= job
->GetProgress(&job_progress
);
215 if (job_progress
.BytesTransferred
<= kint64max
)
216 *downloaded_bytes
= job_progress
.BytesTransferred
;
218 if (job_progress
.BytesTotal
<= kint64max
&&
219 job_progress
.BytesTotal
!= BG_SIZE_UNKNOWN
)
220 *total_bytes
= job_progress
.BytesTotal
;
225 HRESULT
GetJobDescription(IBackgroundCopyJob
* job
, const base::string16
* name
) {
226 ScopedCoMem
<base::char16
> description
;
227 return job
->GetDescription(&description
);
230 // Returns the job error code in |error_code| if the job is in the transient
231 // or the final error state. Otherwise, the job error is not available and
232 // the function fails.
233 HRESULT
GetJobError(IBackgroundCopyJob
* job
, HRESULT
* error_code_out
) {
234 *error_code_out
= S_OK
;
235 ScopedComPtr
<IBackgroundCopyError
> copy_error
;
236 HRESULT hr
= job
->GetError(copy_error
.Receive());
240 BG_ERROR_CONTEXT error_context
= BG_ERROR_CONTEXT_NONE
;
241 HRESULT error_code
= S_OK
;
242 hr
= copy_error
->GetError(&error_context
, &error_code
);
246 *error_code_out
= FAILED(error_code
) ? error_code
: E_FAIL
;
250 // Finds the component updater jobs matching the given predicate.
251 // Returns S_OK if the function has found at least one job, returns S_FALSE if
252 // no job was found, and it returns an error otherwise.
253 template <class Predicate
>
254 HRESULT
FindBitsJobIf(Predicate pred
,
255 IBackgroundCopyManager
* bits_manager
,
256 std::vector
<ScopedComPtr
<IBackgroundCopyJob
> >* jobs
) {
257 ScopedComPtr
<IEnumBackgroundCopyJobs
> enum_jobs
;
258 HRESULT hr
= bits_manager
->EnumJobs(0, enum_jobs
.Receive());
263 hr
= enum_jobs
->GetCount(&job_count
);
267 // Iterate over jobs, run the predicate, and select the job only if
268 // the job description matches the component updater jobs.
269 for (ULONG i
= 0; i
!= job_count
; ++i
) {
270 ScopedComPtr
<IBackgroundCopyJob
> current_job
;
271 if (enum_jobs
->Next(1, current_job
.Receive(), NULL
) == S_OK
&&
273 base::string16 job_description
;
274 hr
= GetJobDescription(current_job
, &job_description
);
275 if (job_description
.compare(kJobDescription
) == 0)
276 jobs
->push_back(current_job
);
280 return jobs
->empty() ? S_FALSE
: S_OK
;
283 // Compares the job creation time and returns true if the job creation time
284 // is older than |num_days|.
285 struct JobCreationOlderThanDays
286 : public std::binary_function
<IBackgroundCopyJob
*, int, bool> {
287 bool operator()(IBackgroundCopyJob
* job
, int num_days
) const;
290 bool JobCreationOlderThanDays::operator()(IBackgroundCopyJob
* job
,
291 int num_days
) const {
292 BG_JOB_TIMES times
= {0};
293 HRESULT hr
= job
->GetTimes(×
);
297 const base::TimeDelta
time_delta(base::TimeDelta::FromDays(num_days
));
298 const base::Time
creation_time(base::Time::FromFileTime(times
.CreationTime
));
300 return creation_time
+ time_delta
< base::Time::Now();
303 // Compares the url of a file in a job and returns true if the remote name
304 // of any file in a job matches the argument.
305 struct JobFileUrlEqual
: public std::binary_function
<IBackgroundCopyJob
*,
306 const base::string16
&,
308 bool operator()(IBackgroundCopyJob
* job
,
309 const base::string16
& remote_name
) const;
312 bool JobFileUrlEqual::operator()(IBackgroundCopyJob
* job
,
313 const base::string16
& remote_name
) const {
314 std::vector
<ScopedComPtr
<IBackgroundCopyFile
> > files
;
315 HRESULT hr
= GetFilesInJob(job
, &files
);
319 for (size_t i
= 0; i
!= files
.size(); ++i
) {
320 ScopedCoMem
<base::char16
> name
;
321 if (SUCCEEDED(files
[i
]->GetRemoteName(&name
)) &&
322 remote_name
.compare(name
) == 0)
329 // Creates an instance of the BITS manager.
330 HRESULT
GetBitsManager(IBackgroundCopyManager
** bits_manager
) {
331 ScopedComPtr
<IBackgroundCopyManager
> object
;
332 HRESULT hr
= object
.CreateInstance(__uuidof(BackgroundCopyManager
));
336 *bits_manager
= object
.Detach();
340 void CleanupJobFiles(IBackgroundCopyJob
* job
) {
341 std::vector
<ScopedComPtr
<IBackgroundCopyFile
> > files
;
342 if (FAILED(GetFilesInJob(job
, &files
)))
344 for (size_t i
= 0; i
!= files
.size(); ++i
) {
345 base::string16 local_name
;
346 HRESULT
hr(GetJobFileProperties(files
[i
], &local_name
, NULL
, NULL
));
348 DeleteFileAndEmptyParentDirectory(base::FilePath(local_name
));
352 // Cleans up incompleted jobs that are too old.
353 HRESULT
CleanupStaleJobs(
354 base::win::ScopedComPtr
<IBackgroundCopyManager
> bits_manager
) {
358 static base::Time last_sweep
;
360 const base::TimeDelta
time_delta(
361 base::TimeDelta::FromDays(kPurgeStaleJobsIntervalBetweenChecksDays
));
362 const base::Time
current_time(base::Time::Now());
363 if (last_sweep
+ time_delta
> current_time
)
366 last_sweep
= current_time
;
368 std::vector
<ScopedComPtr
<IBackgroundCopyJob
> > jobs
;
369 HRESULT hr
= FindBitsJobIf(
370 std::bind2nd(JobCreationOlderThanDays(), kPurgeStaleJobsAfterDays
),
376 for (size_t i
= 0; i
!= jobs
.size(); ++i
) {
378 CleanupJobFiles(jobs
[i
]);
386 BackgroundDownloader::BackgroundDownloader(
387 scoped_ptr
<CrxDownloader
> successor
,
388 net::URLRequestContextGetter
* context_getter
,
389 scoped_refptr
<base::SingleThreadTaskRunner
> task_runner
)
390 : CrxDownloader(successor
.Pass()),
391 main_task_runner_(base::MessageLoopProxy::current()),
392 context_getter_(context_getter
),
393 task_runner_(task_runner
),
394 is_completed_(false) {
397 BackgroundDownloader::~BackgroundDownloader() {
398 DCHECK(thread_checker_
.CalledOnValidThread());
400 // The following objects have thread affinity and can't be destroyed on the
401 // main thread. The resources managed by these objects are acquired at the
402 // beginning of a download and released at the end of the download. Most of
403 // the time, when this destructor is called, these resources have been already
404 // disposed by. Releasing the ownership here is a NOP. However, if the browser
405 // is shutting down while a download is in progress, the timer is active and
406 // the interface pointers are valid. Releasing the ownership means leaking
407 // these objects and their associated resources.
409 bits_manager_
.Detach();
413 void BackgroundDownloader::DoStartDownload(const GURL
& url
) {
414 DCHECK(thread_checker_
.CalledOnValidThread());
416 task_runner_
->PostTask(
419 &BackgroundDownloader::BeginDownload
, base::Unretained(this), url
));
422 // Called once when this class is asked to do a download. Creates or opens
423 // an existing bits job, hooks up the notifications, and starts the timer.
424 void BackgroundDownloader::BeginDownload(const GURL
& url
) {
425 DCHECK(task_runner_
->RunsTasksOnCurrentThread());
429 is_completed_
= false;
430 download_start_time_
= base::Time::Now();
431 job_stuck_begin_time_
= download_start_time_
;
433 HRESULT hr
= QueueBitsJob(url
);
439 // A repeating timer retains the user task. This timer can be stopped and
440 // reset multiple times.
441 timer_
.reset(new base::RepeatingTimer
<BackgroundDownloader
>);
442 timer_
->Start(FROM_HERE
,
443 base::TimeDelta::FromSeconds(kJobPollingIntervalSec
),
445 &BackgroundDownloader::OnDownloading
);
448 // Called any time the timer fires.
449 void BackgroundDownloader::OnDownloading() {
450 DCHECK(task_runner_
->RunsTasksOnCurrentThread());
454 DCHECK(!is_completed_
);
458 BG_JOB_STATE job_state
= BG_JOB_STATE_ERROR
;
459 HRESULT hr
= job_
->GetState(&job_state
);
466 case BG_JOB_STATE_TRANSFERRED
:
467 OnStateTransferred();
470 case BG_JOB_STATE_ERROR
:
474 case BG_JOB_STATE_CANCELLED
:
478 case BG_JOB_STATE_ACKNOWLEDGED
:
479 OnStateAcknowledged();
482 case BG_JOB_STATE_QUEUED
:
484 case BG_JOB_STATE_CONNECTING
:
486 case BG_JOB_STATE_SUSPENDED
:
490 case BG_JOB_STATE_TRANSIENT_ERROR
:
491 OnStateTransientError();
494 case BG_JOB_STATE_TRANSFERRING
:
495 OnStateTransferring();
503 // Completes the BITS download, picks up the file path of the response, and
504 // notifies the CrxDownloader. The function should be called only once.
505 void BackgroundDownloader::EndDownload(HRESULT error
) {
506 DCHECK(task_runner_
->RunsTasksOnCurrentThread());
508 DCHECK(!is_completed_
);
509 is_completed_
= true;
513 const base::Time
download_end_time(base::Time::Now());
514 const base::TimeDelta download_time
=
515 download_end_time
>= download_start_time_
516 ? download_end_time
- download_start_time_
519 int64 downloaded_bytes
= -1;
520 int64 total_bytes
= -1;
521 GetJobByteCount(job_
, &downloaded_bytes
, &total_bytes
);
523 if (FAILED(error
) && job_
) {
525 CleanupJobFiles(job_
);
530 CleanupStaleJobs(bits_manager_
);
531 bits_manager_
= NULL
;
533 // Consider the url handled if it has been successfully downloaded or a
534 // 5xx has been received.
535 const bool is_handled
=
536 SUCCEEDED(error
) || IsHttpServerError(GetHttpStatusFromBitsError(error
));
538 const int error_to_report
= SUCCEEDED(error
) ? 0 : error
;
540 DownloadMetrics download_metrics
;
541 download_metrics
.url
= url();
542 download_metrics
.downloader
= DownloadMetrics::kBits
;
543 download_metrics
.error
= error_to_report
;
544 download_metrics
.downloaded_bytes
= downloaded_bytes
;
545 download_metrics
.total_bytes
= total_bytes
;
546 download_metrics
.download_time_ms
= download_time
.InMilliseconds();
549 result
.error
= error_to_report
;
550 result
.response
= response_
;
551 result
.downloaded_bytes
= downloaded_bytes
;
552 result
.total_bytes
= total_bytes
;
553 main_task_runner_
->PostTask(
555 base::Bind(&BackgroundDownloader::OnDownloadComplete
,
556 base::Unretained(this),
561 // Once the task is posted to the the main thread, this object may be deleted
562 // by its owner. It is not safe to access members of this object on the
563 // task runner from this point on. The timer is stopped and all BITS
564 // interface pointers have been released.
567 // Called when the BITS job has been transferred successfully. Completes the
568 // BITS job by removing it from the BITS queue and making the download
569 // available to the caller.
570 void BackgroundDownloader::OnStateTransferred() {
571 EndDownload(CompleteJob());
574 // Called when the job has encountered an error and no further progress can
575 // be made. Cancels this job and removes it from the BITS queue.
576 void BackgroundDownloader::OnStateError() {
577 HRESULT error_code
= S_OK
;
578 HRESULT hr
= GetJobError(job_
, &error_code
);
581 DCHECK(FAILED(error_code
));
582 EndDownload(error_code
);
585 // Called when the job has encountered a transient error, such as a
586 // network disconnect, a server error, or some other recoverable error.
587 void BackgroundDownloader::OnStateTransientError() {
588 // If the job appears to be stuck, handle the transient error as if
589 // it were a final error. This causes the job to be cancelled and a specific
590 // error be returned, if the error was available.
596 // Don't retry at all if the transient error was a 5xx.
597 HRESULT error_code
= S_OK
;
598 HRESULT hr
= GetJobError(job_
, &error_code
);
600 IsHttpServerError(GetHttpStatusFromBitsError(error_code
))) {
606 void BackgroundDownloader::OnStateQueued() {
608 EndDownload(E_ABORT
); // Return a generic error for now.
611 void BackgroundDownloader::OnStateTransferring() {
612 // Resets the baseline for detecting a stuck job since the job is transferring
613 // data and it is making progress.
614 job_stuck_begin_time_
= base::Time::Now();
616 int64 downloaded_bytes
= -1;
617 int64 total_bytes
= -1;
618 HRESULT hr
= GetJobByteCount(job_
, &downloaded_bytes
, &total_bytes
);
623 result
.downloaded_bytes
= downloaded_bytes
;
624 result
.total_bytes
= total_bytes
;
626 main_task_runner_
->PostTask(
628 base::Bind(&BackgroundDownloader::OnDownloadProgress
,
629 base::Unretained(this),
633 // Called when the download was cancelled. Since the observer should have
634 // been disconnected by now, this notification must not be seen.
635 void BackgroundDownloader::OnStateCancelled() {
636 EndDownload(E_UNEXPECTED
);
639 // Called when the download was completed. Same as above.
640 void BackgroundDownloader::OnStateAcknowledged() {
641 EndDownload(E_UNEXPECTED
);
644 // Creates or opens a job for the given url and queues it up. Tries to
645 // install a job observer but continues on if an observer can't be set up.
646 HRESULT
BackgroundDownloader::QueueBitsJob(const GURL
& url
) {
647 DCHECK(task_runner_
->RunsTasksOnCurrentThread());
650 if (bits_manager_
== NULL
) {
651 hr
= GetBitsManager(bits_manager_
.Receive());
656 hr
= CreateOrOpenJob(url
);
661 hr
= InitializeNewJob(url
);
666 return job_
->Resume();
669 HRESULT
BackgroundDownloader::CreateOrOpenJob(const GURL
& url
) {
670 std::vector
<ScopedComPtr
<IBackgroundCopyJob
> > jobs
;
671 HRESULT hr
= FindBitsJobIf(
672 std::bind2nd(JobFileUrlEqual(), base::SysUTF8ToWide(url
.spec())),
675 if (SUCCEEDED(hr
) && !jobs
.empty()) {
680 // Use kJobDescription as a temporary job display name until the proper
681 // display name is initialized later on.
683 ScopedComPtr
<IBackgroundCopyJob
> job
;
684 hr
= bits_manager_
->CreateJob(
685 kJobDescription
, BG_JOB_TYPE_DOWNLOAD
, &guid
, job
.Receive());
693 HRESULT
BackgroundDownloader::InitializeNewJob(const GURL
& url
) {
694 const base::string16
filename(base::SysUTF8ToWide(url
.ExtractFileName()));
696 base::FilePath tempdir
;
697 if (!base::CreateNewTempDirectory(FILE_PATH_LITERAL("chrome_BITS_"),
701 HRESULT hr
= job_
->AddFile(base::SysUTF8ToWide(url
.spec()).c_str(),
702 tempdir
.Append(filename
).AsUTF16Unsafe().c_str());
706 hr
= job_
->SetDisplayName(filename
.c_str());
710 hr
= job_
->SetDescription(kJobDescription
);
714 hr
= job_
->SetPriority(BG_JOB_PRIORITY_NORMAL
);
718 hr
= job_
->SetMinimumRetryDelay(60 * kMinimumRetryDelayMin
);
722 const int kSecondsDay
= 60 * 60 * 24;
723 hr
= job_
->SetNoProgressTimeout(kSecondsDay
* kSetNoProgressTimeoutDays
);
730 bool BackgroundDownloader::IsStuck() {
731 const base::TimeDelta
job_stuck_timeout(
732 base::TimeDelta::FromMinutes(kJobStuckTimeoutMin
));
733 return job_stuck_begin_time_
+ job_stuck_timeout
< base::Time::Now();
736 HRESULT
BackgroundDownloader::CompleteJob() {
737 HRESULT hr
= job_
->Complete();
738 if (FAILED(hr
) && hr
!= BG_S_UNABLE_TO_DELETE_FILES
)
741 std::vector
<ScopedComPtr
<IBackgroundCopyFile
> > files
;
742 hr
= GetFilesInJob(job_
, &files
);
749 base::string16 local_name
;
750 BG_FILE_PROGRESS progress
= {0};
751 hr
= GetJobFileProperties(files
.front(), &local_name
, NULL
, &progress
);
755 // Sanity check the post-conditions of a successful download, including
756 // the file and job invariants. The byte counts for a job and its file
757 // must match as a job only contains one file.
758 DCHECK(progress
.Completed
);
759 DCHECK_EQ(progress
.BytesTotal
, progress
.BytesTransferred
);
761 response_
= base::FilePath(local_name
);
766 } // namespace component_updater