1 // Copyright 2014 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "chrome/browser/safe_browsing/incident_reporting/download_metadata_manager.h"
10 #include "base/callback.h"
11 #include "base/files/file.h"
12 #include "base/files/file_util.h"
13 #include "base/files/important_file_writer.h"
14 #include "base/location.h"
15 #include "base/macros.h"
16 #include "base/metrics/histogram.h"
17 #include "base/sequenced_task_runner.h"
18 #include "base/threading/sequenced_worker_pool.h"
19 #include "chrome/common/safe_browsing/csd.pb.h"
20 #include "content/public/browser/browser_context.h"
21 #include "content/public/browser/download_item.h"
23 namespace safe_browsing
{
27 // Histogram bucket values for metadata read operations. Do not reorder.
28 enum MetadataReadResult
{
40 // Histogram bucket values for metadata write operations. Do not reorder.
41 enum MetadataWriteResult
{
43 SERIALIZATION_FAILURE
= 1,
48 // The name of the metadata file in the profile directory.
49 const base::FilePath::CharType kDownloadMetadataBasename
[] =
50 FILE_PATH_LITERAL("DownloadMetadata");
53 // DownloadItemData ------------------------------------------------------------
55 // A UserData object that holds the ClientDownloadRequest for a download while
57 class DownloadItemData
: public base::SupportsUserData::Data
{
59 // Returns the data for a given DownloadItem or null if there is none.
60 static DownloadItemData
* GetData(content::DownloadItem
* item
);
62 // Returns the data for a given DownloadItem, creating one if one does not
64 static DownloadItemData
* GetOrCreateData(content::DownloadItem
* item
);
66 // Removes the data associated with a given DownloadItem. The data is
68 static void RemoveData(content::DownloadItem
* item
);
70 // Sets the download request for the item.
71 void set_request(scoped_ptr
<ClientDownloadRequest
> request
) {
72 request_
= request
.Pass();
75 // Returns ownership of the item's download request.
76 scoped_ptr
<ClientDownloadRequest
> TakeRequest() { return request_
.Pass(); }
79 // A unique id for associating metadata with a content::DownloadItem.
80 static const void* const kKey_
;
83 ~DownloadItemData() override
{}
85 scoped_ptr
<ClientDownloadRequest
> request_
;
87 DISALLOW_COPY_AND_ASSIGN(DownloadItemData
);
90 // Make the key's value unique by setting it to its own location.
92 const void* const DownloadItemData::kKey_
= &DownloadItemData::kKey_
;
95 DownloadItemData
* DownloadItemData::GetData(content::DownloadItem
* item
) {
96 return static_cast<DownloadItemData
*>(item
->GetUserData(&kKey_
));
100 DownloadItemData
* DownloadItemData::GetOrCreateData(
101 content::DownloadItem
* item
) {
102 DownloadItemData
* data
= GetData(item
);
104 data
= new DownloadItemData();
105 item
->SetUserData(&kKey_
, data
);
111 void DownloadItemData::RemoveData(content::DownloadItem
* item
) {
112 item
->RemoveUserData(&kKey_
);
116 // Utility functions------------------------------------------------------------
118 // Returns the path to the metadata file for |browser_context|.
119 base::FilePath
GetMetadataPath(content::BrowserContext
* browser_context
) {
120 return browser_context
->GetPath().Append(kDownloadMetadataBasename
);
123 // Returns true if |metadata| appears to be valid.
124 bool MetadataIsValid(const DownloadMetadata
& metadata
) {
125 return (metadata
.has_download_id() &&
126 metadata
.has_download() &&
127 metadata
.download().has_download() &&
128 metadata
.download().download().has_url());
131 // Reads and parses a DownloadMetadata message from |metadata_path| into
133 void ReadMetadataOnWorkerPool(const base::FilePath
& metadata_path
,
134 DownloadMetadata
* metadata
) {
137 MetadataReadResult result
= NUM_READ_RESULTS
;
138 File
metadata_file(metadata_path
, File::FLAG_OPEN
| File::FLAG_READ
);
139 if (metadata_file
.IsValid()) {
140 base::File::Info info
;
141 if (metadata_file
.GetInfo(&info
)) {
142 if (info
.size
<= INT_MAX
) {
143 const int size
= static_cast<int>(info
.size
);
144 scoped_ptr
<char[]> file_data(new char[info
.size
]);
145 if (metadata_file
.Read(0, file_data
.get(), size
)) {
146 if (!metadata
->ParseFromArray(file_data
.get(), size
))
147 result
= PARSE_FAILURE
;
148 else if (!MetadataIsValid(*metadata
))
149 result
= MALFORMED_DATA
;
151 result
= READ_SUCCESS
;
153 result
= READ_FAILURE
;
156 result
= FILE_TOO_BIG
;
159 result
= GET_INFO_FAILURE
;
161 } else if (metadata_file
.error_details() != File::FILE_ERROR_NOT_FOUND
) {
162 result
= OPEN_FAILURE
;
166 if (result
!= READ_SUCCESS
)
168 UMA_HISTOGRAM_ENUMERATION(
169 "SBIRS.DownloadMetadata.ReadResult", result
, NUM_READ_RESULTS
);
172 // Writes |download_metadata| to |metadata_path|.
173 void WriteMetadataOnWorkerPool(const base::FilePath
& metadata_path
,
174 DownloadMetadata
* download_metadata
) {
175 MetadataWriteResult result
= NUM_WRITE_RESULTS
;
176 std::string file_data
;
177 if (download_metadata
->SerializeToString(&file_data
)) {
179 base::ImportantFileWriter::WriteFileAtomically(metadata_path
, file_data
)
183 result
= SERIALIZATION_FAILURE
;
185 UMA_HISTOGRAM_ENUMERATION(
186 "SBIRS.DownloadMetadata.WriteResult", result
, NUM_WRITE_RESULTS
);
189 // Deletes |metadata_path|.
190 void DeleteMetadataOnWorkerPool(const base::FilePath
& metadata_path
) {
191 bool success
= base::DeleteFile(metadata_path
, false /* not recursive */);
192 UMA_HISTOGRAM_BOOLEAN("SBIRS.DownloadMetadata.DeleteSuccess", success
);
195 // Runs |callback| with the DownloadDetails in |download_metadata|.
197 const DownloadMetadataManager::GetDownloadDetailsCallback
& callback
,
198 scoped_ptr
<DownloadMetadata
> download_metadata
) {
199 if (!download_metadata
->has_download_id())
200 callback
.Run(scoped_ptr
<ClientIncidentReport_DownloadDetails
>());
202 callback
.Run(make_scoped_ptr(download_metadata
->release_download()).Pass());
207 // Applies operations to the profile's persistent DownloadMetadata as they occur
208 // on its corresponding download item. An instance can be in one of three
209 // states: waiting for metatada load, waiting for metadata to load after its
210 // corresponding DownloadManager has gone down, and not waiting for metadata to
211 // load. The instance observes all download items beloing to its manager. While
212 // it is waiting for metadata to load, it records all operations on download
213 // items that must be reflected in the metadata. Once the metadata is ready,
214 // recorded operations are applied to the metadata. The instance continues to
215 // observe all download items to keep the existing metadata up to date. While
216 // waiting for metadata to load, an instance also tracks callbacks to be run to
217 // provide consumers with persisted details of a download.
218 class DownloadMetadataManager::ManagerContext
219 : public content::DownloadItem::Observer
{
221 ManagerContext(const scoped_refptr
<base::SequencedTaskRunner
>& read_runner
,
222 const scoped_refptr
<base::SequencedTaskRunner
>& write_runner
,
223 content::DownloadManager
* download_manager
);
225 // Detaches this context from its owner. The owner must not access the context
226 // following this call. The context will be deleted immediately if it is not
227 // waiting for a metadata load with either recorded operations or pending
229 void Detach(content::DownloadManager
* download_manager
);
231 // Notifies the context that |download| has been added to its manager.
232 void OnDownloadCreated(content::DownloadItem
* download
);
234 // Sets |request| as the relevant metadata to persist for |download| if or
235 // when it is complete. If |request| is null, the metadata for |download|
236 // is/will be removed.
237 void SetRequest(content::DownloadItem
* download
,
238 scoped_ptr
<ClientDownloadRequest
> request
);
240 // Gets the persisted DownloadDetails. |callback| will be run immediately if
241 // the data is available. Otherwise, it will be run later on the caller's
243 void GetDownloadDetails(const GetDownloadDetailsCallback
& callback
);
246 // content::DownloadItem::Observer methods.
247 void OnDownloadUpdated(content::DownloadItem
* download
) override
;
248 void OnDownloadOpened(content::DownloadItem
* download
) override
;
249 void OnDownloadRemoved(content::DownloadItem
* download
) override
;
253 // The context is waiting for the metadata file to be loaded.
256 // The context is waiting for the metadata file to be loaded and its
257 // corresponding DownloadManager has gone away.
260 // The context has loaded the metadata file.
265 ItemData() : removed() {}
266 base::Time last_opened_time
;
270 // A mapping of download IDs to their corresponding data.
271 typedef std::map
<uint32_t, ItemData
> ItemDataMap
;
273 ~ManagerContext() override
;
275 // Commits |request| to the DownloadDetails for |item|'s BrowserContext.
276 // Callbacks will be run immediately if the context had been waiting for a
277 // load (which will be abandoned).
278 void CommitRequest(content::DownloadItem
* item
,
279 scoped_ptr
<ClientDownloadRequest
> request
);
281 // Posts a task in the worker pool to read the metadata from disk.
284 // Posts a task in the worker pool to write the metadata to disk.
285 void WriteMetadata();
287 // Removes metadata for the context from memory and posts a task in the worker
288 // pool to delete it on disk.
289 void RemoveMetadata();
291 // Clears the |pending_items_| mapping.
292 void ClearPendingItems();
294 // Runs all |get_details_callbacks_| with the current metadata.
297 // Returns true if metadata corresponding to |item| is available.
298 bool HasMetadataFor(const content::DownloadItem
* item
) const;
300 // A callback run on the main thread with the results from reading the
301 // metadata file from disk.
302 void OnMetadataReady(scoped_ptr
<DownloadMetadata
> download_metadata
);
304 // Updates the last opened time in the metadata and writes it to disk.
305 void UpdateLastOpenedTime(const base::Time
& last_opened_time
);
307 // A task runner to which read tasks are posted.
308 scoped_refptr
<base::SequencedTaskRunner
> read_runner_
;
310 // A task runner to which write tasks are posted.
311 scoped_refptr
<base::SequencedTaskRunner
> write_runner_
;
313 // The path to the metadata file for this context.
314 base::FilePath metadata_path_
;
316 // When not LOAD_COMPLETE, the context is waiting for a pending read operation
317 // to complete. While this is the case, events are temporarily recorded in
318 // |pending_items_|. Once the read completes, pending operations for the item
319 // corresponding to the metadata file are applied to the file and all other
320 // recorded data are dropped. Queued GetDownloadDetailsCallbacks are run upon
321 // read completion as well. The context is moved to the DETACHED_WAIT state if
322 // the corresponding DownloadManager goes away while a read operation is
323 // outstanding. When the read subsequently completes, the context is destroyed
324 // after the processing described above is performed.
327 // The current metadata for the context. May be supplied either by reading
328 // from the file or by having been set via |SetRequest|.
329 scoped_ptr
<DownloadMetadata
> download_metadata_
;
331 // The operation data that accumulates for added download items while the
332 // metadata file is being read.
333 ItemDataMap pending_items_
;
335 // Pending callbacks in response to GetDownloadDetails. The callbacks are run
336 // in order when a pending read operation completes.
337 std::list
<GetDownloadDetailsCallback
> get_details_callbacks_
;
339 base::WeakPtrFactory
<ManagerContext
> weak_factory_
;
341 DISALLOW_COPY_AND_ASSIGN(ManagerContext
);
345 // DownloadMetadataManager -----------------------------------------------------
347 DownloadMetadataManager::DownloadMetadataManager(
348 const scoped_refptr
<base::SequencedWorkerPool
>& worker_pool
) {
349 base::SequencedWorkerPool::SequenceToken
token(
350 worker_pool
->GetSequenceToken());
351 // Do not block shutdown on reads since nothing will come of it.
352 read_runner_
= worker_pool
->GetSequencedTaskRunnerWithShutdownBehavior(
353 token
, base::SequencedWorkerPool::CONTINUE_ON_SHUTDOWN
);
354 // Block shutdown on writes only if they've already begun.
355 write_runner_
= worker_pool
->GetSequencedTaskRunnerWithShutdownBehavior(
356 token
, base::SequencedWorkerPool::SKIP_ON_SHUTDOWN
);
359 DownloadMetadataManager::DownloadMetadataManager(
360 const scoped_refptr
<base::SequencedTaskRunner
>& task_runner
)
361 : read_runner_(task_runner
), write_runner_(task_runner
) {
364 DownloadMetadataManager::~DownloadMetadataManager() {
365 // Destruction may have taken place before managers have gone down.
366 for (const auto& manager_context_pair
: contexts_
) {
367 manager_context_pair
.first
->RemoveObserver(this);
368 manager_context_pair
.second
->Detach(manager_context_pair
.first
);
373 void DownloadMetadataManager::AddDownloadManager(
374 content::DownloadManager
* download_manager
) {
375 DCHECK_EQ(contexts_
.count(download_manager
), 0U);
376 download_manager
->AddObserver(this);
377 contexts_
[download_manager
] =
378 new ManagerContext(read_runner_
, write_runner_
, download_manager
);
381 void DownloadMetadataManager::SetRequest(content::DownloadItem
* item
,
382 const ClientDownloadRequest
* request
) {
383 content::DownloadManager
* download_manager
=
384 GetDownloadManagerForBrowserContext(item
->GetBrowserContext());
385 DCHECK_EQ(contexts_
.count(download_manager
), 1U);
386 contexts_
[download_manager
]->SetRequest(
388 request
? make_scoped_ptr(new ClientDownloadRequest(*request
)) : nullptr);
391 void DownloadMetadataManager::GetDownloadDetails(
392 content::BrowserContext
* browser_context
,
393 const GetDownloadDetailsCallback
& callback
) {
394 DCHECK(browser_context
);
395 // The DownloadManager for |browser_context| may not have been created yet. In
396 // this case, asking for it would cause history to load in the background and
397 // wouldn't really help much. Instead, scan the contexts to see if one belongs
398 // to |browser_context|. If one is not found, read the metadata and return it.
399 scoped_ptr
<ClientIncidentReport_DownloadDetails
> download_details
;
400 for (const auto& manager_context_pair
: contexts_
) {
401 if (manager_context_pair
.first
->GetBrowserContext() == browser_context
) {
402 manager_context_pair
.second
->GetDownloadDetails(callback
);
407 // Fire off a task to load the details and return them to the caller.
408 DownloadMetadata
* metadata
= new DownloadMetadata();
409 read_runner_
->PostTaskAndReply(
411 base::Bind(&ReadMetadataOnWorkerPool
,
412 GetMetadataPath(browser_context
),
415 &ReturnResults
, callback
, base::Passed(make_scoped_ptr(metadata
))));
418 content::DownloadManager
*
419 DownloadMetadataManager::GetDownloadManagerForBrowserContext(
420 content::BrowserContext
* context
) {
421 return content::BrowserContext::GetDownloadManager(context
);
424 void DownloadMetadataManager::OnDownloadCreated(
425 content::DownloadManager
* download_manager
,
426 content::DownloadItem
* item
) {
427 DCHECK_EQ(contexts_
.count(download_manager
), 1U);
428 contexts_
[download_manager
]->OnDownloadCreated(item
);
431 void DownloadMetadataManager::ManagerGoingDown(
432 content::DownloadManager
* download_manager
) {
433 DCHECK_EQ(contexts_
.count(download_manager
), 1U);
434 auto iter
= contexts_
.find(download_manager
);
435 iter
->first
->RemoveObserver(this);
436 iter
->second
->Detach(download_manager
);
437 contexts_
.erase(iter
);
441 // DownloadMetadataManager::ManagerContext -------------------------------------
443 DownloadMetadataManager::ManagerContext::ManagerContext(
444 const scoped_refptr
<base::SequencedTaskRunner
>& read_runner
,
445 const scoped_refptr
<base::SequencedTaskRunner
>& write_runner
,
446 content::DownloadManager
* download_manager
)
447 : read_runner_(read_runner
),
448 write_runner_(write_runner
),
449 metadata_path_(GetMetadataPath(download_manager
->GetBrowserContext())),
450 state_(WAITING_FOR_LOAD
),
451 weak_factory_(this) {
452 // Observe all pre-existing items in the manager.
453 content::DownloadManager::DownloadVector items
;
454 download_manager
->GetAllDownloads(&items
);
455 for (auto* download_item
: items
)
456 download_item
->AddObserver(this);
458 // Start the asynchronous task to read the persistent metadata.
462 void DownloadMetadataManager::ManagerContext::Detach(
463 content::DownloadManager
* download_manager
) {
464 // Stop observing all items belonging to the manager.
465 content::DownloadManager::DownloadVector items
;
466 download_manager
->GetAllDownloads(&items
);
467 for (auto* download_item
: items
)
468 download_item
->RemoveObserver(this);
470 // Delete the instance immediately if there's no work to process after a
471 // pending read completes.
472 if (get_details_callbacks_
.empty() && pending_items_
.empty()) {
475 // delete the instance in OnMetadataReady.
476 state_
= DETACHED_WAIT
;
480 void DownloadMetadataManager::ManagerContext::OnDownloadCreated(
481 content::DownloadItem
* download
) {
482 download
->AddObserver(this);
485 void DownloadMetadataManager::ManagerContext::SetRequest(
486 content::DownloadItem
* download
,
487 scoped_ptr
<ClientDownloadRequest
> request
) {
488 // Hold on to the request for completion time if the download is in progress.
489 // Otherwise, either commit the request or remove metadata, as appropriate.
490 if (download
->GetState() == content::DownloadItem::IN_PROGRESS
)
491 DownloadItemData::GetOrCreateData(download
)->set_request(request
.Pass());
493 CommitRequest(download
, request
.Pass());
498 void DownloadMetadataManager::ManagerContext::GetDownloadDetails(
499 const GetDownloadDetailsCallback
& callback
) {
500 if (state_
!= LOAD_COMPLETE
) {
501 get_details_callbacks_
.push_back(callback
);
503 callback
.Run(download_metadata_
?
504 make_scoped_ptr(new ClientIncidentReport_DownloadDetails(
505 download_metadata_
->download())) :
510 void DownloadMetadataManager::ManagerContext::OnDownloadUpdated(
511 content::DownloadItem
* download
) {
512 const content::DownloadItem::DownloadState state
= download
->GetState();
513 if (state
== content::DownloadItem::IN_PROGRESS
) {
514 // Make sure that ItemData exists for this download so that decisions
515 // regarding its metadata can be made upon completion.
516 ignore_result(DownloadItemData::GetOrCreateData(download
));
517 } else if (state
== content::DownloadItem::COMPLETE
) {
518 // Persist metadata for this download if it has just completed.
519 DownloadItemData
* data
= DownloadItemData::GetData(download
);
522 SetRequest(download
, data
->TakeRequest());
523 // RemoveData below will delete |data|, so clear the pointer to it.
525 // Drop the data from the download item so that subsequent updates will do
527 DownloadItemData::RemoveData(download
);
531 void DownloadMetadataManager::ManagerContext::OnDownloadOpened(
532 content::DownloadItem
* download
) {
533 const base::Time now
= base::Time::Now();
534 if (state_
!= LOAD_COMPLETE
)
535 pending_items_
[download
->GetId()].last_opened_time
= now
;
536 else if (HasMetadataFor(download
))
537 UpdateLastOpenedTime(now
);
540 void DownloadMetadataManager::ManagerContext::OnDownloadRemoved(
541 content::DownloadItem
* download
) {
542 if (state_
!= LOAD_COMPLETE
)
543 pending_items_
[download
->GetId()].removed
= true;
544 else if (HasMetadataFor(download
))
548 DownloadMetadataManager::ManagerContext::~ManagerContext() {
549 // A context should not be deleted while waiting for a load to complete.
550 DCHECK(pending_items_
.empty());
551 DCHECK(get_details_callbacks_
.empty());
554 void DownloadMetadataManager::ManagerContext::CommitRequest(
555 content::DownloadItem
* item
,
556 scoped_ptr
<ClientDownloadRequest
> request
) {
557 DCHECK_EQ(content::DownloadItem::COMPLETE
, item
->GetState());
558 if (state_
!= LOAD_COMPLETE
) {
559 // Abandon the read task since |item| is the new top dog.
560 weak_factory_
.InvalidateWeakPtrs();
561 state_
= LOAD_COMPLETE
;
562 // Drop any recorded operations.
566 download_metadata_
.reset(new DownloadMetadata
);
567 download_metadata_
->set_download_id(item
->GetId());
568 download_metadata_
->mutable_download()->set_allocated_download(
570 download_metadata_
->mutable_download()->set_download_time_msec(
571 item
->GetEndTime().ToJavaTime());
574 // Run callbacks (only present in case of a transition to LOAD_COMPLETE).
578 void DownloadMetadataManager::ManagerContext::ReadMetadata() {
579 DCHECK_NE(state_
, LOAD_COMPLETE
);
581 DownloadMetadata
* metadata
= new DownloadMetadata();
582 // Do not block shutdown on this read since nothing will come of it.
583 read_runner_
->PostTaskAndReply(
585 base::Bind(&ReadMetadataOnWorkerPool
, metadata_path_
, metadata
),
586 base::Bind(&DownloadMetadataManager::ManagerContext::OnMetadataReady
,
587 weak_factory_
.GetWeakPtr(),
588 base::Passed(make_scoped_ptr(metadata
))));
591 void DownloadMetadataManager::ManagerContext::WriteMetadata() {
592 write_runner_
->PostTask(
594 base::Bind(&WriteMetadataOnWorkerPool
,
596 base::Owned(new DownloadMetadata(*download_metadata_
))));
599 void DownloadMetadataManager::ManagerContext::RemoveMetadata() {
600 if (state_
!= LOAD_COMPLETE
) {
601 // Abandon the read task since the file is to be removed.
602 weak_factory_
.InvalidateWeakPtrs();
603 state_
= LOAD_COMPLETE
;
604 // Drop any recorded operations.
607 // Remove any metadata.
608 download_metadata_
.reset();
609 write_runner_
->PostTask(
610 FROM_HERE
, base::Bind(&DeleteMetadataOnWorkerPool
, metadata_path_
));
611 // Run callbacks (only present in case of a transition to LOAD_COMPLETE).
615 void DownloadMetadataManager::ManagerContext::ClearPendingItems() {
616 pending_items_
.clear();
619 void DownloadMetadataManager::ManagerContext::RunCallbacks() {
620 while (!get_details_callbacks_
.empty()) {
621 const auto& callback
= get_details_callbacks_
.front();
622 callback
.Run(download_metadata_
?
623 make_scoped_ptr(new ClientIncidentReport_DownloadDetails(
624 download_metadata_
->download())) :
626 get_details_callbacks_
.pop_front();
630 bool DownloadMetadataManager::ManagerContext::HasMetadataFor(
631 const content::DownloadItem
* item
) const {
632 // There must not be metadata if the load is not complete.
633 DCHECK(state_
== LOAD_COMPLETE
|| !download_metadata_
);
634 return (download_metadata_
&&
635 download_metadata_
->download_id() == item
->GetId());
638 void DownloadMetadataManager::ManagerContext::OnMetadataReady(
639 scoped_ptr
<DownloadMetadata
> download_metadata
) {
640 DCHECK_NE(state_
, LOAD_COMPLETE
);
642 const bool is_detached
= (state_
== DETACHED_WAIT
);
644 // Note that any available data has been read.
645 state_
= LOAD_COMPLETE
;
646 if (download_metadata
->has_download_id())
647 download_metadata_
= download_metadata
.Pass();
649 download_metadata_
.reset();
651 // Process all operations that had been held while waiting for the metadata.
652 if (download_metadata_
) {
653 const auto& iter
= pending_items_
.find(download_metadata_
->download_id());
654 if (iter
!= pending_items_
.end()) {
655 const ItemData
& item_data
= iter
->second
;
656 if (item_data
.removed
)
658 else if (!item_data
.last_opened_time
.is_null())
659 UpdateLastOpenedTime(item_data
.last_opened_time
);
663 // Drop the recorded operations.
669 // Delete the context now if it has been detached.
674 void DownloadMetadataManager::ManagerContext::UpdateLastOpenedTime(
675 const base::Time
& last_opened_time
) {
676 download_metadata_
->mutable_download()->set_open_time_msec(
677 last_opened_time
.ToJavaTime());
681 } // namespace safe_browsing