1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "content/browser/storage_partition_impl_map.h"
8 #include "base/callback.h"
9 #include "base/files/file_enumerator.h"
10 #include "base/files/file_path.h"
11 #include "base/files/file_util.h"
12 #include "base/location.h"
13 #include "base/single_thread_task_runner.h"
14 #include "base/stl_util.h"
15 #include "base/strings/string_number_conversions.h"
16 #include "base/strings/string_util.h"
17 #include "base/strings/stringprintf.h"
18 #include "base/thread_task_runner_handle.h"
19 #include "base/threading/sequenced_worker_pool.h"
20 #include "content/browser/appcache/appcache_interceptor.h"
21 #include "content/browser/appcache/chrome_appcache_service.h"
22 #include "content/browser/fileapi/browser_file_system_helper.h"
23 #include "content/browser/fileapi/chrome_blob_storage_context.h"
24 #include "content/browser/loader/resource_request_info_impl.h"
25 #include "content/browser/resource_context_impl.h"
26 #include "content/browser/service_worker/service_worker_request_handler.h"
27 #include "content/browser/storage_partition_impl.h"
28 #include "content/browser/streams/stream.h"
29 #include "content/browser/streams/stream_context.h"
30 #include "content/browser/streams/stream_registry.h"
31 #include "content/browser/streams/stream_url_request_job.h"
32 #include "content/browser/webui/url_data_manager_backend.h"
33 #include "content/public/browser/browser_context.h"
34 #include "content/public/browser/browser_thread.h"
35 #include "content/public/browser/content_browser_client.h"
36 #include "content/public/browser/navigator_connect_context.h"
37 #include "content/public/browser/navigator_connect_service_factory.h"
38 #include "content/public/browser/storage_partition.h"
39 #include "content/public/common/content_constants.h"
40 #include "content/public/common/url_constants.h"
41 #include "crypto/sha2.h"
42 #include "net/url_request/url_request_context.h"
43 #include "net/url_request/url_request_context_getter.h"
44 #include "storage/browser/blob/blob_storage_context.h"
45 #include "storage/browser/blob/blob_url_request_job_factory.h"
46 #include "storage/browser/fileapi/file_system_url_request_job_factory.h"
48 using storage::FileSystemContext
;
49 using storage::BlobStorageContext
;
55 // A derivative that knows about Streams too.
56 class BlobProtocolHandler
: public net::URLRequestJobFactory::ProtocolHandler
{
58 BlobProtocolHandler(ChromeBlobStorageContext
* blob_storage_context
,
59 StreamContext
* stream_context
,
60 storage::FileSystemContext
* file_system_context
)
61 : blob_storage_context_(blob_storage_context
),
62 stream_context_(stream_context
),
63 file_system_context_(file_system_context
) {}
65 ~BlobProtocolHandler() override
{}
67 net::URLRequestJob
* MaybeCreateJob(
68 net::URLRequest
* request
,
69 net::NetworkDelegate
* network_delegate
) const override
{
70 scoped_refptr
<Stream
> stream
=
71 stream_context_
->registry()->GetStream(request
->url());
73 return new StreamURLRequestJob(request
, network_delegate
, stream
);
75 if (!blob_protocol_handler_
) {
76 // Construction is deferred because 'this' is constructed on
77 // the main thread but we want blob_protocol_handler_ constructed
79 blob_protocol_handler_
.reset(new storage::BlobProtocolHandler(
80 blob_storage_context_
->context(),
81 file_system_context_
.get(),
82 BrowserThread::GetMessageLoopProxyForThread(BrowserThread::FILE)
85 return blob_protocol_handler_
->MaybeCreateJob(request
, network_delegate
);
89 const scoped_refptr
<ChromeBlobStorageContext
> blob_storage_context_
;
90 const scoped_refptr
<StreamContext
> stream_context_
;
91 const scoped_refptr
<storage::FileSystemContext
> file_system_context_
;
92 mutable scoped_ptr
<storage::BlobProtocolHandler
> blob_protocol_handler_
;
93 DISALLOW_COPY_AND_ASSIGN(BlobProtocolHandler
);
96 // These constants are used to create the directory structure under the profile
97 // where renderers with a non-default storage partition keep their persistent
98 // state. This will contain a set of directories that partially mirror the
99 // directory structure of BrowserContext::GetPath().
101 // The kStoragePartitionDirname contains an extensions directory which is
102 // further partitioned by extension id, followed by another level of directories
103 // for the "default" extension storage partition and one directory for each
104 // persistent partition used by a webview tag. Example:
106 // Storage/ext/ABCDEF/def
107 // Storage/ext/ABCDEF/hash(partition name)
109 // The code in GetStoragePartitionPath() constructs these path names.
111 // TODO(nasko): Move extension related path code out of content.
112 const base::FilePath::CharType kStoragePartitionDirname
[] =
113 FILE_PATH_LITERAL("Storage");
114 const base::FilePath::CharType kExtensionsDirname
[] =
115 FILE_PATH_LITERAL("ext");
116 const base::FilePath::CharType kDefaultPartitionDirname
[] =
117 FILE_PATH_LITERAL("def");
118 const base::FilePath::CharType kTrashDirname
[] =
119 FILE_PATH_LITERAL("trash");
121 // Because partition names are user specified, they can be arbitrarily long
122 // which makes them unsuitable for paths names. We use a truncation of a
123 // SHA256 hash to perform a deterministic shortening of the string. The
124 // kPartitionNameHashBytes constant controls the length of the truncation.
125 // We use 6 bytes, which gives us 99.999% reliability against collisions over
126 // 1 million partition domains.
129 // We assume that all partition names within one partition domain are
130 // controlled by the the same entity. Thus there is no chance for adverserial
131 // attack and all we care about is accidental collision. To get 5 9s over
132 // 1 million domains, we need the probability of a collision in any one domain
135 // p < nroot(1000000, .99999) ~= 10^-11
137 // We use the following birthday attack approximation to calculate the max
138 // number of unique names for this probability:
140 // n(p,H) = sqrt(2*H * ln(1/(1-p)))
142 // For a 6-byte hash, H = 2^(6*8). n(10^-11, H) ~= 75
144 // An average partition domain is likely to have less than 10 unique
145 // partition names which is far lower than 75.
147 // Note, that for 4 9s of reliability, the limit is 237 partition names per
149 const int kPartitionNameHashBytes
= 6;
151 // Needed for selecting all files in ObliterateOneDirectory() below.
152 #if defined(OS_POSIX)
153 const int kAllFileTypes
= base::FileEnumerator::FILES
|
154 base::FileEnumerator::DIRECTORIES
|
155 base::FileEnumerator::SHOW_SYM_LINKS
;
157 const int kAllFileTypes
= base::FileEnumerator::FILES
|
158 base::FileEnumerator::DIRECTORIES
;
161 base::FilePath
GetStoragePartitionDomainPath(
162 const std::string
& partition_domain
) {
163 CHECK(base::IsStringUTF8(partition_domain
));
165 return base::FilePath(kStoragePartitionDirname
).Append(kExtensionsDirname
)
166 .Append(base::FilePath::FromUTF8Unsafe(partition_domain
));
169 // Helper function for doing a depth-first deletion of the data on disk.
170 // Examines paths directly in |current_dir| (no recursion) and tries to
171 // delete from disk anything that is in, or isn't a parent of something in
172 // |paths_to_keep|. Paths that need further expansion are added to
173 // |paths_to_consider|.
174 void ObliterateOneDirectory(const base::FilePath
& current_dir
,
175 const std::vector
<base::FilePath
>& paths_to_keep
,
176 std::vector
<base::FilePath
>* paths_to_consider
) {
177 CHECK(current_dir
.IsAbsolute());
179 base::FileEnumerator
enumerator(current_dir
, false, kAllFileTypes
);
180 for (base::FilePath to_delete
= enumerator
.Next(); !to_delete
.empty();
181 to_delete
= enumerator
.Next()) {
182 // Enum tracking which of the 3 possible actions to take for |to_delete|.
183 enum { kSkip
, kEnqueue
, kDelete
} action
= kDelete
;
185 for (std::vector
<base::FilePath
>::const_iterator to_keep
=
186 paths_to_keep
.begin();
187 to_keep
!= paths_to_keep
.end();
189 if (to_delete
== *to_keep
) {
192 } else if (to_delete
.IsParent(*to_keep
)) {
193 // |to_delete| contains a path to keep. Add to stack for further
202 base::DeleteFile(to_delete
, true);
206 paths_to_consider
->push_back(to_delete
);
215 // Synchronously attempts to delete |unnormalized_root|, preserving only
216 // entries in |paths_to_keep|. If there are no entries in |paths_to_keep| on
217 // disk, then it completely removes |unnormalized_root|. All paths must be
219 void BlockingObliteratePath(
220 const base::FilePath
& unnormalized_browser_context_root
,
221 const base::FilePath
& unnormalized_root
,
222 const std::vector
<base::FilePath
>& paths_to_keep
,
223 const scoped_refptr
<base::TaskRunner
>& closure_runner
,
224 const base::Closure
& on_gc_required
) {
225 // Early exit required because MakeAbsoluteFilePath() will fail on POSIX
226 // if |unnormalized_root| does not exist. This is safe because there is
227 // nothing to do in this situation anwyays.
228 if (!base::PathExists(unnormalized_root
)) {
232 // Never try to obliterate things outside of the browser context root or the
233 // browser context root itself. Die hard.
234 base::FilePath root
= base::MakeAbsoluteFilePath(unnormalized_root
);
235 base::FilePath browser_context_root
=
236 base::MakeAbsoluteFilePath(unnormalized_browser_context_root
);
237 CHECK(!root
.empty());
238 CHECK(!browser_context_root
.empty());
239 CHECK(browser_context_root
.IsParent(root
) && browser_context_root
!= root
);
241 // Reduce |paths_to_keep| set to those under the root and actually on disk.
242 std::vector
<base::FilePath
> valid_paths_to_keep
;
243 for (std::vector
<base::FilePath
>::const_iterator it
= paths_to_keep
.begin();
244 it
!= paths_to_keep
.end();
246 if (root
.IsParent(*it
) && base::PathExists(*it
))
247 valid_paths_to_keep
.push_back(*it
);
250 // If none of the |paths_to_keep| are valid anymore then we just whack the
251 // root and be done with it. Otherwise, signal garbage collection and do
252 // a best-effort delete of the on-disk structures.
253 if (valid_paths_to_keep
.empty()) {
254 base::DeleteFile(root
, true);
257 closure_runner
->PostTask(FROM_HERE
, on_gc_required
);
259 // Otherwise, start at the root and delete everything that is not in
260 // |valid_paths_to_keep|.
261 std::vector
<base::FilePath
> paths_to_consider
;
262 paths_to_consider
.push_back(root
);
263 while(!paths_to_consider
.empty()) {
264 base::FilePath path
= paths_to_consider
.back();
265 paths_to_consider
.pop_back();
266 ObliterateOneDirectory(path
, valid_paths_to_keep
, &paths_to_consider
);
270 // Ensures each path in |active_paths| is a direct child of storage_root.
271 void NormalizeActivePaths(const base::FilePath
& storage_root
,
272 base::hash_set
<base::FilePath
>* active_paths
) {
273 base::hash_set
<base::FilePath
> normalized_active_paths
;
275 for (base::hash_set
<base::FilePath
>::iterator iter
= active_paths
->begin();
276 iter
!= active_paths
->end(); ++iter
) {
277 base::FilePath relative_path
;
278 if (!storage_root
.AppendRelativePath(*iter
, &relative_path
))
281 std::vector
<base::FilePath::StringType
> components
;
282 relative_path
.GetComponents(&components
);
284 DCHECK(!relative_path
.empty());
285 normalized_active_paths
.insert(storage_root
.Append(components
.front()));
288 active_paths
->swap(normalized_active_paths
);
291 // Deletes all entries inside the |storage_root| that are not in the
292 // |active_paths|. Deletion is done in 2 steps:
294 // (1) Moving all garbage collected paths into a trash directory.
295 // (2) Asynchronously deleting the trash directory.
297 // The deletion is asynchronous because after (1) completes, calling code can
298 // safely continue to use the paths that had just been garbage collected
299 // without fear of race conditions.
301 // This code also ignores failed moves rather than attempting a smarter retry.
302 // Moves shouldn't fail here unless there is some out-of-band error (eg.,
303 // FS corruption). Retry logic is dangerous in the general case because
304 // there is not necessarily a guaranteed case where the logic may succeed.
306 // This function is still named BlockingGarbageCollect() because it does
307 // execute a few filesystem operations synchronously.
308 void BlockingGarbageCollect(
309 const base::FilePath
& storage_root
,
310 const scoped_refptr
<base::TaskRunner
>& file_access_runner
,
311 scoped_ptr
<base::hash_set
<base::FilePath
> > active_paths
) {
312 CHECK(storage_root
.IsAbsolute());
314 NormalizeActivePaths(storage_root
, active_paths
.get());
316 base::FileEnumerator
enumerator(storage_root
, false, kAllFileTypes
);
317 base::FilePath trash_directory
;
318 if (!base::CreateTemporaryDirInDir(storage_root
, kTrashDirname
,
320 // Unable to continue without creating the trash directory so give up.
323 for (base::FilePath path
= enumerator
.Next(); !path
.empty();
324 path
= enumerator
.Next()) {
325 if (active_paths
->find(path
) == active_paths
->end() &&
326 path
!= trash_directory
) {
327 // Since |trash_directory| is unique for each run of this function there
328 // can be no colllisions on the move.
329 base::Move(path
, trash_directory
.Append(path
.BaseName()));
333 file_access_runner
->PostTask(
335 base::Bind(base::IgnoreResult(&base::DeleteFile
), trash_directory
, true));
341 base::FilePath
StoragePartitionImplMap::GetStoragePartitionPath(
342 const std::string
& partition_domain
,
343 const std::string
& partition_name
) {
344 if (partition_domain
.empty())
345 return base::FilePath();
347 base::FilePath path
= GetStoragePartitionDomainPath(partition_domain
);
349 // TODO(ajwong): Mangle in-memory into this somehow, either by putting
350 // it into the partition_name, or by manually adding another path component
351 // here. Otherwise, it's possible to have an in-memory StoragePartition and
352 // a persistent one that return the same FilePath for GetPath().
353 if (!partition_name
.empty()) {
354 // For analysis of why we can ignore collisions, see the comment above
355 // kPartitionNameHashBytes.
356 char buffer
[kPartitionNameHashBytes
];
357 crypto::SHA256HashString(partition_name
, &buffer
[0],
359 return path
.AppendASCII(base::HexEncode(buffer
, sizeof(buffer
)));
362 return path
.Append(kDefaultPartitionDirname
);
365 StoragePartitionImplMap::StoragePartitionImplMap(
366 BrowserContext
* browser_context
)
367 : browser_context_(browser_context
),
368 resource_context_initialized_(false) {
369 // Doing here instead of initializer list cause it's just too ugly to read.
370 base::SequencedWorkerPool
* blocking_pool
= BrowserThread::GetBlockingPool();
371 file_access_runner_
=
372 blocking_pool
->GetSequencedTaskRunner(blocking_pool
->GetSequenceToken());
375 StoragePartitionImplMap::~StoragePartitionImplMap() {
376 STLDeleteContainerPairSecondPointers(partitions_
.begin(),
380 StoragePartitionImpl
* StoragePartitionImplMap::Get(
381 const std::string
& partition_domain
,
382 const std::string
& partition_name
,
384 // Find the previously created partition if it's available.
385 StoragePartitionConfig
partition_config(
386 partition_domain
, partition_name
, in_memory
);
388 PartitionMap::const_iterator it
= partitions_
.find(partition_config
);
389 if (it
!= partitions_
.end())
392 base::FilePath partition_path
=
393 browser_context_
->GetPath().Append(
394 GetStoragePartitionPath(partition_domain
, partition_name
));
395 StoragePartitionImpl
* partition
=
396 StoragePartitionImpl::Create(browser_context_
, in_memory
,
398 partitions_
[partition_config
] = partition
;
400 ChromeBlobStorageContext
* blob_storage_context
=
401 ChromeBlobStorageContext::GetFor(browser_context_
);
402 StreamContext
* stream_context
= StreamContext::GetFor(browser_context_
);
403 ProtocolHandlerMap protocol_handlers
;
404 protocol_handlers
[url::kBlobScheme
] =
405 linked_ptr
<net::URLRequestJobFactory::ProtocolHandler
>(
406 new BlobProtocolHandler(blob_storage_context
,
408 partition
->GetFileSystemContext()));
409 protocol_handlers
[url::kFileSystemScheme
] =
410 linked_ptr
<net::URLRequestJobFactory::ProtocolHandler
>(
411 CreateFileSystemProtocolHandler(partition_domain
,
412 partition
->GetFileSystemContext()));
413 protocol_handlers
[kChromeUIScheme
] =
414 linked_ptr
<net::URLRequestJobFactory::ProtocolHandler
>(
415 URLDataManagerBackend::CreateProtocolHandler(
416 browser_context_
->GetResourceContext(),
417 browser_context_
->IsOffTheRecord(),
418 partition
->GetAppCacheService(),
419 blob_storage_context
));
420 std::vector
<std::string
> additional_webui_schemes
;
421 GetContentClient()->browser()->GetAdditionalWebUISchemes(
422 &additional_webui_schemes
);
423 for (std::vector
<std::string
>::const_iterator it
=
424 additional_webui_schemes
.begin();
425 it
!= additional_webui_schemes
.end();
427 protocol_handlers
[*it
] =
428 linked_ptr
<net::URLRequestJobFactory::ProtocolHandler
>(
429 URLDataManagerBackend::CreateProtocolHandler(
430 browser_context_
->GetResourceContext(),
431 browser_context_
->IsOffTheRecord(),
432 partition
->GetAppCacheService(),
433 blob_storage_context
));
435 protocol_handlers
[kChromeDevToolsScheme
] =
436 linked_ptr
<net::URLRequestJobFactory::ProtocolHandler
>(
437 CreateDevToolsProtocolHandler(browser_context_
->GetResourceContext(),
438 browser_context_
->IsOffTheRecord()));
440 URLRequestInterceptorScopedVector request_interceptors
;
441 request_interceptors
.push_back(
442 ServiceWorkerRequestHandler::CreateInterceptor(
443 browser_context_
->GetResourceContext()).release());
444 request_interceptors
.push_back(new AppCacheInterceptor());
446 // These calls must happen after StoragePartitionImpl::Create().
447 if (partition_domain
.empty()) {
448 partition
->SetURLRequestContext(
449 GetContentClient()->browser()->CreateRequestContext(
452 request_interceptors
.Pass()));
454 partition
->SetURLRequestContext(
455 GetContentClient()->browser()->CreateRequestContextForStoragePartition(
457 partition
->GetPath(),
460 request_interceptors
.Pass()));
462 partition
->SetMediaURLRequestContext(
463 partition_domain
.empty() ?
464 browser_context_
->GetMediaRequestContext() :
465 browser_context_
->GetMediaRequestContextForStoragePartition(
466 partition
->GetPath(), in_memory
));
468 GetContentClient()->browser()->GetAdditionalNavigatorConnectServices(
469 partition
->GetNavigatorConnectContext());
471 PostCreateInitialization(partition
, in_memory
);
476 void StoragePartitionImplMap::AsyncObliterate(
478 const base::Closure
& on_gc_required
) {
479 // This method should avoid creating any StoragePartition (which would
480 // create more open file handles) so that it can delete as much of the
481 // data off disk as possible.
482 std::string partition_domain
;
483 std::string partition_name
;
484 bool in_memory
= false;
485 GetContentClient()->browser()->GetStoragePartitionConfigForSite(
486 browser_context_
, site
, false, &partition_domain
,
487 &partition_name
, &in_memory
);
489 // Find the active partitions for the domain. Because these partitions are
490 // active, it is not possible to just delete the directories that contain
491 // the backing data structures without causing the browser to crash. Instead,
492 // of deleteing the directory, we tell each storage context later to
493 // remove any data they have saved. This will leave the directory structure
494 // intact but it will only contain empty databases.
495 std::vector
<StoragePartitionImpl
*> active_partitions
;
496 std::vector
<base::FilePath
> paths_to_keep
;
497 for (PartitionMap::const_iterator it
= partitions_
.begin();
498 it
!= partitions_
.end();
500 const StoragePartitionConfig
& config
= it
->first
;
501 if (config
.partition_domain
== partition_domain
) {
502 it
->second
->ClearData(
503 // All except shader cache.
504 ~StoragePartition::REMOVE_DATA_MASK_SHADER_CACHE
,
505 StoragePartition::QUOTA_MANAGED_STORAGE_MASK_ALL
,
507 StoragePartition::OriginMatcherFunction(),
508 base::Time(), base::Time::Max(),
509 base::Bind(&base::DoNothing
));
510 if (!config
.in_memory
) {
511 paths_to_keep
.push_back(it
->second
->GetPath());
516 // Start a best-effort delete of the on-disk storage excluding paths that are
517 // known to still be in use. This is to delete any previously created
518 // StoragePartition state that just happens to not have been used during this
519 // run of the browser.
520 base::FilePath domain_root
= browser_context_
->GetPath().Append(
521 GetStoragePartitionDomainPath(partition_domain
));
523 BrowserThread::PostBlockingPoolTask(
525 base::Bind(&BlockingObliteratePath
, browser_context_
->GetPath(),
526 domain_root
, paths_to_keep
,
527 base::ThreadTaskRunnerHandle::Get(), on_gc_required
));
530 void StoragePartitionImplMap::GarbageCollect(
531 scoped_ptr
<base::hash_set
<base::FilePath
> > active_paths
,
532 const base::Closure
& done
) {
533 // Include all paths for current StoragePartitions in the active_paths since
534 // they cannot be deleted safely.
535 for (PartitionMap::const_iterator it
= partitions_
.begin();
536 it
!= partitions_
.end();
538 const StoragePartitionConfig
& config
= it
->first
;
539 if (!config
.in_memory
)
540 active_paths
->insert(it
->second
->GetPath());
543 // Find the directory holding the StoragePartitions and delete everything in
544 // there that isn't considered active.
545 base::FilePath storage_root
= browser_context_
->GetPath().Append(
546 GetStoragePartitionDomainPath(std::string()));
547 file_access_runner_
->PostTaskAndReply(
549 base::Bind(&BlockingGarbageCollect
, storage_root
,
551 base::Passed(&active_paths
)),
555 void StoragePartitionImplMap::ForEach(
556 const BrowserContext::StoragePartitionCallback
& callback
) {
557 for (PartitionMap::const_iterator it
= partitions_
.begin();
558 it
!= partitions_
.end();
560 callback
.Run(it
->second
);
564 void StoragePartitionImplMap::PostCreateInitialization(
565 StoragePartitionImpl
* partition
,
567 // TODO(ajwong): ResourceContexts no longer have any storage related state.
568 // We should move this into a place where it is called once per
569 // BrowserContext creation rather than piggybacking off the default context
571 // Note: moving this into Get() before partitions_[] is set causes reentrency.
572 if (!resource_context_initialized_
) {
573 resource_context_initialized_
= true;
574 InitializeResourceContext(browser_context_
);
577 // Check first to avoid memory leak in unittests.
578 if (BrowserThread::IsMessageLoopValid(BrowserThread::IO
)) {
579 BrowserThread::PostTask(
580 BrowserThread::IO
, FROM_HERE
,
581 base::Bind(&ChromeAppCacheService::InitializeOnIOThread
,
582 partition
->GetAppCacheService(),
583 in_memory
? base::FilePath() :
584 partition
->GetPath().Append(kAppCacheDirname
),
585 browser_context_
->GetResourceContext(),
586 make_scoped_refptr(partition
->GetURLRequestContext()),
588 browser_context_
->GetSpecialStoragePolicy())));
590 BrowserThread::PostTask(
591 BrowserThread::IO
, FROM_HERE
,
592 base::Bind(&CacheStorageContextImpl::SetBlobParametersForCache
,
593 partition
->GetCacheStorageContext(),
594 make_scoped_refptr(partition
->GetURLRequestContext()),
596 ChromeBlobStorageContext::GetFor(browser_context_
))));
598 // We do not call InitializeURLRequestContext() for media contexts because,
599 // other than the HTTP cache, the media contexts share the same backing
600 // objects as their associated "normal" request context. Thus, the previous
601 // call serves to initialize the media request context for this storage
602 // partition as well.
606 } // namespace content