Add remoting and PPAPI tests to GN build
[chromium-blink-merge.git] / content / browser / storage_partition_impl_map.cc
blob00fb52fa608474e4e4cc8f2f46150714bcbbce6a
1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "content/browser/storage_partition_impl_map.h"
7 #include "base/bind.h"
8 #include "base/callback.h"
9 #include "base/files/file_enumerator.h"
10 #include "base/files/file_path.h"
11 #include "base/files/file_util.h"
12 #include "base/stl_util.h"
13 #include "base/strings/string_number_conversions.h"
14 #include "base/strings/string_util.h"
15 #include "base/strings/stringprintf.h"
16 #include "base/threading/sequenced_worker_pool.h"
17 #include "content/browser/appcache/appcache_interceptor.h"
18 #include "content/browser/appcache/chrome_appcache_service.h"
19 #include "content/browser/fileapi/browser_file_system_helper.h"
20 #include "content/browser/fileapi/chrome_blob_storage_context.h"
21 #include "content/browser/loader/resource_request_info_impl.h"
22 #include "content/browser/resource_context_impl.h"
23 #include "content/browser/service_worker/service_worker_request_handler.h"
24 #include "content/browser/storage_partition_impl.h"
25 #include "content/browser/streams/stream.h"
26 #include "content/browser/streams/stream_context.h"
27 #include "content/browser/streams/stream_registry.h"
28 #include "content/browser/streams/stream_url_request_job.h"
29 #include "content/browser/webui/url_data_manager_backend.h"
30 #include "content/public/browser/browser_context.h"
31 #include "content/public/browser/browser_thread.h"
32 #include "content/public/browser/content_browser_client.h"
33 #include "content/public/browser/navigator_connect_context.h"
34 #include "content/public/browser/navigator_connect_service_factory.h"
35 #include "content/public/browser/storage_partition.h"
36 #include "content/public/common/content_constants.h"
37 #include "content/public/common/url_constants.h"
38 #include "crypto/sha2.h"
39 #include "net/url_request/url_request_context.h"
40 #include "net/url_request/url_request_context_getter.h"
41 #include "storage/browser/blob/blob_storage_context.h"
42 #include "storage/browser/blob/blob_url_request_job_factory.h"
43 #include "storage/browser/fileapi/file_system_url_request_job_factory.h"
45 using storage::FileSystemContext;
46 using storage::BlobStorageContext;
48 namespace content {
50 namespace {
52 // A derivative that knows about Streams too.
53 class BlobProtocolHandler : public net::URLRequestJobFactory::ProtocolHandler {
54 public:
55 BlobProtocolHandler(ChromeBlobStorageContext* blob_storage_context,
56 StreamContext* stream_context,
57 storage::FileSystemContext* file_system_context)
58 : blob_storage_context_(blob_storage_context),
59 stream_context_(stream_context),
60 file_system_context_(file_system_context) {}
62 ~BlobProtocolHandler() override {}
64 net::URLRequestJob* MaybeCreateJob(
65 net::URLRequest* request,
66 net::NetworkDelegate* network_delegate) const override {
67 scoped_refptr<Stream> stream =
68 stream_context_->registry()->GetStream(request->url());
69 if (stream.get())
70 return new StreamURLRequestJob(request, network_delegate, stream);
72 if (!blob_protocol_handler_) {
73 // Construction is deferred because 'this' is constructed on
74 // the main thread but we want blob_protocol_handler_ constructed
75 // on the IO thread.
76 blob_protocol_handler_.reset(new storage::BlobProtocolHandler(
77 blob_storage_context_->context(),
78 file_system_context_.get(),
79 BrowserThread::GetMessageLoopProxyForThread(BrowserThread::FILE)
80 .get()));
82 return blob_protocol_handler_->MaybeCreateJob(request, network_delegate);
85 private:
86 const scoped_refptr<ChromeBlobStorageContext> blob_storage_context_;
87 const scoped_refptr<StreamContext> stream_context_;
88 const scoped_refptr<storage::FileSystemContext> file_system_context_;
89 mutable scoped_ptr<storage::BlobProtocolHandler> blob_protocol_handler_;
90 DISALLOW_COPY_AND_ASSIGN(BlobProtocolHandler);
93 // These constants are used to create the directory structure under the profile
94 // where renderers with a non-default storage partition keep their persistent
95 // state. This will contain a set of directories that partially mirror the
96 // directory structure of BrowserContext::GetPath().
98 // The kStoragePartitionDirname contains an extensions directory which is
99 // further partitioned by extension id, followed by another level of directories
100 // for the "default" extension storage partition and one directory for each
101 // persistent partition used by a webview tag. Example:
103 // Storage/ext/ABCDEF/def
104 // Storage/ext/ABCDEF/hash(partition name)
106 // The code in GetStoragePartitionPath() constructs these path names.
108 // TODO(nasko): Move extension related path code out of content.
109 const base::FilePath::CharType kStoragePartitionDirname[] =
110 FILE_PATH_LITERAL("Storage");
111 const base::FilePath::CharType kExtensionsDirname[] =
112 FILE_PATH_LITERAL("ext");
113 const base::FilePath::CharType kDefaultPartitionDirname[] =
114 FILE_PATH_LITERAL("def");
115 const base::FilePath::CharType kTrashDirname[] =
116 FILE_PATH_LITERAL("trash");
118 // Because partition names are user specified, they can be arbitrarily long
119 // which makes them unsuitable for paths names. We use a truncation of a
120 // SHA256 hash to perform a deterministic shortening of the string. The
121 // kPartitionNameHashBytes constant controls the length of the truncation.
122 // We use 6 bytes, which gives us 99.999% reliability against collisions over
123 // 1 million partition domains.
125 // Analysis:
126 // We assume that all partition names within one partition domain are
127 // controlled by the the same entity. Thus there is no chance for adverserial
128 // attack and all we care about is accidental collision. To get 5 9s over
129 // 1 million domains, we need the probability of a collision in any one domain
130 // to be
132 // p < nroot(1000000, .99999) ~= 10^-11
134 // We use the following birthday attack approximation to calculate the max
135 // number of unique names for this probability:
137 // n(p,H) = sqrt(2*H * ln(1/(1-p)))
139 // For a 6-byte hash, H = 2^(6*8). n(10^-11, H) ~= 75
141 // An average partition domain is likely to have less than 10 unique
142 // partition names which is far lower than 75.
144 // Note, that for 4 9s of reliability, the limit is 237 partition names per
145 // partition domain.
146 const int kPartitionNameHashBytes = 6;
148 // Needed for selecting all files in ObliterateOneDirectory() below.
149 #if defined(OS_POSIX)
150 const int kAllFileTypes = base::FileEnumerator::FILES |
151 base::FileEnumerator::DIRECTORIES |
152 base::FileEnumerator::SHOW_SYM_LINKS;
153 #else
154 const int kAllFileTypes = base::FileEnumerator::FILES |
155 base::FileEnumerator::DIRECTORIES;
156 #endif
158 base::FilePath GetStoragePartitionDomainPath(
159 const std::string& partition_domain) {
160 CHECK(base::IsStringUTF8(partition_domain));
162 return base::FilePath(kStoragePartitionDirname).Append(kExtensionsDirname)
163 .Append(base::FilePath::FromUTF8Unsafe(partition_domain));
166 // Helper function for doing a depth-first deletion of the data on disk.
167 // Examines paths directly in |current_dir| (no recursion) and tries to
168 // delete from disk anything that is in, or isn't a parent of something in
169 // |paths_to_keep|. Paths that need further expansion are added to
170 // |paths_to_consider|.
171 void ObliterateOneDirectory(const base::FilePath& current_dir,
172 const std::vector<base::FilePath>& paths_to_keep,
173 std::vector<base::FilePath>* paths_to_consider) {
174 CHECK(current_dir.IsAbsolute());
176 base::FileEnumerator enumerator(current_dir, false, kAllFileTypes);
177 for (base::FilePath to_delete = enumerator.Next(); !to_delete.empty();
178 to_delete = enumerator.Next()) {
179 // Enum tracking which of the 3 possible actions to take for |to_delete|.
180 enum { kSkip, kEnqueue, kDelete } action = kDelete;
182 for (std::vector<base::FilePath>::const_iterator to_keep =
183 paths_to_keep.begin();
184 to_keep != paths_to_keep.end();
185 ++to_keep) {
186 if (to_delete == *to_keep) {
187 action = kSkip;
188 break;
189 } else if (to_delete.IsParent(*to_keep)) {
190 // |to_delete| contains a path to keep. Add to stack for further
191 // processing.
192 action = kEnqueue;
193 break;
197 switch (action) {
198 case kDelete:
199 base::DeleteFile(to_delete, true);
200 break;
202 case kEnqueue:
203 paths_to_consider->push_back(to_delete);
204 break;
206 case kSkip:
207 break;
212 // Synchronously attempts to delete |unnormalized_root|, preserving only
213 // entries in |paths_to_keep|. If there are no entries in |paths_to_keep| on
214 // disk, then it completely removes |unnormalized_root|. All paths must be
215 // absolute paths.
216 void BlockingObliteratePath(
217 const base::FilePath& unnormalized_browser_context_root,
218 const base::FilePath& unnormalized_root,
219 const std::vector<base::FilePath>& paths_to_keep,
220 const scoped_refptr<base::TaskRunner>& closure_runner,
221 const base::Closure& on_gc_required) {
222 // Early exit required because MakeAbsoluteFilePath() will fail on POSIX
223 // if |unnormalized_root| does not exist. This is safe because there is
224 // nothing to do in this situation anwyays.
225 if (!base::PathExists(unnormalized_root)) {
226 return;
229 // Never try to obliterate things outside of the browser context root or the
230 // browser context root itself. Die hard.
231 base::FilePath root = base::MakeAbsoluteFilePath(unnormalized_root);
232 base::FilePath browser_context_root =
233 base::MakeAbsoluteFilePath(unnormalized_browser_context_root);
234 CHECK(!root.empty());
235 CHECK(!browser_context_root.empty());
236 CHECK(browser_context_root.IsParent(root) && browser_context_root != root);
238 // Reduce |paths_to_keep| set to those under the root and actually on disk.
239 std::vector<base::FilePath> valid_paths_to_keep;
240 for (std::vector<base::FilePath>::const_iterator it = paths_to_keep.begin();
241 it != paths_to_keep.end();
242 ++it) {
243 if (root.IsParent(*it) && base::PathExists(*it))
244 valid_paths_to_keep.push_back(*it);
247 // If none of the |paths_to_keep| are valid anymore then we just whack the
248 // root and be done with it. Otherwise, signal garbage collection and do
249 // a best-effort delete of the on-disk structures.
250 if (valid_paths_to_keep.empty()) {
251 base::DeleteFile(root, true);
252 return;
254 closure_runner->PostTask(FROM_HERE, on_gc_required);
256 // Otherwise, start at the root and delete everything that is not in
257 // |valid_paths_to_keep|.
258 std::vector<base::FilePath> paths_to_consider;
259 paths_to_consider.push_back(root);
260 while(!paths_to_consider.empty()) {
261 base::FilePath path = paths_to_consider.back();
262 paths_to_consider.pop_back();
263 ObliterateOneDirectory(path, valid_paths_to_keep, &paths_to_consider);
267 // Ensures each path in |active_paths| is a direct child of storage_root.
268 void NormalizeActivePaths(const base::FilePath& storage_root,
269 base::hash_set<base::FilePath>* active_paths) {
270 base::hash_set<base::FilePath> normalized_active_paths;
272 for (base::hash_set<base::FilePath>::iterator iter = active_paths->begin();
273 iter != active_paths->end(); ++iter) {
274 base::FilePath relative_path;
275 if (!storage_root.AppendRelativePath(*iter, &relative_path))
276 continue;
278 std::vector<base::FilePath::StringType> components;
279 relative_path.GetComponents(&components);
281 DCHECK(!relative_path.empty());
282 normalized_active_paths.insert(storage_root.Append(components.front()));
285 active_paths->swap(normalized_active_paths);
288 // Deletes all entries inside the |storage_root| that are not in the
289 // |active_paths|. Deletion is done in 2 steps:
291 // (1) Moving all garbage collected paths into a trash directory.
292 // (2) Asynchronously deleting the trash directory.
294 // The deletion is asynchronous because after (1) completes, calling code can
295 // safely continue to use the paths that had just been garbage collected
296 // without fear of race conditions.
298 // This code also ignores failed moves rather than attempting a smarter retry.
299 // Moves shouldn't fail here unless there is some out-of-band error (eg.,
300 // FS corruption). Retry logic is dangerous in the general case because
301 // there is not necessarily a guaranteed case where the logic may succeed.
303 // This function is still named BlockingGarbageCollect() because it does
304 // execute a few filesystem operations synchronously.
305 void BlockingGarbageCollect(
306 const base::FilePath& storage_root,
307 const scoped_refptr<base::TaskRunner>& file_access_runner,
308 scoped_ptr<base::hash_set<base::FilePath> > active_paths) {
309 CHECK(storage_root.IsAbsolute());
311 NormalizeActivePaths(storage_root, active_paths.get());
313 base::FileEnumerator enumerator(storage_root, false, kAllFileTypes);
314 base::FilePath trash_directory;
315 if (!base::CreateTemporaryDirInDir(storage_root, kTrashDirname,
316 &trash_directory)) {
317 // Unable to continue without creating the trash directory so give up.
318 return;
320 for (base::FilePath path = enumerator.Next(); !path.empty();
321 path = enumerator.Next()) {
322 if (active_paths->find(path) == active_paths->end() &&
323 path != trash_directory) {
324 // Since |trash_directory| is unique for each run of this function there
325 // can be no colllisions on the move.
326 base::Move(path, trash_directory.Append(path.BaseName()));
330 file_access_runner->PostTask(
331 FROM_HERE,
332 base::Bind(base::IgnoreResult(&base::DeleteFile), trash_directory, true));
335 } // namespace
337 // static
338 base::FilePath StoragePartitionImplMap::GetStoragePartitionPath(
339 const std::string& partition_domain,
340 const std::string& partition_name) {
341 if (partition_domain.empty())
342 return base::FilePath();
344 base::FilePath path = GetStoragePartitionDomainPath(partition_domain);
346 // TODO(ajwong): Mangle in-memory into this somehow, either by putting
347 // it into the partition_name, or by manually adding another path component
348 // here. Otherwise, it's possible to have an in-memory StoragePartition and
349 // a persistent one that return the same FilePath for GetPath().
350 if (!partition_name.empty()) {
351 // For analysis of why we can ignore collisions, see the comment above
352 // kPartitionNameHashBytes.
353 char buffer[kPartitionNameHashBytes];
354 crypto::SHA256HashString(partition_name, &buffer[0],
355 sizeof(buffer));
356 return path.AppendASCII(base::HexEncode(buffer, sizeof(buffer)));
359 return path.Append(kDefaultPartitionDirname);
362 StoragePartitionImplMap::StoragePartitionImplMap(
363 BrowserContext* browser_context)
364 : browser_context_(browser_context),
365 resource_context_initialized_(false) {
366 // Doing here instead of initializer list cause it's just too ugly to read.
367 base::SequencedWorkerPool* blocking_pool = BrowserThread::GetBlockingPool();
368 file_access_runner_ =
369 blocking_pool->GetSequencedTaskRunner(blocking_pool->GetSequenceToken());
372 StoragePartitionImplMap::~StoragePartitionImplMap() {
373 STLDeleteContainerPairSecondPointers(partitions_.begin(),
374 partitions_.end());
377 StoragePartitionImpl* StoragePartitionImplMap::Get(
378 const std::string& partition_domain,
379 const std::string& partition_name,
380 bool in_memory) {
381 // Find the previously created partition if it's available.
382 StoragePartitionConfig partition_config(
383 partition_domain, partition_name, in_memory);
385 PartitionMap::const_iterator it = partitions_.find(partition_config);
386 if (it != partitions_.end())
387 return it->second;
389 base::FilePath partition_path =
390 browser_context_->GetPath().Append(
391 GetStoragePartitionPath(partition_domain, partition_name));
392 StoragePartitionImpl* partition =
393 StoragePartitionImpl::Create(browser_context_, in_memory,
394 partition_path);
395 partitions_[partition_config] = partition;
397 ChromeBlobStorageContext* blob_storage_context =
398 ChromeBlobStorageContext::GetFor(browser_context_);
399 StreamContext* stream_context = StreamContext::GetFor(browser_context_);
400 ProtocolHandlerMap protocol_handlers;
401 protocol_handlers[url::kBlobScheme] =
402 linked_ptr<net::URLRequestJobFactory::ProtocolHandler>(
403 new BlobProtocolHandler(blob_storage_context,
404 stream_context,
405 partition->GetFileSystemContext()));
406 protocol_handlers[url::kFileSystemScheme] =
407 linked_ptr<net::URLRequestJobFactory::ProtocolHandler>(
408 CreateFileSystemProtocolHandler(partition_domain,
409 partition->GetFileSystemContext()));
410 protocol_handlers[kChromeUIScheme] =
411 linked_ptr<net::URLRequestJobFactory::ProtocolHandler>(
412 URLDataManagerBackend::CreateProtocolHandler(
413 browser_context_->GetResourceContext(),
414 browser_context_->IsOffTheRecord(),
415 partition->GetAppCacheService(),
416 blob_storage_context));
417 std::vector<std::string> additional_webui_schemes;
418 GetContentClient()->browser()->GetAdditionalWebUISchemes(
419 &additional_webui_schemes);
420 for (std::vector<std::string>::const_iterator it =
421 additional_webui_schemes.begin();
422 it != additional_webui_schemes.end();
423 ++it) {
424 protocol_handlers[*it] =
425 linked_ptr<net::URLRequestJobFactory::ProtocolHandler>(
426 URLDataManagerBackend::CreateProtocolHandler(
427 browser_context_->GetResourceContext(),
428 browser_context_->IsOffTheRecord(),
429 partition->GetAppCacheService(),
430 blob_storage_context));
432 protocol_handlers[kChromeDevToolsScheme] =
433 linked_ptr<net::URLRequestJobFactory::ProtocolHandler>(
434 CreateDevToolsProtocolHandler(browser_context_->GetResourceContext(),
435 browser_context_->IsOffTheRecord()));
437 URLRequestInterceptorScopedVector request_interceptors;
438 request_interceptors.push_back(
439 ServiceWorkerRequestHandler::CreateInterceptor(
440 browser_context_->GetResourceContext()).release());
441 request_interceptors.push_back(new AppCacheInterceptor());
443 // These calls must happen after StoragePartitionImpl::Create().
444 if (partition_domain.empty()) {
445 partition->SetURLRequestContext(
446 GetContentClient()->browser()->CreateRequestContext(
447 browser_context_,
448 &protocol_handlers,
449 request_interceptors.Pass()));
450 } else {
451 partition->SetURLRequestContext(
452 GetContentClient()->browser()->CreateRequestContextForStoragePartition(
453 browser_context_,
454 partition->GetPath(),
455 in_memory,
456 &protocol_handlers,
457 request_interceptors.Pass()));
459 partition->SetMediaURLRequestContext(
460 partition_domain.empty() ?
461 browser_context_->GetMediaRequestContext() :
462 browser_context_->GetMediaRequestContextForStoragePartition(
463 partition->GetPath(), in_memory));
465 GetContentClient()->browser()->GetAdditionalNavigatorConnectServices(
466 partition->GetNavigatorConnectContext());
468 PostCreateInitialization(partition, in_memory);
470 return partition;
473 void StoragePartitionImplMap::AsyncObliterate(
474 const GURL& site,
475 const base::Closure& on_gc_required) {
476 // This method should avoid creating any StoragePartition (which would
477 // create more open file handles) so that it can delete as much of the
478 // data off disk as possible.
479 std::string partition_domain;
480 std::string partition_name;
481 bool in_memory = false;
482 GetContentClient()->browser()->GetStoragePartitionConfigForSite(
483 browser_context_, site, false, &partition_domain,
484 &partition_name, &in_memory);
486 // Find the active partitions for the domain. Because these partitions are
487 // active, it is not possible to just delete the directories that contain
488 // the backing data structures without causing the browser to crash. Instead,
489 // of deleteing the directory, we tell each storage context later to
490 // remove any data they have saved. This will leave the directory structure
491 // intact but it will only contain empty databases.
492 std::vector<StoragePartitionImpl*> active_partitions;
493 std::vector<base::FilePath> paths_to_keep;
494 for (PartitionMap::const_iterator it = partitions_.begin();
495 it != partitions_.end();
496 ++it) {
497 const StoragePartitionConfig& config = it->first;
498 if (config.partition_domain == partition_domain) {
499 it->second->ClearData(
500 // All except shader cache.
501 ~StoragePartition::REMOVE_DATA_MASK_SHADER_CACHE,
502 StoragePartition::QUOTA_MANAGED_STORAGE_MASK_ALL,
503 GURL(),
504 StoragePartition::OriginMatcherFunction(),
505 base::Time(), base::Time::Max(),
506 base::Bind(&base::DoNothing));
507 if (!config.in_memory) {
508 paths_to_keep.push_back(it->second->GetPath());
513 // Start a best-effort delete of the on-disk storage excluding paths that are
514 // known to still be in use. This is to delete any previously created
515 // StoragePartition state that just happens to not have been used during this
516 // run of the browser.
517 base::FilePath domain_root = browser_context_->GetPath().Append(
518 GetStoragePartitionDomainPath(partition_domain));
520 BrowserThread::PostBlockingPoolTask(
521 FROM_HERE,
522 base::Bind(&BlockingObliteratePath, browser_context_->GetPath(),
523 domain_root, paths_to_keep,
524 base::MessageLoopProxy::current(), on_gc_required));
527 void StoragePartitionImplMap::GarbageCollect(
528 scoped_ptr<base::hash_set<base::FilePath> > active_paths,
529 const base::Closure& done) {
530 // Include all paths for current StoragePartitions in the active_paths since
531 // they cannot be deleted safely.
532 for (PartitionMap::const_iterator it = partitions_.begin();
533 it != partitions_.end();
534 ++it) {
535 const StoragePartitionConfig& config = it->first;
536 if (!config.in_memory)
537 active_paths->insert(it->second->GetPath());
540 // Find the directory holding the StoragePartitions and delete everything in
541 // there that isn't considered active.
542 base::FilePath storage_root = browser_context_->GetPath().Append(
543 GetStoragePartitionDomainPath(std::string()));
544 file_access_runner_->PostTaskAndReply(
545 FROM_HERE,
546 base::Bind(&BlockingGarbageCollect, storage_root,
547 file_access_runner_,
548 base::Passed(&active_paths)),
549 done);
552 void StoragePartitionImplMap::ForEach(
553 const BrowserContext::StoragePartitionCallback& callback) {
554 for (PartitionMap::const_iterator it = partitions_.begin();
555 it != partitions_.end();
556 ++it) {
557 callback.Run(it->second);
561 void StoragePartitionImplMap::PostCreateInitialization(
562 StoragePartitionImpl* partition,
563 bool in_memory) {
564 // TODO(ajwong): ResourceContexts no longer have any storage related state.
565 // We should move this into a place where it is called once per
566 // BrowserContext creation rather than piggybacking off the default context
567 // creation.
568 // Note: moving this into Get() before partitions_[] is set causes reentrency.
569 if (!resource_context_initialized_) {
570 resource_context_initialized_ = true;
571 InitializeResourceContext(browser_context_);
574 // Check first to avoid memory leak in unittests.
575 if (BrowserThread::IsMessageLoopValid(BrowserThread::IO)) {
576 BrowserThread::PostTask(
577 BrowserThread::IO, FROM_HERE,
578 base::Bind(&ChromeAppCacheService::InitializeOnIOThread,
579 partition->GetAppCacheService(),
580 in_memory ? base::FilePath() :
581 partition->GetPath().Append(kAppCacheDirname),
582 browser_context_->GetResourceContext(),
583 make_scoped_refptr(partition->GetURLRequestContext()),
584 make_scoped_refptr(
585 browser_context_->GetSpecialStoragePolicy())));
587 BrowserThread::PostTask(
588 BrowserThread::IO,
589 FROM_HERE,
590 base::Bind(&ServiceWorkerContextWrapper::SetBlobParametersForCache,
591 partition->GetServiceWorkerContext(),
592 make_scoped_refptr(partition->GetURLRequestContext()),
593 make_scoped_refptr(
594 ChromeBlobStorageContext::GetFor(browser_context_))));
596 // We do not call InitializeURLRequestContext() for media contexts because,
597 // other than the HTTP cache, the media contexts share the same backing
598 // objects as their associated "normal" request context. Thus, the previous
599 // call serves to initialize the media request context for this storage
600 // partition as well.
604 } // namespace content