[content shell] implement testRunner.overridePreference
[chromium-blink-merge.git] / content / browser / storage_partition_impl_map.cc
blobe1381e12c6f01270caad0225a5c46428e24f46c3
1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "content/browser/storage_partition_impl_map.h"
7 #include "base/bind.h"
8 #include "base/callback.h"
9 #include "base/file_path.h"
10 #include "base/file_util.h"
11 #include "base/stl_util.h"
12 #include "base/string_number_conversions.h"
13 #include "base/string_util.h"
14 #include "base/stringprintf.h"
15 #include "base/threading/sequenced_worker_pool.h"
16 #include "content/browser/appcache/chrome_appcache_service.h"
17 #include "content/browser/fileapi/browser_file_system_helper.h"
18 #include "content/browser/fileapi/chrome_blob_storage_context.h"
19 #include "content/browser/histogram_internals_request_job.h"
20 #include "content/browser/loader/resource_request_info_impl.h"
21 #include "content/browser/net/view_blob_internals_job_factory.h"
22 #include "content/browser/net/view_http_cache_job_factory.h"
23 #include "content/browser/resource_context_impl.h"
24 #include "content/browser/storage_partition_impl.h"
25 #include "content/browser/tcmalloc_internals_request_job.h"
26 #include "content/public/browser/browser_context.h"
27 #include "content/public/browser/browser_thread.h"
28 #include "content/public/browser/content_browser_client.h"
29 #include "content/public/browser/storage_partition.h"
30 #include "content/public/common/content_constants.h"
31 #include "content/public/common/url_constants.h"
32 #include "crypto/sha2.h"
33 #include "net/url_request/url_request_context_getter.h"
34 #include "net/url_request/url_request_context.h"
35 #include "webkit/appcache/view_appcache_internals_job.h"
36 #include "webkit/blob/blob_data.h"
37 #include "webkit/blob/blob_url_request_job_factory.h"
38 #include "webkit/fileapi/file_system_url_request_job_factory.h"
40 using appcache::AppCacheService;
41 using fileapi::FileSystemContext;
42 using webkit_blob::BlobStorageController;
44 namespace content {
46 namespace {
48 class BlobProtocolHandler : public webkit_blob::BlobProtocolHandler {
49 public:
50 BlobProtocolHandler(
51 webkit_blob::BlobStorageController* blob_storage_controller,
52 fileapi::FileSystemContext* file_system_context,
53 base::MessageLoopProxy* loop_proxy)
54 : webkit_blob::BlobProtocolHandler(blob_storage_controller,
55 file_system_context,
56 loop_proxy) {}
58 virtual ~BlobProtocolHandler() {}
60 private:
61 virtual scoped_refptr<webkit_blob::BlobData>
62 LookupBlobData(net::URLRequest* request) const {
63 const ResourceRequestInfoImpl* info =
64 ResourceRequestInfoImpl::ForRequest(request);
65 if (!info)
66 return NULL;
67 return info->requested_blob_data();
70 DISALLOW_COPY_AND_ASSIGN(BlobProtocolHandler);
73 // Adds a bunch of debugging urls. We use an interceptor instead of a protocol
74 // handler because we want to reuse the chrome://scheme (everyone is familiar
75 // with it, and no need to expose the content/chrome separation through our UI).
76 class DeveloperProtocolHandler
77 : public net::URLRequestJobFactory::Interceptor {
78 public:
79 DeveloperProtocolHandler(
80 AppCacheService* appcache_service,
81 BlobStorageController* blob_storage_controller)
82 : appcache_service_(appcache_service),
83 blob_storage_controller_(blob_storage_controller) {}
84 virtual ~DeveloperProtocolHandler() {}
86 virtual net::URLRequestJob* MaybeIntercept(
87 net::URLRequest* request,
88 net::NetworkDelegate* network_delegate) const OVERRIDE {
89 // Check for chrome://view-http-cache/*, which uses its own job type.
90 if (ViewHttpCacheJobFactory::IsSupportedURL(request->url()))
91 return ViewHttpCacheJobFactory::CreateJobForRequest(request,
92 network_delegate);
94 // Next check for chrome://appcache-internals/, which uses its own job type.
95 if (request->url().SchemeIs(chrome::kChromeUIScheme) &&
96 request->url().host() == chrome::kChromeUIAppCacheInternalsHost) {
97 return appcache::ViewAppCacheInternalsJobFactory::CreateJobForRequest(
98 request, network_delegate, appcache_service_);
101 // Next check for chrome://blob-internals/, which uses its own job type.
102 if (ViewBlobInternalsJobFactory::IsSupportedURL(request->url())) {
103 return ViewBlobInternalsJobFactory::CreateJobForRequest(
104 request, network_delegate, blob_storage_controller_);
107 #if defined(USE_TCMALLOC)
108 // Next check for chrome://tcmalloc/, which uses its own job type.
109 if (request->url().SchemeIs(chrome::kChromeUIScheme) &&
110 request->url().host() == chrome::kChromeUITcmallocHost) {
111 return new TcmallocInternalsRequestJob(request, network_delegate);
113 #endif
115 // Next check for chrome://histograms/, which uses its own job type.
116 if (request->url().SchemeIs(chrome::kChromeUIScheme) &&
117 request->url().host() == chrome::kChromeUIHistogramHost) {
118 return new HistogramInternalsRequestJob(request, network_delegate);
121 return NULL;
124 virtual net::URLRequestJob* MaybeInterceptRedirect(
125 const GURL& location,
126 net::URLRequest* request,
127 net::NetworkDelegate* network_delegate) const OVERRIDE {
128 return NULL;
131 virtual net::URLRequestJob* MaybeInterceptResponse(
132 net::URLRequest* request,
133 net::NetworkDelegate* network_delegate) const OVERRIDE {
134 return NULL;
137 virtual bool WillHandleProtocol(const std::string& protocol) const {
138 return protocol == chrome::kChromeUIScheme;
141 private:
142 AppCacheService* appcache_service_;
143 BlobStorageController* blob_storage_controller_;
146 void InitializeURLRequestContext(
147 net::URLRequestContextGetter* context_getter,
148 AppCacheService* appcache_service,
149 FileSystemContext* file_system_context,
150 ChromeBlobStorageContext* blob_storage_context) {
151 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
152 if (!context_getter)
153 return; // tests.
155 // This code only modifies the URLRequestJobFactory on the context
156 // to handle blob: URLs, filesystem: URLs, and to let AppCache intercept
157 // the appropriate requests. This is in addition to the slew of other
158 // initializtion that is done in during creation of the URLRequestContext.
159 // We cannot yet centralize this code because URLRequestContext needs
160 // to be created before the StoragePartition context.
162 // TODO(ajwong): Fix the ordering so all the initialization is in one spot.
163 net::URLRequestContext* context = context_getter->GetURLRequestContext();
164 net::URLRequestJobFactory* job_factory =
165 const_cast<net::URLRequestJobFactory*>(context->job_factory());
167 // Note: if this is called twice with 2 request contexts that share one job
168 // factory (as is the case with a media request context and its related
169 // normal request context) then this will early exit.
170 if (job_factory->IsHandledProtocol(chrome::kBlobScheme))
171 return; // Already initialized this JobFactory.
173 bool set_protocol = job_factory->SetProtocolHandler(
174 chrome::kBlobScheme,
175 new BlobProtocolHandler(
176 blob_storage_context->controller(),
177 file_system_context,
178 BrowserThread::GetMessageLoopProxyForThread(BrowserThread::FILE)));
179 DCHECK(set_protocol);
180 set_protocol = job_factory->SetProtocolHandler(
181 chrome::kFileSystemScheme,
182 CreateFileSystemProtocolHandler(file_system_context));
183 DCHECK(set_protocol);
185 job_factory->AddInterceptor(
186 new DeveloperProtocolHandler(appcache_service,
187 blob_storage_context->controller()));
189 // TODO(jam): Add the ProtocolHandlerRegistryIntercepter here!
192 // These constants are used to create the directory structure under the profile
193 // where renderers with a non-default storage partition keep their persistent
194 // state. This will contain a set of directories that partially mirror the
195 // directory structure of BrowserContext::GetPath().
197 // The kStoragePartitionDirname contains an extensions directory which is
198 // further partitioned by extension id, followed by another level of directories
199 // for the "default" extension storage partition and one directory for each
200 // persistent partition used by a webview tag. Example:
202 // Storage/ext/ABCDEF/def
203 // Storage/ext/ABCDEF/hash(partition name)
205 // The code in GetStoragePartitionPath() constructs these path names.
207 // TODO(nasko): Move extension related path code out of content.
208 const FilePath::CharType kStoragePartitionDirname[] =
209 FILE_PATH_LITERAL("Storage");
210 const FilePath::CharType kExtensionsDirname[] =
211 FILE_PATH_LITERAL("ext");
212 const FilePath::CharType kDefaultPartitionDirname[] =
213 FILE_PATH_LITERAL("def");
214 const FilePath::CharType kTrashDirname[] =
215 FILE_PATH_LITERAL("trash");
217 // Because partition names are user specified, they can be arbitrarily long
218 // which makes them unsuitable for paths names. We use a truncation of a
219 // SHA256 hash to perform a deterministic shortening of the string. The
220 // kPartitionNameHashBytes constant controls the length of the truncation.
221 // We use 6 bytes, which gives us 99.999% reliability against collisions over
222 // 1 million partition domains.
224 // Analysis:
225 // We assume that all partition names within one partition domain are
226 // controlled by the the same entity. Thus there is no chance for adverserial
227 // attack and all we care about is accidental collision. To get 5 9s over
228 // 1 million domains, we need the probability of a collision in any one domain
229 // to be
231 // p < nroot(1000000, .99999) ~= 10^-11
233 // We use the following birthday attack approximation to calculate the max
234 // number of unique names for this probability:
236 // n(p,H) = sqrt(2*H * ln(1/(1-p)))
238 // For a 6-byte hash, H = 2^(6*8). n(10^-11, H) ~= 75
240 // An average partition domain is likely to have less than 10 unique
241 // partition names which is far lower than 75.
243 // Note, that for 4 9s of reliability, the limit is 237 partition names per
244 // partition domain.
245 const int kPartitionNameHashBytes = 6;
247 // Needed for selecting all files in ObliterateOneDirectory() below.
248 #if defined(OS_POSIX)
249 const int kAllFileTypes = file_util::FileEnumerator::FILES |
250 file_util::FileEnumerator::DIRECTORIES |
251 file_util::FileEnumerator::SHOW_SYM_LINKS;
252 #else
253 const int kAllFileTypes = file_util::FileEnumerator::FILES |
254 file_util::FileEnumerator::DIRECTORIES;
255 #endif
257 FilePath GetStoragePartitionDomainPath(
258 const std::string& partition_domain) {
259 CHECK(IsStringUTF8(partition_domain));
261 return FilePath(kStoragePartitionDirname).Append(kExtensionsDirname)
262 .Append(FilePath::FromUTF8Unsafe(partition_domain));
265 // Helper function for doing a depth-first deletion of the data on disk.
266 // Examines paths directly in |current_dir| (no recursion) and tries to
267 // delete from disk anything that is in, or isn't a parent of something in
268 // |paths_to_keep|. Paths that need further expansion are added to
269 // |paths_to_consider|.
270 void ObliterateOneDirectory(const FilePath& current_dir,
271 const std::vector<FilePath>& paths_to_keep,
272 std::vector<FilePath>* paths_to_consider) {
273 CHECK(current_dir.IsAbsolute());
275 file_util::FileEnumerator enumerator(current_dir, false, kAllFileTypes);
276 for (FilePath to_delete = enumerator.Next(); !to_delete.empty();
277 to_delete = enumerator.Next()) {
278 // Enum tracking which of the 3 possible actions to take for |to_delete|.
279 enum { kSkip, kEnqueue, kDelete } action = kDelete;
281 for (std::vector<FilePath>::const_iterator to_keep = paths_to_keep.begin();
282 to_keep != paths_to_keep.end();
283 ++to_keep) {
284 if (to_delete == *to_keep) {
285 action = kSkip;
286 break;
287 } else if (to_delete.IsParent(*to_keep)) {
288 // |to_delete| contains a path to keep. Add to stack for further
289 // processing.
290 action = kEnqueue;
291 break;
295 switch (action) {
296 case kDelete:
297 file_util::Delete(to_delete, true);
298 break;
300 case kEnqueue:
301 paths_to_consider->push_back(to_delete);
302 break;
304 case kSkip:
305 break;
310 // Synchronously attempts to delete |unnormalized_root|, preserving only
311 // entries in |paths_to_keep|. If there are no entries in |paths_to_keep| on
312 // disk, then it completely removes |unnormalized_root|. All paths must be
313 // absolute paths.
314 void BlockingObliteratePath(
315 const FilePath& unnormalized_browser_context_root,
316 const FilePath& unnormalized_root,
317 const std::vector<FilePath>& paths_to_keep,
318 const scoped_refptr<base::TaskRunner>& closure_runner,
319 const base::Closure& on_gc_required) {
320 // Early exit required because file_util::AbsolutePath() will fail on POSIX
321 // if |unnormalized_root| does not exist. This is safe because there is
322 // nothing to do in this situation anwyays.
323 if (!file_util::PathExists(unnormalized_root)) {
324 return;
327 // Never try to obliterate things outside of the browser context root or the
328 // browser context root itself. Die hard.
329 FilePath root = unnormalized_root;
330 FilePath browser_context_root = unnormalized_browser_context_root;
331 CHECK(file_util::AbsolutePath(&root));
332 CHECK(file_util::AbsolutePath(&browser_context_root));
333 CHECK(file_util::ContainsPath(browser_context_root, root) &&
334 browser_context_root != root);
336 // Reduce |paths_to_keep| set to those under the root and actually on disk.
337 std::vector<FilePath> valid_paths_to_keep;
338 for (std::vector<FilePath>::const_iterator it = paths_to_keep.begin();
339 it != paths_to_keep.end();
340 ++it) {
341 if (root.IsParent(*it) && file_util::PathExists(*it))
342 valid_paths_to_keep.push_back(*it);
345 // If none of the |paths_to_keep| are valid anymore then we just whack the
346 // root and be done with it. Otherwise, signal garbage collection and do
347 // a best-effort delete of the on-disk structures.
348 if (valid_paths_to_keep.empty()) {
349 file_util::Delete(root, true);
350 return;
352 closure_runner->PostTask(FROM_HERE, on_gc_required);
354 // Otherwise, start at the root and delete everything that is not in
355 // |valid_paths_to_keep|.
356 std::vector<FilePath> paths_to_consider;
357 paths_to_consider.push_back(root);
358 while(!paths_to_consider.empty()) {
359 FilePath path = paths_to_consider.back();
360 paths_to_consider.pop_back();
361 ObliterateOneDirectory(path, valid_paths_to_keep, &paths_to_consider);
365 // Deletes all entries inside the |storage_root| that are not in the
366 // |active_paths|. Deletion is done in 2 steps:
368 // (1) Moving all garbage collected paths into a trash directory.
369 // (2) Asynchronously deleting the trash directory.
371 // The deletion is asynchronous because after (1) completes, calling code can
372 // safely continue to use the paths that had just been garbage collected
373 // without fear of race conditions.
375 // This code also ignores failed moves rather than attempting a smarter retry.
376 // Moves shouldn't fail here unless there is some out-of-band error (eg.,
377 // FS corruption). Retry logic is dangerous in the general case because
378 // there is not necessarily a guaranteed case where the logic may succeed.
380 // This function is still named BlockingGarbageCollect() because it does
381 // execute a few filesystem operations synchronously.
382 void BlockingGarbageCollect(
383 const FilePath& storage_root,
384 const scoped_refptr<base::TaskRunner>& file_access_runner,
385 scoped_ptr<base::hash_set<FilePath> > active_paths) {
386 CHECK(storage_root.IsAbsolute());
388 file_util::FileEnumerator enumerator(storage_root, false, kAllFileTypes);
389 FilePath trash_directory;
390 if (!file_util::CreateTemporaryDirInDir(storage_root, kTrashDirname,
391 &trash_directory)) {
392 // Unable to continue without creating the trash directory so give up.
393 return;
395 for (FilePath path = enumerator.Next(); !path.empty();
396 path = enumerator.Next()) {
397 if (active_paths->find(path) == active_paths->end() &&
398 path != trash_directory) {
399 // Since |trash_directory| is unique for each run of this function there
400 // can be no colllisions on the move.
401 file_util::Move(path, trash_directory.Append(path.BaseName()));
405 file_access_runner->PostTask(
406 FROM_HERE,
407 base::Bind(base::IgnoreResult(&file_util::Delete), trash_directory,
408 true));
411 } // namespace
413 // static
414 FilePath StoragePartitionImplMap::GetStoragePartitionPath(
415 const std::string& partition_domain,
416 const std::string& partition_name) {
417 if (partition_domain.empty())
418 return FilePath();
420 FilePath path = GetStoragePartitionDomainPath(partition_domain);
422 // TODO(ajwong): Mangle in-memory into this somehow, either by putting
423 // it into the partition_name, or by manually adding another path component
424 // here. Otherwise, it's possible to have an in-memory StoragePartition and
425 // a persistent one that return the same FilePath for GetPath().
426 if (!partition_name.empty()) {
427 // For analysis of why we can ignore collisions, see the comment above
428 // kPartitionNameHashBytes.
429 char buffer[kPartitionNameHashBytes];
430 crypto::SHA256HashString(partition_name, &buffer[0],
431 sizeof(buffer));
432 return path.AppendASCII(base::HexEncode(buffer, sizeof(buffer)));
435 return path.Append(kDefaultPartitionDirname);
438 StoragePartitionImplMap::StoragePartitionImplMap(
439 BrowserContext* browser_context)
440 : browser_context_(browser_context),
441 resource_context_initialized_(false) {
442 // Doing here instead of initializer list cause it's just too ugly to read.
443 base::SequencedWorkerPool* blocking_pool = BrowserThread::GetBlockingPool();
444 file_access_runner_ =
445 blocking_pool->GetSequencedTaskRunner(blocking_pool->GetSequenceToken());
448 StoragePartitionImplMap::~StoragePartitionImplMap() {
449 STLDeleteContainerPairSecondPointers(partitions_.begin(),
450 partitions_.end());
453 StoragePartitionImpl* StoragePartitionImplMap::Get(
454 const std::string& partition_domain,
455 const std::string& partition_name,
456 bool in_memory) {
457 // TODO(ajwong): ResourceContexts no longer have any storage related state.
458 // We should move this into a place where it is called once per
459 // BrowserContext creation rather than piggybacking off the default context
460 // creation.
461 if (!resource_context_initialized_) {
462 resource_context_initialized_ = true;
463 InitializeResourceContext(browser_context_);
466 // Find the previously created partition if it's available.
467 StoragePartitionConfig partition_config(
468 partition_domain, partition_name, in_memory);
470 PartitionMap::const_iterator it = partitions_.find(partition_config);
471 if (it != partitions_.end())
472 return it->second;
474 FilePath partition_path =
475 browser_context_->GetPath().Append(
476 GetStoragePartitionPath(partition_domain, partition_name));
477 StoragePartitionImpl* partition =
478 StoragePartitionImpl::Create(browser_context_, in_memory,
479 partition_path);
480 partitions_[partition_config] = partition;
482 // These calls must happen after StoragePartitionImpl::Create().
483 partition->SetURLRequestContext(
484 partition_domain.empty() ?
485 browser_context_->GetRequestContext() :
486 browser_context_->GetRequestContextForStoragePartition(
487 partition->GetPath(), in_memory));
488 partition->SetMediaURLRequestContext(
489 partition_domain.empty() ?
490 browser_context_->GetMediaRequestContext() :
491 browser_context_->GetMediaRequestContextForStoragePartition(
492 partition->GetPath(), in_memory));
494 PostCreateInitialization(partition, in_memory);
496 return partition;
499 void StoragePartitionImplMap::AsyncObliterate(
500 const GURL& site,
501 const base::Closure& on_gc_required) {
502 // This method should avoid creating any StoragePartition (which would
503 // create more open file handles) so that it can delete as much of the
504 // data off disk as possible.
505 std::string partition_domain;
506 std::string partition_name;
507 bool in_memory = false;
508 GetContentClient()->browser()->GetStoragePartitionConfigForSite(
509 browser_context_, site, false, &partition_domain,
510 &partition_name, &in_memory);
512 // Find the active partitions for the domain. Because these partitions are
513 // active, it is not possible to just delete the directories that contain
514 // the backing data structures without causing the browser to crash. Instead,
515 // of deleteing the directory, we tell each storage context later to
516 // remove any data they have saved. This will leave the directory structure
517 // intact but it will only contain empty databases.
518 std::vector<StoragePartitionImpl*> active_partitions;
519 std::vector<FilePath> paths_to_keep;
520 for (PartitionMap::const_iterator it = partitions_.begin();
521 it != partitions_.end();
522 ++it) {
523 const StoragePartitionConfig& config = it->first;
524 if (config.partition_domain == partition_domain) {
525 it->second->AsyncClearAllData();
526 if (!config.in_memory) {
527 paths_to_keep.push_back(it->second->GetPath());
532 // Start a best-effort delete of the on-disk storage excluding paths that are
533 // known to still be in use. This is to delete any previously created
534 // StoragePartition state that just happens to not have been used during this
535 // run of the browser.
536 FilePath domain_root = browser_context_->GetPath().Append(
537 GetStoragePartitionDomainPath(partition_domain));
539 BrowserThread::PostBlockingPoolTask(
540 FROM_HERE,
541 base::Bind(&BlockingObliteratePath, browser_context_->GetPath(),
542 domain_root, paths_to_keep,
543 base::MessageLoopProxy::current(), on_gc_required));
546 void StoragePartitionImplMap::GarbageCollect(
547 scoped_ptr<base::hash_set<FilePath> > active_paths,
548 const base::Closure& done) {
549 // Include all paths for current StoragePartitions in the active_paths since
550 // they cannot be deleted safely.
551 for (PartitionMap::const_iterator it = partitions_.begin();
552 it != partitions_.end();
553 ++it) {
554 const StoragePartitionConfig& config = it->first;
555 if (!config.in_memory)
556 active_paths->insert(it->second->GetPath());
559 // Find the directory holding the StoragePartitions and delete everything in
560 // there that isn't considered active.
561 FilePath storage_root = browser_context_->GetPath().Append(
562 GetStoragePartitionDomainPath(std::string()));
563 file_access_runner_->PostTaskAndReply(
564 FROM_HERE,
565 base::Bind(&BlockingGarbageCollect, storage_root,
566 file_access_runner_,
567 base::Passed(&active_paths)),
568 done);
571 void StoragePartitionImplMap::ForEach(
572 const BrowserContext::StoragePartitionCallback& callback) {
573 for (PartitionMap::const_iterator it = partitions_.begin();
574 it != partitions_.end();
575 ++it) {
576 callback.Run(it->second);
580 void StoragePartitionImplMap::PostCreateInitialization(
581 StoragePartitionImpl* partition,
582 bool in_memory) {
583 // Check first to avoid memory leak in unittests.
584 if (BrowserThread::IsMessageLoopValid(BrowserThread::IO)) {
585 BrowserThread::PostTask(
586 BrowserThread::IO, FROM_HERE,
587 base::Bind(&ChromeAppCacheService::InitializeOnIOThread,
588 partition->GetAppCacheService(),
589 in_memory ? FilePath() :
590 partition->GetPath().Append(kAppCacheDirname),
591 browser_context_->GetResourceContext(),
592 make_scoped_refptr(partition->GetURLRequestContext()),
593 make_scoped_refptr(
594 browser_context_->GetSpecialStoragePolicy())));
596 // Add content's URLRequestContext's hooks.
597 BrowserThread::PostTask(
598 BrowserThread::IO, FROM_HERE,
599 base::Bind(
600 &InitializeURLRequestContext,
601 make_scoped_refptr(partition->GetURLRequestContext()),
602 make_scoped_refptr(partition->GetAppCacheService()),
603 make_scoped_refptr(partition->GetFileSystemContext()),
604 make_scoped_refptr(
605 ChromeBlobStorageContext::GetFor(browser_context_))));
607 // We do not call InitializeURLRequestContext() for media contexts because,
608 // other than the HTTP cache, the media contexts share the same backing
609 // objects as their associated "normal" request context. Thus, the previous
610 // call serves to initialize the media request context for this storage
611 // partition as well.
615 } // namespace content