1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "content/browser/indexed_db/indexed_db_context_impl.h"
10 #include "base/bind.h"
11 #include "base/command_line.h"
12 #include "base/files/file_enumerator.h"
13 #include "base/files/file_util.h"
14 #include "base/logging.h"
15 #include "base/metrics/histogram.h"
16 #include "base/sequenced_task_runner.h"
17 #include "base/strings/string_util.h"
18 #include "base/strings/utf_string_conversions.h"
19 #include "base/threading/thread_restrictions.h"
20 #include "base/time/time.h"
21 #include "base/values.h"
22 #include "content/browser/browser_main_loop.h"
23 #include "content/browser/indexed_db/indexed_db_connection.h"
24 #include "content/browser/indexed_db/indexed_db_database.h"
25 #include "content/browser/indexed_db/indexed_db_dispatcher_host.h"
26 #include "content/browser/indexed_db/indexed_db_factory_impl.h"
27 #include "content/browser/indexed_db/indexed_db_quota_client.h"
28 #include "content/browser/indexed_db/indexed_db_tracing.h"
29 #include "content/browser/indexed_db/indexed_db_transaction.h"
30 #include "content/public/browser/browser_thread.h"
31 #include "content/public/browser/indexed_db_info.h"
32 #include "content/public/common/content_switches.h"
33 #include "storage/browser/database/database_util.h"
34 #include "storage/browser/quota/quota_manager_proxy.h"
35 #include "storage/browser/quota/special_storage_policy.h"
36 #include "storage/common/database/database_identifier.h"
37 #include "ui/base/text/bytes_formatting.h"
39 using base::DictionaryValue
;
40 using base::ListValue
;
41 using storage::DatabaseUtil
;
44 const base::FilePath::CharType
IndexedDBContextImpl::kIndexedDBDirectory
[] =
45 FILE_PATH_LITERAL("IndexedDB");
47 static const base::FilePath::CharType kBlobExtension
[] =
48 FILE_PATH_LITERAL(".blob");
50 static const base::FilePath::CharType kIndexedDBExtension
[] =
51 FILE_PATH_LITERAL(".indexeddb");
53 static const base::FilePath::CharType kLevelDBExtension
[] =
54 FILE_PATH_LITERAL(".leveldb");
58 // This may be called after the IndexedDBContext is destroyed.
59 void GetAllOriginsAndPaths(const base::FilePath
& indexeddb_path
,
60 std::vector
<GURL
>* origins
,
61 std::vector
<base::FilePath
>* file_paths
) {
62 // TODO(jsbell): DCHECK that this is running on an IndexedDB thread,
63 // if a global handle to it is ever available.
64 if (indexeddb_path
.empty())
66 base::FileEnumerator
file_enumerator(
67 indexeddb_path
, false, base::FileEnumerator::DIRECTORIES
);
68 for (base::FilePath file_path
= file_enumerator
.Next(); !file_path
.empty();
69 file_path
= file_enumerator
.Next()) {
70 if (file_path
.Extension() == kLevelDBExtension
&&
71 file_path
.RemoveExtension().Extension() == kIndexedDBExtension
) {
72 std::string origin_id
= file_path
.BaseName().RemoveExtension()
73 .RemoveExtension().MaybeAsASCII();
74 origins
->push_back(storage::GetOriginFromIdentifier(origin_id
));
76 file_paths
->push_back(file_path
);
81 // This will be called after the IndexedDBContext is destroyed.
82 void ClearSessionOnlyOrigins(
83 const base::FilePath
& indexeddb_path
,
84 scoped_refptr
<storage::SpecialStoragePolicy
> special_storage_policy
) {
85 // TODO(jsbell): DCHECK that this is running on an IndexedDB thread,
86 // if a global handle to it is ever available.
87 std::vector
<GURL
> origins
;
88 std::vector
<base::FilePath
> file_paths
;
89 GetAllOriginsAndPaths(indexeddb_path
, &origins
, &file_paths
);
90 DCHECK_EQ(origins
.size(), file_paths
.size());
91 std::vector
<base::FilePath
>::const_iterator file_path_iter
=
93 for (std::vector
<GURL
>::const_iterator iter
= origins
.begin();
94 iter
!= origins
.end();
95 ++iter
, ++file_path_iter
) {
96 if (!special_storage_policy
->IsStorageSessionOnly(*iter
))
98 if (special_storage_policy
->IsStorageProtected(*iter
))
100 base::DeleteFile(*file_path_iter
, true);
106 IndexedDBContextImpl::IndexedDBContextImpl(
107 const base::FilePath
& data_path
,
108 storage::SpecialStoragePolicy
* special_storage_policy
,
109 storage::QuotaManagerProxy
* quota_manager_proxy
,
110 base::SequencedTaskRunner
* task_runner
)
111 : force_keep_session_state_(false),
112 special_storage_policy_(special_storage_policy
),
113 quota_manager_proxy_(quota_manager_proxy
),
114 task_runner_(task_runner
) {
116 if (!data_path
.empty())
117 data_path_
= data_path
.Append(kIndexedDBDirectory
);
118 if (quota_manager_proxy
) {
119 quota_manager_proxy
->RegisterClient(new IndexedDBQuotaClient(this));
123 IndexedDBFactory
* IndexedDBContextImpl::GetIDBFactory() {
124 DCHECK(TaskRunner()->RunsTasksOnCurrentThread());
125 if (!factory_
.get()) {
126 // Prime our cache of origins with existing databases so we can
127 // detect when dbs are newly created.
129 factory_
= new IndexedDBFactoryImpl(this);
131 return factory_
.get();
134 std::vector
<GURL
> IndexedDBContextImpl::GetAllOrigins() {
135 DCHECK(TaskRunner()->RunsTasksOnCurrentThread());
136 std::set
<GURL
>* origins_set
= GetOriginSet();
137 return std::vector
<GURL
>(origins_set
->begin(), origins_set
->end());
140 std::vector
<IndexedDBInfo
> IndexedDBContextImpl::GetAllOriginsInfo() {
141 DCHECK(TaskRunner()->RunsTasksOnCurrentThread());
142 std::vector
<GURL
> origins
= GetAllOrigins();
143 std::vector
<IndexedDBInfo
> result
;
144 for (const auto& origin_url
: origins
) {
145 size_t connection_count
= GetConnectionCount(origin_url
);
146 result
.push_back(IndexedDBInfo(origin_url
,
147 GetOriginDiskUsage(origin_url
),
148 GetOriginLastModified(origin_url
),
154 static bool HostNameComparator(const GURL
& i
, const GURL
& j
) {
155 return i
.host() < j
.host();
158 base::ListValue
* IndexedDBContextImpl::GetAllOriginsDetails() {
159 DCHECK(TaskRunner()->RunsTasksOnCurrentThread());
160 std::vector
<GURL
> origins
= GetAllOrigins();
162 std::sort(origins
.begin(), origins
.end(), HostNameComparator
);
164 scoped_ptr
<base::ListValue
> list(new base::ListValue());
165 for (const auto& origin_url
: origins
) {
166 scoped_ptr
<base::DictionaryValue
> info(new base::DictionaryValue());
167 info
->SetString("url", origin_url
.spec());
168 info
->SetString("size", ui::FormatBytes(GetOriginDiskUsage(origin_url
)));
169 info
->SetDouble("last_modified",
170 GetOriginLastModified(origin_url
).ToJsTime());
171 if (!is_incognito()) {
172 scoped_ptr
<base::ListValue
> paths(new base::ListValue());
173 for (const base::FilePath
& path
: GetStoragePaths(origin_url
))
174 paths
->AppendString(path
.value());
175 info
->Set("paths", paths
.release());
177 info
->SetDouble("connection_count", GetConnectionCount(origin_url
));
179 // This ends up being O(n^2) since we iterate over all open databases
180 // to extract just those in the origin, and we're iterating over all
181 // origins in the outer loop.
183 if (factory_
.get()) {
184 std::pair
<IndexedDBFactory::OriginDBMapIterator
,
185 IndexedDBFactory::OriginDBMapIterator
> range
=
186 factory_
->GetOpenDatabasesForOrigin(origin_url
);
187 // TODO(jsbell): Sort by name?
188 scoped_ptr
<base::ListValue
> database_list(new base::ListValue());
190 for (IndexedDBFactory::OriginDBMapIterator it
= range
.first
;
193 const IndexedDBDatabase
* db
= it
->second
;
194 scoped_ptr
<base::DictionaryValue
> db_info(new base::DictionaryValue());
196 db_info
->SetString("name", db
->name());
197 db_info
->SetDouble("pending_opens", db
->PendingOpenCount());
198 db_info
->SetDouble("pending_upgrades", db
->PendingUpgradeCount());
199 db_info
->SetDouble("running_upgrades", db
->RunningUpgradeCount());
200 db_info
->SetDouble("pending_deletes", db
->PendingDeleteCount());
201 db_info
->SetDouble("connection_count",
202 db
->ConnectionCount() - db
->PendingUpgradeCount() -
203 db
->RunningUpgradeCount());
205 scoped_ptr
<base::ListValue
> transaction_list(new base::ListValue());
206 std::vector
<const IndexedDBTransaction
*> transactions
=
207 db
->transaction_coordinator().GetTransactions();
208 for (const auto* transaction
: transactions
) {
209 scoped_ptr
<base::DictionaryValue
> transaction_info(
210 new base::DictionaryValue());
212 const char* kModes
[] = { "readonly", "readwrite", "versionchange" };
213 transaction_info
->SetString("mode", kModes
[transaction
->mode()]);
214 switch (transaction
->state()) {
215 case IndexedDBTransaction::CREATED
:
216 transaction_info
->SetString("status", "blocked");
218 case IndexedDBTransaction::STARTED
:
219 if (transaction
->diagnostics().tasks_scheduled
> 0)
220 transaction_info
->SetString("status", "running");
222 transaction_info
->SetString("status", "started");
224 case IndexedDBTransaction::COMMITTING
:
225 transaction_info
->SetString("status", "committing");
227 case IndexedDBTransaction::FINISHED
:
228 transaction_info
->SetString("status", "finished");
232 transaction_info
->SetDouble(
234 IndexedDBDispatcherHost::TransactionIdToProcessId(
236 transaction_info
->SetDouble(
238 IndexedDBDispatcherHost::TransactionIdToRendererTransactionId(
240 transaction_info
->SetDouble(
242 (base::Time::Now() - transaction
->diagnostics().creation_time
)
244 transaction_info
->SetDouble(
246 (base::Time::Now() - transaction
->diagnostics().start_time
)
248 transaction_info
->SetDouble(
249 "tasks_scheduled", transaction
->diagnostics().tasks_scheduled
);
250 transaction_info
->SetDouble(
251 "tasks_completed", transaction
->diagnostics().tasks_completed
);
253 scoped_ptr
<base::ListValue
> scope(new base::ListValue());
254 for (const auto& id
: transaction
->scope()) {
255 IndexedDBDatabaseMetadata::ObjectStoreMap::const_iterator it
=
256 db
->metadata().object_stores
.find(id
);
257 if (it
!= db
->metadata().object_stores
.end())
258 scope
->AppendString(it
->second
.name
);
261 transaction_info
->Set("scope", scope
.release());
262 transaction_list
->Append(transaction_info
.release());
264 db_info
->Set("transactions", transaction_list
.release());
266 database_list
->Append(db_info
.release());
268 info
->Set("databases", database_list
.release());
271 list
->Append(info
.release());
273 return list
.release();
276 int IndexedDBContextImpl::GetOriginBlobFileCount(const GURL
& origin_url
) {
277 DCHECK(TaskRunner()->RunsTasksOnCurrentThread());
279 base::FileEnumerator
file_enumerator(
280 GetBlobPath(storage::GetIdentifierFromOrigin(origin_url
)), true,
281 base::FileEnumerator::FILES
);
282 for (base::FilePath file_path
= file_enumerator
.Next(); !file_path
.empty();
283 file_path
= file_enumerator
.Next()) {
289 int64
IndexedDBContextImpl::GetOriginDiskUsage(const GURL
& origin_url
) {
290 DCHECK(TaskRunner()->RunsTasksOnCurrentThread());
291 if (data_path_
.empty() || !IsInOriginSet(origin_url
))
293 EnsureDiskUsageCacheInitialized(origin_url
);
294 return origin_size_map_
[origin_url
];
297 base::Time
IndexedDBContextImpl::GetOriginLastModified(const GURL
& origin_url
) {
298 DCHECK(TaskRunner()->RunsTasksOnCurrentThread());
299 if (data_path_
.empty() || !IsInOriginSet(origin_url
))
301 base::FilePath idb_directory
= GetLevelDBPath(origin_url
);
302 base::File::Info file_info
;
303 if (!base::GetFileInfo(idb_directory
, &file_info
))
305 return file_info
.last_modified
;
308 void IndexedDBContextImpl::DeleteForOrigin(const GURL
& origin_url
) {
309 DCHECK(TaskRunner()->RunsTasksOnCurrentThread());
310 ForceClose(origin_url
, FORCE_CLOSE_DELETE_ORIGIN
);
311 if (data_path_
.empty() || !IsInOriginSet(origin_url
))
314 base::FilePath idb_directory
= GetLevelDBPath(origin_url
);
315 EnsureDiskUsageCacheInitialized(origin_url
);
316 leveldb::Status s
= LevelDBDatabase::Destroy(idb_directory
);
318 LOG(WARNING
) << "Failed to delete LevelDB database: "
319 << idb_directory
.AsUTF8Unsafe();
321 // LevelDB does not delete empty directories; work around this.
322 // TODO(jsbell): Remove when upstream bug is fixed.
323 // https://code.google.com/p/leveldb/issues/detail?id=209
324 const bool kNonRecursive
= false;
325 base::DeleteFile(idb_directory
, kNonRecursive
);
327 base::DeleteFile(GetBlobPath(storage::GetIdentifierFromOrigin(origin_url
)),
328 true /* recursive */);
329 QueryDiskAndUpdateQuotaUsage(origin_url
);
331 RemoveFromOriginSet(origin_url
);
332 origin_size_map_
.erase(origin_url
);
333 space_available_map_
.erase(origin_url
);
337 void IndexedDBContextImpl::CopyOriginData(const GURL
& origin_url
,
338 IndexedDBContext
* dest_context
) {
339 DCHECK(TaskRunner()->RunsTasksOnCurrentThread());
341 if (data_path_
.empty() || !IsInOriginSet(origin_url
))
344 IndexedDBContextImpl
* dest_context_impl
=
345 static_cast<IndexedDBContextImpl
*>(dest_context
);
347 ForceClose(origin_url
, FORCE_CLOSE_COPY_ORIGIN
);
348 std::string origin_id
= storage::GetIdentifierFromOrigin(origin_url
);
350 // Make sure we're not about to delete our own database.
351 CHECK_NE(dest_context_impl
->data_path().value(), data_path().value());
353 // Delete any existing storage paths in the destination context.
354 // A previously failed migration may have left behind partially copied
356 for (const base::FilePath
& dest_path
:
357 dest_context_impl
->GetStoragePaths(origin_url
))
358 base::DeleteFile(dest_path
, true);
360 base::FilePath dest_data_path
= dest_context_impl
->data_path();
361 base::CreateDirectory(dest_data_path
);
363 for (const base::FilePath
& src_data_path
: GetStoragePaths(origin_url
)) {
364 if (base::PathExists(src_data_path
)) {
365 base::CopyDirectory(src_data_path
, dest_data_path
, true);
370 void IndexedDBContextImpl::ForceClose(const GURL origin_url
,
371 ForceCloseReason reason
) {
372 DCHECK(TaskRunner()->RunsTasksOnCurrentThread());
373 UMA_HISTOGRAM_ENUMERATION("WebCore.IndexedDB.Context.ForceCloseReason",
375 FORCE_CLOSE_REASON_MAX
);
377 if (data_path_
.empty() || !IsInOriginSet(origin_url
))
381 factory_
->ForceClose(origin_url
);
382 DCHECK_EQ(0UL, GetConnectionCount(origin_url
));
385 size_t IndexedDBContextImpl::GetConnectionCount(const GURL
& origin_url
) {
386 DCHECK(TaskRunner()->RunsTasksOnCurrentThread());
387 if (data_path_
.empty() || !IsInOriginSet(origin_url
))
393 return factory_
->GetConnectionCount(origin_url
);
396 base::FilePath
IndexedDBContextImpl::GetLevelDBPath(
397 const GURL
& origin_url
) const {
398 std::string origin_id
= storage::GetIdentifierFromOrigin(origin_url
);
399 return GetLevelDBPath(origin_id
);
402 std::vector
<base::FilePath
> IndexedDBContextImpl::GetStoragePaths(
403 const GURL
& origin_url
) const {
404 std::string origin_id
= storage::GetIdentifierFromOrigin(origin_url
);
405 std::vector
<base::FilePath
> paths
;
406 paths
.push_back(GetLevelDBPath(origin_id
));
407 paths
.push_back(GetBlobPath(origin_id
));
411 base::FilePath
IndexedDBContextImpl::GetFilePathForTesting(
412 const std::string
& origin_id
) const {
413 return GetLevelDBPath(origin_id
);
416 void IndexedDBContextImpl::SetTaskRunnerForTesting(
417 base::SequencedTaskRunner
* task_runner
) {
418 DCHECK(!task_runner_
.get());
419 task_runner_
= task_runner
;
422 void IndexedDBContextImpl::ConnectionOpened(const GURL
& origin_url
,
423 IndexedDBConnection
* connection
) {
424 DCHECK(TaskRunner()->RunsTasksOnCurrentThread());
425 if (quota_manager_proxy()) {
426 quota_manager_proxy()->NotifyStorageAccessed(
427 storage::QuotaClient::kIndexedDatabase
,
429 storage::kStorageTypeTemporary
);
431 if (AddToOriginSet(origin_url
)) {
432 // A newly created db, notify the quota system.
433 QueryDiskAndUpdateQuotaUsage(origin_url
);
435 EnsureDiskUsageCacheInitialized(origin_url
);
437 QueryAvailableQuota(origin_url
);
440 void IndexedDBContextImpl::ConnectionClosed(const GURL
& origin_url
,
441 IndexedDBConnection
* connection
) {
442 DCHECK(TaskRunner()->RunsTasksOnCurrentThread());
443 if (quota_manager_proxy()) {
444 quota_manager_proxy()->NotifyStorageAccessed(
445 storage::QuotaClient::kIndexedDatabase
,
447 storage::kStorageTypeTemporary
);
449 if (factory_
.get() && factory_
->GetConnectionCount(origin_url
) == 0)
450 QueryDiskAndUpdateQuotaUsage(origin_url
);
453 void IndexedDBContextImpl::TransactionComplete(const GURL
& origin_url
) {
454 DCHECK(!factory_
.get() || factory_
->GetConnectionCount(origin_url
) > 0);
455 QueryDiskAndUpdateQuotaUsage(origin_url
);
456 QueryAvailableQuota(origin_url
);
459 void IndexedDBContextImpl::DatabaseDeleted(const GURL
& origin_url
) {
460 AddToOriginSet(origin_url
);
461 QueryDiskAndUpdateQuotaUsage(origin_url
);
462 QueryAvailableQuota(origin_url
);
465 bool IndexedDBContextImpl::WouldBeOverQuota(const GURL
& origin_url
,
466 int64 additional_bytes
) {
467 if (space_available_map_
.find(origin_url
) == space_available_map_
.end()) {
468 // We haven't heard back from the QuotaManager yet, just let it through.
471 bool over_quota
= additional_bytes
> space_available_map_
[origin_url
];
475 bool IndexedDBContextImpl::IsOverQuota(const GURL
& origin_url
) {
476 const int kOneAdditionalByte
= 1;
477 return WouldBeOverQuota(origin_url
, kOneAdditionalByte
);
480 storage::QuotaManagerProxy
* IndexedDBContextImpl::quota_manager_proxy() {
481 return quota_manager_proxy_
.get();
484 IndexedDBContextImpl::~IndexedDBContextImpl() {
485 if (factory_
.get()) {
486 TaskRunner()->PostTask(
487 FROM_HERE
, base::Bind(&IndexedDBFactory::ContextDestroyed
, factory_
));
491 if (data_path_
.empty())
494 if (force_keep_session_state_
)
497 bool has_session_only_databases
=
498 special_storage_policy_
.get() &&
499 special_storage_policy_
->HasSessionOnlyOrigins();
501 // Clearing only session-only databases, and there are none.
502 if (!has_session_only_databases
)
505 TaskRunner()->PostTask(
508 &ClearSessionOnlyOrigins
, data_path_
, special_storage_policy_
));
511 base::FilePath
IndexedDBContextImpl::GetBlobPath(
512 const std::string
& origin_id
) const {
513 DCHECK(!data_path_
.empty());
514 return data_path_
.AppendASCII(origin_id
).AddExtension(kIndexedDBExtension
)
515 .AddExtension(kBlobExtension
);
518 base::FilePath
IndexedDBContextImpl::GetLevelDBPath(
519 const std::string
& origin_id
) const {
520 DCHECK(!data_path_
.empty());
521 return data_path_
.AppendASCII(origin_id
).AddExtension(kIndexedDBExtension
)
522 .AddExtension(kLevelDBExtension
);
525 int64
IndexedDBContextImpl::ReadUsageFromDisk(const GURL
& origin_url
) const {
526 if (data_path_
.empty())
528 int64 total_size
= 0;
529 for (const base::FilePath
& path
: GetStoragePaths(origin_url
))
530 total_size
+= base::ComputeDirectorySize(path
);
534 void IndexedDBContextImpl::EnsureDiskUsageCacheInitialized(
535 const GURL
& origin_url
) {
536 if (origin_size_map_
.find(origin_url
) == origin_size_map_
.end())
537 origin_size_map_
[origin_url
] = ReadUsageFromDisk(origin_url
);
540 void IndexedDBContextImpl::QueryDiskAndUpdateQuotaUsage(
541 const GURL
& origin_url
) {
542 int64 former_disk_usage
= origin_size_map_
[origin_url
];
543 int64 current_disk_usage
= ReadUsageFromDisk(origin_url
);
544 int64 difference
= current_disk_usage
- former_disk_usage
;
546 origin_size_map_
[origin_url
] = current_disk_usage
;
547 // quota_manager_proxy() is NULL in unit tests.
548 if (quota_manager_proxy()) {
549 quota_manager_proxy()->NotifyStorageModified(
550 storage::QuotaClient::kIndexedDatabase
,
552 storage::kStorageTypeTemporary
,
558 void IndexedDBContextImpl::GotUsageAndQuota(const GURL
& origin_url
,
559 storage::QuotaStatusCode status
,
562 DCHECK_CURRENTLY_ON(BrowserThread::IO
);
563 DCHECK(status
== storage::kQuotaStatusOk
||
564 status
== storage::kQuotaErrorAbort
)
565 << "status was " << status
;
566 if (status
== storage::kQuotaErrorAbort
) {
567 // We seem to no longer care to wait around for the answer.
570 TaskRunner()->PostTask(FROM_HERE
,
571 base::Bind(&IndexedDBContextImpl::GotUpdatedQuota
,
578 void IndexedDBContextImpl::GotUpdatedQuota(const GURL
& origin_url
,
581 DCHECK(TaskRunner()->RunsTasksOnCurrentThread());
582 space_available_map_
[origin_url
] = quota
- usage
;
585 void IndexedDBContextImpl::QueryAvailableQuota(const GURL
& origin_url
) {
586 if (!BrowserThread::CurrentlyOn(BrowserThread::IO
)) {
587 DCHECK(TaskRunner()->RunsTasksOnCurrentThread());
588 if (quota_manager_proxy()) {
589 BrowserThread::PostTask(
593 &IndexedDBContextImpl::QueryAvailableQuota
, this, origin_url
));
597 DCHECK_CURRENTLY_ON(BrowserThread::IO
);
598 if (!quota_manager_proxy() || !quota_manager_proxy()->quota_manager())
600 quota_manager_proxy()->quota_manager()->GetUsageAndQuota(
602 storage::kStorageTypeTemporary
,
603 base::Bind(&IndexedDBContextImpl::GotUsageAndQuota
, this, origin_url
));
606 std::set
<GURL
>* IndexedDBContextImpl::GetOriginSet() {
608 std::vector
<GURL
> origins
;
609 GetAllOriginsAndPaths(data_path_
, &origins
, NULL
);
610 origin_set_
.reset(new std::set
<GURL
>(origins
.begin(), origins
.end()));
612 return origin_set_
.get();
615 void IndexedDBContextImpl::ResetCaches() {
617 origin_size_map_
.clear();
618 space_available_map_
.clear();
621 base::SequencedTaskRunner
* IndexedDBContextImpl::TaskRunner() const {
622 return task_runner_
.get();
625 } // namespace content