1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "webkit/dom_storage/session_storage_database.h"
7 #include "base/file_util.h"
8 #include "base/logging.h"
9 #include "base/metrics/histogram.h"
10 #include "base/stringprintf.h"
11 #include "base/strings/string_number_conversions.h"
12 #include "base/utf_string_conversions.h"
13 #include "googleurl/src/gurl.h"
14 #include "third_party/leveldatabase/src/include/leveldb/db.h"
15 #include "third_party/leveldatabase/src/include/leveldb/iterator.h"
16 #include "third_party/leveldatabase/src/include/leveldb/status.h"
17 #include "third_party/leveldatabase/src/include/leveldb/options.h"
18 #include "third_party/leveldatabase/src/include/leveldb/write_batch.h"
23 const char session_storage_uma_name
[] = "SessionStorageDatabase.Open";
25 enum SessionStorageUMA
{
26 SESSION_STORAGE_UMA_SUCCESS
,
27 SESSION_STORAGE_UMA_RECREATED
,
28 SESSION_STORAGE_UMA_FAIL
,
29 SESSION_STORAGE_UMA_MAX
34 // Layout of the database:
36 // -----------------------------------------------------------------------
37 // | map-1- | 2 (refcount, start of map-1-* keys)|
38 // | map-1-a | b (a = b in map 1) |
40 // | namespace- | dummy (start of namespace-* keys) |
41 // | namespace-1- (1 = namespace id)| dummy (start of namespace-1-* keys)|
42 // | namespace-1-origin1 | 1 (mapid) |
43 // | namespace-1-origin2 | 2 |
44 // | namespace-2- | dummy |
45 // | namespace-2-origin1 | 1 (shallow copy) |
46 // | namespace-2-origin2 | 2 (shallow copy) |
47 // | namespace-3- | dummy |
48 // | namespace-3-origin1 | 3 (deep copy) |
49 // | namespace-3-origin2 | 2 (shallow copy) |
50 // | next-map-id | 4 |
52 namespace dom_storage
{
54 SessionStorageDatabase::SessionStorageDatabase(const base::FilePath
& file_path
)
55 : file_path_(file_path
),
57 is_inconsistent_(false) {
60 SessionStorageDatabase::~SessionStorageDatabase() {
63 void SessionStorageDatabase::ReadAreaValues(const std::string
& namespace_id
,
66 // We don't create a database if it doesn't exist. In that case, there is
67 // nothing to be added to the result.
71 // While ReadAreaValues is in progress, another thread can call
72 // CommitAreaChanges. CommitAreaChanges might update map ref count key while
73 // this thread is iterating over the map ref count key. To protect the reading
74 // operation, create a snapshot and read from it.
75 leveldb::ReadOptions options
;
76 options
.snapshot
= db_
->GetSnapshot();
80 if (GetMapForArea(namespace_id
, origin
.spec(), options
, &exists
, &map_id
) &&
82 ReadMap(map_id
, options
, result
, false);
83 db_
->ReleaseSnapshot(options
.snapshot
);
86 bool SessionStorageDatabase::CommitAreaChanges(const std::string
& namespace_id
,
89 const ValuesMap
& changes
) {
90 // Even if |changes| is empty, we need to write the appropriate placeholders
91 // in the database, so that it can be later shallow-copied succssfully.
95 leveldb::WriteBatch batch
;
96 // Ensure that the keys "namespace-" "namespace-N" (see the schema above)
98 const bool kOkIfExists
= true;
99 if (!CreateNamespace(namespace_id
, kOkIfExists
, &batch
))
104 if (!GetMapForArea(namespace_id
, origin
.spec(), leveldb::ReadOptions(),
109 if (!GetMapRefCount(map_id
, &ref_count
))
112 if (!DeepCopyArea(namespace_id
, origin
, !clear_all_first
,
116 else if (clear_all_first
) {
117 if (!ClearMap(map_id
, &batch
))
121 // Map doesn't exist, create it now if needed.
122 if (!changes
.empty()) {
123 if (!CreateMapForArea(namespace_id
, origin
, &map_id
, &batch
))
128 WriteValuesToMap(map_id
, changes
, &batch
);
130 leveldb::Status s
= db_
->Write(leveldb::WriteOptions(), &batch
);
131 return DatabaseErrorCheck(s
.ok());
134 bool SessionStorageDatabase::CloneNamespace(
135 const std::string
& namespace_id
, const std::string
& new_namespace_id
) {
136 // Go through all origins in the namespace |namespace_id|, create placeholders
137 // for them in |new_namespace_id|, and associate them with the existing maps.
139 // Example, data before shallow copy:
140 // | map-1- | 1 (refcount) |
142 // | namespace-1- (1 = namespace id)| dummy |
143 // | namespace-1-origin1 | 1 (mapid) |
145 // Example, data after shallow copy:
146 // | map-1- | 2 (inc. refcount) |
148 // | namespace-1-(1 = namespace id) | dummy |
149 // | namespace-1-origin1 | 1 (mapid) |
150 // | namespace-2- | dummy |
151 // | namespace-2-origin1 | 1 (mapid) << references the same map
156 leveldb::WriteBatch batch
;
157 const bool kOkIfExists
= false;
158 if (!CreateNamespace(new_namespace_id
, kOkIfExists
, &batch
))
161 std::map
<std::string
, std::string
> areas
;
162 if (!GetAreasInNamespace(namespace_id
, &areas
))
165 for (std::map
<std::string
, std::string
>::const_iterator it
= areas
.begin();
166 it
!= areas
.end(); ++it
) {
167 const std::string
& origin
= it
->first
;
168 const std::string
& map_id
= it
->second
;
169 if (!IncreaseMapRefCount(map_id
, &batch
))
171 AddAreaToNamespace(new_namespace_id
, origin
, map_id
, &batch
);
173 leveldb::Status s
= db_
->Write(leveldb::WriteOptions(), &batch
);
174 return DatabaseErrorCheck(s
.ok());
177 bool SessionStorageDatabase::DeleteArea(const std::string
& namespace_id
,
178 const GURL
& origin
) {
179 if (!LazyOpen(false)) {
180 // No need to create the database if it doesn't exist.
183 leveldb::WriteBatch batch
;
184 if (!DeleteAreaHelper(namespace_id
, origin
.spec(), &batch
))
186 leveldb::Status s
= db_
->Write(leveldb::WriteOptions(), &batch
);
187 return DatabaseErrorCheck(s
.ok());
190 bool SessionStorageDatabase::DeleteNamespace(const std::string
& namespace_id
) {
191 if (!LazyOpen(false)) {
192 // No need to create the database if it doesn't exist.
195 // Itereate through the areas in the namespace.
196 leveldb::WriteBatch batch
;
197 std::map
<std::string
, std::string
> areas
;
198 if (!GetAreasInNamespace(namespace_id
, &areas
))
200 for (std::map
<std::string
, std::string
>::const_iterator it
= areas
.begin();
201 it
!= areas
.end(); ++it
) {
202 const std::string
& origin
= it
->first
;
203 if (!DeleteAreaHelper(namespace_id
, origin
, &batch
))
206 batch
.Delete(NamespaceStartKey(namespace_id
));
207 leveldb::Status s
= db_
->Write(leveldb::WriteOptions(), &batch
);
208 return DatabaseErrorCheck(s
.ok());
211 bool SessionStorageDatabase::ReadNamespacesAndOrigins(
212 std::map
<std::string
, std::vector
<GURL
> >* namespaces_and_origins
) {
216 // While ReadNamespacesAndOrigins is in progress, another thread can call
217 // CommitAreaChanges. To protect the reading operation, create a snapshot and
219 leveldb::ReadOptions options
;
220 options
.snapshot
= db_
->GetSnapshot();
222 std::string namespace_prefix
= NamespacePrefix();
223 scoped_ptr
<leveldb::Iterator
> it(db_
->NewIterator(options
));
224 it
->Seek(namespace_prefix
);
225 // If the key is not found, the status of the iterator won't be IsNotFound(),
226 // but the iterator will be invalid.
228 db_
->ReleaseSnapshot(options
.snapshot
);
232 if (!DatabaseErrorCheck(it
->status().ok())) {
233 db_
->ReleaseSnapshot(options
.snapshot
);
237 // Skip the dummy entry "namespace-" and iterate the namespaces.
238 std::string current_namespace_start_key
;
239 std::string current_namespace_id
;
240 for (it
->Next(); it
->Valid(); it
->Next()) {
241 std::string key
= it
->key().ToString();
242 if (key
.find(namespace_prefix
) != 0) {
243 // Iterated past the "namespace-" keys.
246 // For each namespace, the first key is "namespace-<namespaceid>-", and the
247 // subsequent keys are "namespace-<namespaceid>-<origin>". Read the unique
248 // "<namespaceid>" parts from the keys.
249 if (current_namespace_start_key
.empty() ||
250 key
.substr(0, current_namespace_start_key
.length()) !=
251 current_namespace_start_key
) {
252 // The key is of the form "namespace-<namespaceid>-" for a new
254 current_namespace_start_key
= key
;
255 current_namespace_id
=
256 key
.substr(namespace_prefix
.length(),
257 key
.length() - namespace_prefix
.length() - 1);
258 // Ensure that we keep track of the namespace even if it doesn't contain
260 namespaces_and_origins
->insert(
261 std::make_pair(current_namespace_id
, std::vector
<GURL
>()));
263 // The key is of the form "namespace-<namespaceid>-<origin>".
264 std::string origin
= key
.substr(current_namespace_start_key
.length());
265 (*namespaces_and_origins
)[current_namespace_id
].push_back(GURL(origin
));
268 db_
->ReleaseSnapshot(options
.snapshot
);
272 bool SessionStorageDatabase::LazyOpen(bool create_if_needed
) {
273 base::AutoLock
auto_lock(db_lock_
);
274 if (db_error_
|| is_inconsistent_
) {
275 // Don't try to open a database that we know has failed already.
281 if (!create_if_needed
&&
282 (!file_util::PathExists(file_path_
) ||
283 file_util::IsDirectoryEmpty(file_path_
))) {
284 // If the directory doesn't exist already and we haven't been asked to
285 // create a file on disk, then we don't bother opening the database. This
286 // means we wait until we absolutely need to put something onto disk before
292 leveldb::Status s
= TryToOpen(&db
);
294 LOG(WARNING
) << "Failed to open leveldb in " << file_path_
.value()
295 << ", error: " << s
.ToString();
298 // Clear the directory and try again.
299 file_util::Delete(file_path_
, true);
302 LOG(WARNING
) << "Failed to open leveldb in " << file_path_
.value()
303 << ", error: " << s
.ToString();
304 UMA_HISTOGRAM_ENUMERATION(session_storage_uma_name
,
305 SESSION_STORAGE_UMA_FAIL
,
306 SESSION_STORAGE_UMA_MAX
);
311 UMA_HISTOGRAM_ENUMERATION(session_storage_uma_name
,
312 SESSION_STORAGE_UMA_RECREATED
,
313 SESSION_STORAGE_UMA_MAX
);
315 UMA_HISTOGRAM_ENUMERATION(session_storage_uma_name
,
316 SESSION_STORAGE_UMA_SUCCESS
,
317 SESSION_STORAGE_UMA_MAX
);
323 leveldb::Status
SessionStorageDatabase::TryToOpen(leveldb::DB
** db
) {
324 leveldb::Options options
;
325 // The directory exists but a valid leveldb database might not exist inside it
326 // (e.g., a subset of the needed files might be missing). Handle this
327 // situation gracefully by creating the database now.
328 options
.create_if_missing
= true;
330 return leveldb::DB::Open(options
, WideToUTF8(file_path_
.value()), db
);
331 #elif defined(OS_POSIX)
332 return leveldb::DB::Open(options
, file_path_
.value(), db
);
336 bool SessionStorageDatabase::IsOpen() const {
337 return db_
.get() != NULL
;
340 bool SessionStorageDatabase::CallerErrorCheck(bool ok
) const {
345 bool SessionStorageDatabase::ConsistencyCheck(bool ok
) {
348 base::AutoLock
auto_lock(db_lock_
);
350 is_inconsistent_
= true;
351 // We cannot recover the database during this run, e.g., the upper layer can
352 // have a different understanding of the database state (shallow and deep
354 // TODO(marja): Error handling.
358 bool SessionStorageDatabase::DatabaseErrorCheck(bool ok
) {
361 base::AutoLock
auto_lock(db_lock_
);
363 // TODO(marja): Error handling.
367 bool SessionStorageDatabase::CreateNamespace(const std::string
& namespace_id
,
369 leveldb::WriteBatch
* batch
) {
370 leveldb::Slice namespace_prefix
= NamespacePrefix();
372 leveldb::Status s
= db_
->Get(leveldb::ReadOptions(), namespace_prefix
,
374 if (!DatabaseErrorCheck(s
.ok() || s
.IsNotFound()))
377 batch
->Put(namespace_prefix
, "");
379 std::string namespace_start_key
= NamespaceStartKey(namespace_id
);
380 s
= db_
->Get(leveldb::ReadOptions(), namespace_start_key
, &dummy
);
381 if (!DatabaseErrorCheck(s
.ok() || s
.IsNotFound()))
383 if (s
.IsNotFound()) {
384 batch
->Put(namespace_start_key
, "");
387 return CallerErrorCheck(ok_if_exists
);
390 bool SessionStorageDatabase::GetAreasInNamespace(
391 const std::string
& namespace_id
,
392 std::map
<std::string
, std::string
>* areas
) {
393 std::string namespace_start_key
= NamespaceStartKey(namespace_id
);
394 scoped_ptr
<leveldb::Iterator
> it(db_
->NewIterator(leveldb::ReadOptions()));
395 it
->Seek(namespace_start_key
);
396 // If the key is not found, the status of the iterator won't be IsNotFound(),
397 // but the iterator will be invalid.
399 // The namespace_start_key is not found when the namespace doesn't contain
400 // any areas. We don't need to do anything.
403 if (!DatabaseErrorCheck(it
->status().ok()))
406 // Skip the dummy entry "namespace-<namespaceid>-" and iterate the origins.
407 for (it
->Next(); it
->Valid(); it
->Next()) {
408 std::string key
= it
->key().ToString();
409 if (key
.find(namespace_start_key
) != 0) {
410 // Iterated past the origins for this namespace.
413 std::string origin
= key
.substr(namespace_start_key
.length());
414 std::string map_id
= it
->value().ToString();
415 (*areas
)[origin
] = map_id
;
420 void SessionStorageDatabase::AddAreaToNamespace(const std::string
& namespace_id
,
421 const std::string
& origin
,
422 const std::string
& map_id
,
423 leveldb::WriteBatch
* batch
) {
424 std::string namespace_key
= NamespaceKey(namespace_id
, origin
);
425 batch
->Put(namespace_key
, map_id
);
428 bool SessionStorageDatabase::DeleteAreaHelper(
429 const std::string
& namespace_id
,
430 const std::string
& origin
,
431 leveldb::WriteBatch
* batch
) {
434 if (!GetMapForArea(namespace_id
, origin
, leveldb::ReadOptions(), &exists
,
438 return true; // Nothing to delete.
439 if (!DecreaseMapRefCount(map_id
, 1, batch
))
441 std::string namespace_key
= NamespaceKey(namespace_id
, origin
);
442 batch
->Delete(namespace_key
);
444 // If this was the only area in the namespace, delete the namespace start key,
446 std::string namespace_start_key
= NamespaceStartKey(namespace_id
);
447 scoped_ptr
<leveldb::Iterator
> it(db_
->NewIterator(leveldb::ReadOptions()));
448 it
->Seek(namespace_start_key
);
449 if (!ConsistencyCheck(it
->Valid()))
451 // Advance the iterator 2 times (we still haven't really deleted
454 if (!ConsistencyCheck(it
->Valid()))
459 std::string key
= it
->key().ToString();
460 if (key
.find(namespace_start_key
) != 0)
461 batch
->Delete(namespace_start_key
);
465 bool SessionStorageDatabase::GetMapForArea(const std::string
& namespace_id
,
466 const std::string
& origin
,
467 const leveldb::ReadOptions
& options
,
468 bool* exists
, std::string
* map_id
) {
469 std::string namespace_key
= NamespaceKey(namespace_id
, origin
);
470 leveldb::Status s
= db_
->Get(options
, namespace_key
, map_id
);
471 if (s
.IsNotFound()) {
476 return DatabaseErrorCheck(s
.ok());
479 bool SessionStorageDatabase::CreateMapForArea(const std::string
& namespace_id
,
482 leveldb::WriteBatch
* batch
) {
483 leveldb::Slice next_map_id_key
= NextMapIdKey();
484 leveldb::Status s
= db_
->Get(leveldb::ReadOptions(), next_map_id_key
, map_id
);
485 if (!DatabaseErrorCheck(s
.ok() || s
.IsNotFound()))
487 int64 next_map_id
= 0;
488 if (s
.IsNotFound()) {
491 bool conversion_ok
= base::StringToInt64(*map_id
, &next_map_id
);
492 if (!ConsistencyCheck(conversion_ok
))
495 batch
->Put(next_map_id_key
, base::Int64ToString(++next_map_id
));
496 std::string namespace_key
= NamespaceKey(namespace_id
, origin
.spec());
497 batch
->Put(namespace_key
, *map_id
);
498 batch
->Put(MapRefCountKey(*map_id
), "1");
502 bool SessionStorageDatabase::ReadMap(const std::string
& map_id
,
503 const leveldb::ReadOptions
& options
,
506 scoped_ptr
<leveldb::Iterator
> it(db_
->NewIterator(options
));
507 std::string map_start_key
= MapRefCountKey(map_id
);
508 it
->Seek(map_start_key
);
509 // If the key is not found, the status of the iterator won't be IsNotFound(),
510 // but the iterator will be invalid. The map needs to exist, otherwise we have
511 // a stale map_id in the database.
512 if (!ConsistencyCheck(it
->Valid()))
514 if (!DatabaseErrorCheck(it
->status().ok()))
516 // Skip the dummy entry "map-<mapid>-".
517 for (it
->Next(); it
->Valid(); it
->Next()) {
518 std::string key
= it
->key().ToString();
519 if (key
.find(map_start_key
) != 0) {
520 // Iterated past the keys in this map.
523 // Key is of the form "map-<mapid>-<key>".
524 base::string16 key16
= UTF8ToUTF16(key
.substr(map_start_key
.length()));
526 (*result
)[key16
] = NullableString16(true);
528 // Convert the raw data stored in std::string (it->value()) to raw data
529 // stored in base::string16.
530 size_t len
= it
->value().size() / sizeof(char16
);
531 const char16
* data_ptr
=
532 reinterpret_cast<const char16
*>(it
->value().data());
533 (*result
)[key16
] = NullableString16(base::string16(data_ptr
, len
), false);
539 void SessionStorageDatabase::WriteValuesToMap(const std::string
& map_id
,
540 const ValuesMap
& values
,
541 leveldb::WriteBatch
* batch
) {
542 for (ValuesMap::const_iterator it
= values
.begin(); it
!= values
.end();
544 NullableString16 value
= it
->second
;
545 std::string key
= MapKey(map_id
, UTF16ToUTF8(it
->first
));
546 if (value
.is_null()) {
549 // Convert the raw data stored in base::string16 to raw data stored in
551 const char* data
= reinterpret_cast<const char*>(value
.string().data());
552 size_t size
= value
.string().size() * 2;
553 batch
->Put(key
, leveldb::Slice(data
, size
));
558 bool SessionStorageDatabase::GetMapRefCount(const std::string
& map_id
,
560 std::string ref_count_string
;
561 leveldb::Status s
= db_
->Get(leveldb::ReadOptions(),
562 MapRefCountKey(map_id
), &ref_count_string
);
563 if (!ConsistencyCheck(s
.ok()))
565 bool conversion_ok
= base::StringToInt64(ref_count_string
, ref_count
);
566 return ConsistencyCheck(conversion_ok
);
569 bool SessionStorageDatabase::IncreaseMapRefCount(const std::string
& map_id
,
570 leveldb::WriteBatch
* batch
) {
571 // Increase the ref count for the map.
573 if (!GetMapRefCount(map_id
, &old_ref_count
))
575 batch
->Put(MapRefCountKey(map_id
), base::Int64ToString(++old_ref_count
));
579 bool SessionStorageDatabase::DecreaseMapRefCount(const std::string
& map_id
,
581 leveldb::WriteBatch
* batch
) {
582 // Decrease the ref count for the map.
584 if (!GetMapRefCount(map_id
, &ref_count
))
586 if (!ConsistencyCheck(decrease
<= ref_count
))
588 ref_count
-= decrease
;
590 batch
->Put(MapRefCountKey(map_id
), base::Int64ToString(ref_count
));
592 // Clear all keys in the map.
593 if (!ClearMap(map_id
, batch
))
595 batch
->Delete(MapRefCountKey(map_id
));
600 bool SessionStorageDatabase::ClearMap(const std::string
& map_id
,
601 leveldb::WriteBatch
* batch
) {
603 if (!ReadMap(map_id
, leveldb::ReadOptions(), &values
, true))
605 for (ValuesMap::const_iterator it
= values
.begin(); it
!= values
.end(); ++it
)
606 batch
->Delete(MapKey(map_id
, UTF16ToUTF8(it
->first
)));
610 bool SessionStorageDatabase::DeepCopyArea(
611 const std::string
& namespace_id
, const GURL
& origin
, bool copy_data
,
612 std::string
* map_id
, leveldb::WriteBatch
* batch
) {
613 // Example, data before deep copy:
614 // | namespace-1- (1 = namespace id)| dummy |
615 // | namespace-1-origin1 | 1 (mapid) |
616 // | namespace-2- | dummy |
617 // | namespace-2-origin1 | 1 (mapid) << references the same map
618 // | map-1- | 2 (refcount) |
621 // Example, data after deep copy copy:
622 // | namespace-1-(1 = namespace id) | dummy |
623 // | namespace-1-origin1 | 1 (mapid) |
624 // | namespace-2- | dummy |
625 // | namespace-2-origin1 | 2 (mapid) << references the new map
626 // | map-1- | 1 (dec. refcount) |
628 // | map-2- | 1 (refcount) |
631 // Read the values from the old map here. If we don't need to copy the data,
632 // this can stay empty.
634 if (copy_data
&& !ReadMap(*map_id
, leveldb::ReadOptions(), &values
, false))
636 if (!DecreaseMapRefCount(*map_id
, 1, batch
))
638 // Create a new map (this will also break the association to the old map) and
639 // write the old data into it. This will write the id of the created map into
641 if (!CreateMapForArea(namespace_id
, origin
, map_id
, batch
))
643 WriteValuesToMap(*map_id
, values
, batch
);
647 std::string
SessionStorageDatabase::NamespaceStartKey(
648 const std::string
& namespace_id
) {
649 return base::StringPrintf("namespace-%s-", namespace_id
.c_str());
652 std::string
SessionStorageDatabase::NamespaceKey(
653 const std::string
& namespace_id
, const std::string
& origin
) {
654 return base::StringPrintf("namespace-%s-%s", namespace_id
.c_str(),
658 const char* SessionStorageDatabase::NamespacePrefix() {
662 std::string
SessionStorageDatabase::MapRefCountKey(const std::string
& map_id
) {
663 return base::StringPrintf("map-%s-", map_id
.c_str());
666 std::string
SessionStorageDatabase::MapKey(const std::string
& map_id
,
667 const std::string
& key
) {
668 return base::StringPrintf("map-%s-%s", map_id
.c_str(), key
.c_str());
671 const char* SessionStorageDatabase::NextMapIdKey() {
672 return "next-map-id";
675 } // namespace dom_storage