content/browser/indexed_db/leveldb/leveldb_database.cc

   1 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "content/browser/indexed_db/leveldb/leveldb_database.h"
   6
   7 #include <cerrno>
   8
   9 #include "base/basictypes.h"
  10 #include "base/files/file.h"
  11 #include "base/logging.h"
  12 #include "base/memory/scoped_ptr.h"
  13 #include "base/metrics/histogram.h"
  14 #include "base/strings/string16.h"
  15 #include "base/strings/string_piece.h"
  16 #include "base/strings/stringprintf.h"
  17 #include "base/strings/utf_string_conversions.h"
  18 #include "base/sys_info.h"
  19 #include "content/browser/indexed_db/leveldb/leveldb_comparator.h"
  20 #include "content/browser/indexed_db/leveldb/leveldb_iterator.h"
  21 #include "content/browser/indexed_db/leveldb/leveldb_write_batch.h"
  22 #include "third_party/leveldatabase/env_chromium.h"
  23 #include "third_party/leveldatabase/env_idb.h"
  24 #include "third_party/leveldatabase/src/helpers/memenv/memenv.h"
  25 #include "third_party/leveldatabase/src/include/leveldb/db.h"
  26 #include "third_party/leveldatabase/src/include/leveldb/env.h"
  27 #include "third_party/leveldatabase/src/include/leveldb/slice.h"
  28
  29 using base::StringPiece;
  30
  31 namespace content {
  32
  33 // Forcing flushes to disk at the end of a transaction guarantees that the
  34 // data hit disk, but drastically impacts throughput when the filesystem is
  35 // busy with background compactions. Not syncing trades off reliability for
  36 // performance. Note that background compactions which move data from the
  37 // log to SSTs are always done with reliable writes.
  38 //
  39 // Sync writes are necessary on Windows for quota calculations; POSIX
  40 // calculates file sizes correctly even when not synced to disk.
  41 #if defined(OS_WIN)
  42 static const bool kSyncWrites = true;
  43 #else
  44 // TODO(dgrogan): Either remove the #if block or change this back to false.
  45 // See http://crbug.com/338385.
  46 static const bool kSyncWrites = true;
  47 #endif
  48
  49 static leveldb::Slice MakeSlice(const StringPiece& s) {
  50   return leveldb::Slice(s.begin(), s.size());
  51 }
  52
  53 static StringPiece MakeStringPiece(const leveldb::Slice& s) {
  54   return StringPiece(s.data(), s.size());
  55 }
  56
  57 LevelDBDatabase::ComparatorAdapter::ComparatorAdapter(
  58     const LevelDBComparator* comparator)
  59     : comparator_(comparator) {}
  60
  61 int LevelDBDatabase::ComparatorAdapter::Compare(const leveldb::Slice& a,
  62                                                 const leveldb::Slice& b) const {
  63   return comparator_->Compare(MakeStringPiece(a), MakeStringPiece(b));
  64 }
  65
  66 const char* LevelDBDatabase::ComparatorAdapter::Name() const {
  67   return comparator_->Name();
  68 }
  69
  70 // TODO(jsbell): Support the methods below in the future.
  71 void LevelDBDatabase::ComparatorAdapter::FindShortestSeparator(
  72     std::string* start,
  73     const leveldb::Slice& limit) const {}
  74
  75 void LevelDBDatabase::ComparatorAdapter::FindShortSuccessor(
  76     std::string* key) const {}
  77
  78 LevelDBSnapshot::LevelDBSnapshot(LevelDBDatabase* db)
  79     : db_(db->db_.get()), snapshot_(db_->GetSnapshot()) {}
  80
  81 LevelDBSnapshot::~LevelDBSnapshot() { db_->ReleaseSnapshot(snapshot_); }
  82
  83 LevelDBDatabase::LevelDBDatabase() {}
  84
  85 LevelDBDatabase::~LevelDBDatabase() {
  86   // db_'s destructor uses comparator_adapter_; order of deletion is important.
  87   db_.reset();
  88   comparator_adapter_.reset();
  89   env_.reset();
  90 }
  91
  92 static leveldb::Status OpenDB(leveldb::Comparator* comparator,
  93                               leveldb::Env* env,
  94                               const base::FilePath& path,
  95                               leveldb::DB** db) {
  96   leveldb::Options options;
  97   options.comparator = comparator;
  98   options.create_if_missing = true;
  99   options.paranoid_checks = true;
 100   options.compression = leveldb::kSnappyCompression;
 101
 102   // For info about the troubles we've run into with this parameter, see:
 103   // https://code.google.com/p/chromium/issues/detail?id=227313#c11
 104   options.max_open_files = 80;
 105   options.env = env;
 106
 107   // ChromiumEnv assumes UTF8, converts back to FilePath before using.
 108   return leveldb::DB::Open(options, path.AsUTF8Unsafe(), db);
 109 }
 110
 111 leveldb::Status LevelDBDatabase::Destroy(const base::FilePath& file_name) {
 112   leveldb::Options options;
 113   options.env = leveldb::IDBEnv();
 114   // ChromiumEnv assumes UTF8, converts back to FilePath before using.
 115   return leveldb::DestroyDB(file_name.AsUTF8Unsafe(), options);
 116 }
 117
 118 namespace {
 119 class LockImpl : public LevelDBLock {
 120  public:
 121   explicit LockImpl(leveldb::Env* env, leveldb::FileLock* lock)
 122       : env_(env), lock_(lock) {}
 123   virtual ~LockImpl() { env_->UnlockFile(lock_); }
 124  private:
 125   leveldb::Env* env_;
 126   leveldb::FileLock* lock_;
 127 };
 128 }
 129
 130 scoped_ptr<LevelDBLock> LevelDBDatabase::LockForTesting(
 131     const base::FilePath& file_name) {
 132   leveldb::Env* env = leveldb::IDBEnv();
 133   base::FilePath lock_path = file_name.AppendASCII("LOCK");
 134   leveldb::FileLock* lock = NULL;
 135   leveldb::Status status = env->LockFile(lock_path.AsUTF8Unsafe(), &lock);
 136   if (!status.ok())
 137     return scoped_ptr<LevelDBLock>();
 138   DCHECK(lock);
 139   return scoped_ptr<LevelDBLock>(new LockImpl(env, lock));
 140 }
 141
 142 static int CheckFreeSpace(const char* const type,
 143                           const base::FilePath& file_name) {
 144   std::string name =
 145       std::string("WebCore.IndexedDB.LevelDB.Open") + type + "FreeDiskSpace";
 146   int64 free_disk_space_in_k_bytes =
 147       base::SysInfo::AmountOfFreeDiskSpace(file_name) / 1024;
 148   if (free_disk_space_in_k_bytes < 0) {
 149     base::Histogram::FactoryGet(
 150         "WebCore.IndexedDB.LevelDB.FreeDiskSpaceFailure",
 151         1,
 152         2 /*boundary*/,
 153         2 /*boundary*/ + 1,
 154         base::HistogramBase::kUmaTargetedHistogramFlag)->Add(1 /*sample*/);
 155     return -1;
 156   }
 157   int clamped_disk_space_k_bytes = free_disk_space_in_k_bytes > INT_MAX
 158                                        ? INT_MAX
 159                                        : free_disk_space_in_k_bytes;
 160   const uint64 histogram_max = static_cast<uint64>(1e9);
 161   COMPILE_ASSERT(histogram_max <= INT_MAX, histogram_max_too_big);
 162   base::Histogram::FactoryGet(name,
 163                               1,
 164                               histogram_max,
 165                               11 /*buckets*/,
 166                               base::HistogramBase::kUmaTargetedHistogramFlag)
 167       ->Add(clamped_disk_space_k_bytes);
 168   return clamped_disk_space_k_bytes;
 169 }
 170
 171 static void ParseAndHistogramIOErrorDetails(const std::string& histogram_name,
 172                                             const leveldb::Status& s) {
 173   leveldb_env::MethodID method;
 174   int error = -1;
 175   leveldb_env::ErrorParsingResult result =
 176       leveldb_env::ParseMethodAndError(s.ToString().c_str(), &method, &error);
 177   if (result == leveldb_env::NONE)
 178     return;
 179   std::string method_histogram_name(histogram_name);
 180   method_histogram_name.append(".EnvMethod");
 181   base::LinearHistogram::FactoryGet(
 182       method_histogram_name,
 183       1,
 184       leveldb_env::kNumEntries,
 185       leveldb_env::kNumEntries + 1,
 186       base::HistogramBase::kUmaTargetedHistogramFlag)->Add(method);
 187
 188   std::string error_histogram_name(histogram_name);
 189
 190   if (result == leveldb_env::METHOD_AND_PFE) {
 191     DCHECK(error < 0);
 192     error_histogram_name.append(std::string(".PFE.") +
 193                                 leveldb_env::MethodIDToString(method));
 194     base::LinearHistogram::FactoryGet(
 195         error_histogram_name,
 196         1,
 197         -base::File::FILE_ERROR_MAX,
 198         -base::File::FILE_ERROR_MAX + 1,
 199         base::HistogramBase::kUmaTargetedHistogramFlag)->Add(-error);
 200   } else if (result == leveldb_env::METHOD_AND_ERRNO) {
 201     error_histogram_name.append(std::string(".Errno.") +
 202                                 leveldb_env::MethodIDToString(method));
 203     base::LinearHistogram::FactoryGet(
 204         error_histogram_name,
 205         1,
 206         ERANGE + 1,
 207         ERANGE + 2,
 208         base::HistogramBase::kUmaTargetedHistogramFlag)->Add(error);
 209   }
 210 }
 211
 212 static void ParseAndHistogramCorruptionDetails(
 213     const std::string& histogram_name,
 214     const leveldb::Status& status) {
 215   int error = leveldb_env::GetCorruptionCode(status);
 216   DCHECK(error >= 0);
 217   std::string corruption_histogram_name(histogram_name);
 218   corruption_histogram_name.append(".Corruption");
 219   const int kNumPatterns = leveldb_env::GetNumCorruptionCodes();
 220   base::LinearHistogram::FactoryGet(
 221       corruption_histogram_name,
 222       1,
 223       kNumPatterns,
 224       kNumPatterns + 1,
 225       base::HistogramBase::kUmaTargetedHistogramFlag)->Add(error);
 226 }
 227
 228 static void HistogramLevelDBError(const std::string& histogram_name,
 229                                   const leveldb::Status& s) {
 230   if (s.ok()) {
 231     NOTREACHED();
 232     return;
 233   }
 234   enum {
 235     LEVEL_DB_NOT_FOUND,
 236     LEVEL_DB_CORRUPTION,
 237     LEVEL_DB_IO_ERROR,
 238     LEVEL_DB_OTHER,
 239     LEVEL_DB_MAX_ERROR
 240   };
 241   int leveldb_error = LEVEL_DB_OTHER;
 242   if (s.IsNotFound())
 243     leveldb_error = LEVEL_DB_NOT_FOUND;
 244   else if (s.IsCorruption())
 245     leveldb_error = LEVEL_DB_CORRUPTION;
 246   else if (s.IsIOError())
 247     leveldb_error = LEVEL_DB_IO_ERROR;
 248   base::Histogram::FactoryGet(histogram_name,
 249                               1,
 250                               LEVEL_DB_MAX_ERROR,
 251                               LEVEL_DB_MAX_ERROR + 1,
 252                               base::HistogramBase::kUmaTargetedHistogramFlag)
 253       ->Add(leveldb_error);
 254   if (s.IsIOError())
 255     ParseAndHistogramIOErrorDetails(histogram_name, s);
 256   else
 257     ParseAndHistogramCorruptionDetails(histogram_name, s);
 258 }
 259
 260 leveldb::Status LevelDBDatabase::Open(const base::FilePath& file_name,
 261                                       const LevelDBComparator* comparator,
 262                                       scoped_ptr<LevelDBDatabase>* result,
 263                                       bool* is_disk_full) {
 264   scoped_ptr<ComparatorAdapter> comparator_adapter(
 265       new ComparatorAdapter(comparator));
 266
 267   leveldb::DB* db;
 268   const leveldb::Status s =
 269       OpenDB(comparator_adapter.get(), leveldb::IDBEnv(), file_name, &db);
 270
 271   if (!s.ok()) {
 272     HistogramLevelDBError("WebCore.IndexedDB.LevelDBOpenErrors", s);
 273     int free_space_k_bytes = CheckFreeSpace("Failure", file_name);
 274     // Disks with <100k of free space almost never succeed in opening a
 275     // leveldb database.
 276     if (is_disk_full)
 277       *is_disk_full = free_space_k_bytes >= 0 && free_space_k_bytes < 100;
 278
 279     LOG(ERROR) << "Failed to open LevelDB database from "
 280                << file_name.AsUTF8Unsafe() << "," << s.ToString();
 281     return s;
 282   }
 283
 284   CheckFreeSpace("Success", file_name);
 285
 286   (*result).reset(new LevelDBDatabase);
 287   (*result)->db_ = make_scoped_ptr(db);
 288   (*result)->comparator_adapter_ = comparator_adapter.Pass();
 289   (*result)->comparator_ = comparator;
 290
 291   return s;
 292 }
 293
 294 scoped_ptr<LevelDBDatabase> LevelDBDatabase::OpenInMemory(
 295     const LevelDBComparator* comparator) {
 296   scoped_ptr<ComparatorAdapter> comparator_adapter(
 297       new ComparatorAdapter(comparator));
 298   scoped_ptr<leveldb::Env> in_memory_env(leveldb::NewMemEnv(leveldb::IDBEnv()));
 299
 300   leveldb::DB* db;
 301   const leveldb::Status s = OpenDB(
 302       comparator_adapter.get(), in_memory_env.get(), base::FilePath(), &db);
 303
 304   if (!s.ok()) {
 305     LOG(ERROR) << "Failed to open in-memory LevelDB database: " << s.ToString();
 306     return scoped_ptr<LevelDBDatabase>();
 307   }
 308
 309   scoped_ptr<LevelDBDatabase> result(new LevelDBDatabase);
 310   result->env_ = in_memory_env.Pass();
 311   result->db_ = make_scoped_ptr(db);
 312   result->comparator_adapter_ = comparator_adapter.Pass();
 313   result->comparator_ = comparator;
 314
 315   return result.Pass();
 316 }
 317
 318 leveldb::Status LevelDBDatabase::Put(const StringPiece& key,
 319                                      std::string* value) {
 320   leveldb::WriteOptions write_options;
 321   write_options.sync = kSyncWrites;
 322
 323   const leveldb::Status s =
 324       db_->Put(write_options, MakeSlice(key), MakeSlice(*value));
 325   if (!s.ok())
 326     LOG(ERROR) << "LevelDB put failed: " << s.ToString();
 327   return s;
 328 }
 329
 330 leveldb::Status LevelDBDatabase::Remove(const StringPiece& key) {
 331   leveldb::WriteOptions write_options;
 332   write_options.sync = kSyncWrites;
 333
 334   const leveldb::Status s = db_->Delete(write_options, MakeSlice(key));
 335   if (!s.IsNotFound())
 336     LOG(ERROR) << "LevelDB remove failed: " << s.ToString();
 337   return s;
 338 }
 339
 340 leveldb::Status LevelDBDatabase::Get(const StringPiece& key,
 341                                      std::string* value,
 342                                      bool* found,
 343                                      const LevelDBSnapshot* snapshot) {
 344   *found = false;
 345   leveldb::ReadOptions read_options;
 346   read_options.verify_checksums = true;  // TODO(jsbell): Disable this if the
 347                                          // performance impact is too great.
 348   read_options.snapshot = snapshot ? snapshot->snapshot_ : 0;
 349
 350   const leveldb::Status s = db_->Get(read_options, MakeSlice(key), value);
 351   if (s.ok()) {
 352     *found = true;
 353     return s;
 354   }
 355   if (s.IsNotFound())
 356     return leveldb::Status::OK();
 357   HistogramLevelDBError("WebCore.IndexedDB.LevelDBReadErrors", s);
 358   LOG(ERROR) << "LevelDB get failed: " << s.ToString();
 359   return s;
 360 }
 361
 362 leveldb::Status LevelDBDatabase::Write(const LevelDBWriteBatch& write_batch) {
 363   leveldb::WriteOptions write_options;
 364   write_options.sync = kSyncWrites;
 365
 366   const leveldb::Status s =
 367       db_->Write(write_options, write_batch.write_batch_.get());
 368   if (!s.ok()) {
 369     HistogramLevelDBError("WebCore.IndexedDB.LevelDBWriteErrors", s);
 370     LOG(ERROR) << "LevelDB write failed: " << s.ToString();
 371   }
 372   return s;
 373 }
 374
 375 namespace {
 376 class IteratorImpl : public LevelDBIterator {
 377  public:
 378   virtual ~IteratorImpl() {}
 379
 380   virtual bool IsValid() const OVERRIDE;
 381   virtual void SeekToLast() OVERRIDE;
 382   virtual void Seek(const StringPiece& target) OVERRIDE;
 383   virtual void Next() OVERRIDE;
 384   virtual void Prev() OVERRIDE;
 385   virtual StringPiece Key() const OVERRIDE;
 386   virtual StringPiece Value() const OVERRIDE;
 387
 388  private:
 389   friend class content::LevelDBDatabase;
 390   explicit IteratorImpl(scoped_ptr<leveldb::Iterator> iterator);
 391   void CheckStatus();
 392
 393   scoped_ptr<leveldb::Iterator> iterator_;
 394 };
 395 }
 396
 397 IteratorImpl::IteratorImpl(scoped_ptr<leveldb::Iterator> it)
 398     : iterator_(it.Pass()) {}
 399
 400 void IteratorImpl::CheckStatus() {
 401   const leveldb::Status s = iterator_->status();
 402   if (!s.ok())
 403     LOG(ERROR) << "LevelDB iterator error: " << s.ToString();
 404 }
 405
 406 bool IteratorImpl::IsValid() const { return iterator_->Valid(); }
 407
 408 void IteratorImpl::SeekToLast() {
 409   iterator_->SeekToLast();
 410   CheckStatus();
 411 }
 412
 413 void IteratorImpl::Seek(const StringPiece& target) {
 414   iterator_->Seek(MakeSlice(target));
 415   CheckStatus();
 416 }
 417
 418 void IteratorImpl::Next() {
 419   DCHECK(IsValid());
 420   iterator_->Next();
 421   CheckStatus();
 422 }
 423
 424 void IteratorImpl::Prev() {
 425   DCHECK(IsValid());
 426   iterator_->Prev();
 427   CheckStatus();
 428 }
 429
 430 StringPiece IteratorImpl::Key() const {
 431   DCHECK(IsValid());
 432   return MakeStringPiece(iterator_->key());
 433 }
 434
 435 StringPiece IteratorImpl::Value() const {
 436   DCHECK(IsValid());
 437   return MakeStringPiece(iterator_->value());
 438 }
 439
 440 scoped_ptr<LevelDBIterator> LevelDBDatabase::CreateIterator(
 441     const LevelDBSnapshot* snapshot) {
 442   leveldb::ReadOptions read_options;
 443   read_options.verify_checksums = true;  // TODO(jsbell): Disable this if the
 444                                          // performance impact is too great.
 445   read_options.snapshot = snapshot ? snapshot->snapshot_ : 0;
 446
 447   scoped_ptr<leveldb::Iterator> i(db_->NewIterator(read_options));
 448   return scoped_ptr<LevelDBIterator>(new IteratorImpl(i.Pass()));
 449 }
 450
 451 const LevelDBComparator* LevelDBDatabase::Comparator() const {
 452   return comparator_;
 453 }
 454
 455 void LevelDBDatabase::Compact(const base::StringPiece& start,
 456                               const base::StringPiece& stop) {
 457   const leveldb::Slice start_slice = MakeSlice(start);
 458   const leveldb::Slice stop_slice = MakeSlice(stop);
 459   db_->CompactRange(&start_slice, &stop_slice);
 460 }
 461
 462 }  // namespace content