content/browser/indexed_db/leveldb/leveldb_database.cc

   1 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "content/browser/indexed_db/leveldb/leveldb_database.h"
   6
   7 #include <cerrno>
   8
   9 #include "base/basictypes.h"
  10 #include "base/files/file.h"
  11 #include "base/logging.h"
  12 #include "base/memory/scoped_ptr.h"
  13 #include "base/metrics/histogram.h"
  14 #include "base/strings/string16.h"
  15 #include "base/strings/string_piece.h"
  16 #include "base/strings/stringprintf.h"
  17 #include "base/strings/utf_string_conversions.h"
  18 #include "base/sys_info.h"
  19 #include "content/browser/indexed_db/leveldb/leveldb_comparator.h"
  20 #include "content/browser/indexed_db/leveldb/leveldb_iterator.h"
  21 #include "content/browser/indexed_db/leveldb/leveldb_write_batch.h"
  22 #include "third_party/leveldatabase/env_chromium.h"
  23 #include "third_party/leveldatabase/env_idb.h"
  24 #include "third_party/leveldatabase/src/helpers/memenv/memenv.h"
  25 #include "third_party/leveldatabase/src/include/leveldb/comparator.h"
  26 #include "third_party/leveldatabase/src/include/leveldb/db.h"
  27 #include "third_party/leveldatabase/src/include/leveldb/env.h"
  28 #include "third_party/leveldatabase/src/include/leveldb/slice.h"
  29
  30 using base::StringPiece;
  31
  32 namespace content {
  33
  34 // Forcing flushes to disk at the end of a transaction guarantees that the
  35 // data hit disk, but drastically impacts throughput when the filesystem is
  36 // busy with background compactions. Not syncing trades off reliability for
  37 // performance. Note that background compactions which move data from the
  38 // log to SSTs are always done with reliable writes.
  39 //
  40 // Sync writes are necessary on Windows for quota calculations; POSIX
  41 // calculates file sizes correctly even when not synced to disk.
  42 #if defined(OS_WIN)
  43 static const bool kSyncWrites = true;
  44 #else
  45 // TODO(dgrogan): Either remove the #if block or change this back to false.
  46 // See http://crbug.com/338385.
  47 static const bool kSyncWrites = true;
  48 #endif
  49
  50 static leveldb::Slice MakeSlice(const StringPiece& s) {
  51   return leveldb::Slice(s.begin(), s.size());
  52 }
  53
  54 static StringPiece MakeStringPiece(const leveldb::Slice& s) {
  55   return StringPiece(s.data(), s.size());
  56 }
  57
  58 class ComparatorAdapter : public leveldb::Comparator {
  59  public:
  60   explicit ComparatorAdapter(const LevelDBComparator* comparator)
  61       : comparator_(comparator) {}
  62
  63   virtual int Compare(const leveldb::Slice& a, const leveldb::Slice& b) const
  64       OVERRIDE {
  65     return comparator_->Compare(MakeStringPiece(a), MakeStringPiece(b));
  66   }
  67
  68   virtual const char* Name() const OVERRIDE { return comparator_->Name(); }
  69
  70   // TODO(jsbell): Support the methods below in the future.
  71   virtual void FindShortestSeparator(std::string* start,
  72                                      const leveldb::Slice& limit) const
  73       OVERRIDE {}
  74   virtual void FindShortSuccessor(std::string* key) const OVERRIDE {}
  75
  76  private:
  77   const LevelDBComparator* comparator_;
  78 };
  79
  80 LevelDBSnapshot::LevelDBSnapshot(LevelDBDatabase* db)
  81     : db_(db->db_.get()), snapshot_(db_->GetSnapshot()) {}
  82
  83 LevelDBSnapshot::~LevelDBSnapshot() { db_->ReleaseSnapshot(snapshot_); }
  84
  85 LevelDBDatabase::LevelDBDatabase() {}
  86
  87 LevelDBDatabase::~LevelDBDatabase() {
  88   // db_'s destructor uses comparator_adapter_; order of deletion is important.
  89   db_.reset();
  90   comparator_adapter_.reset();
  91   env_.reset();
  92 }
  93
  94 static leveldb::Status OpenDB(leveldb::Comparator* comparator,
  95                               leveldb::Env* env,
  96                               const base::FilePath& path,
  97                               leveldb::DB** db) {
  98   leveldb::Options options;
  99   options.comparator = comparator;
 100   options.create_if_missing = true;
 101   options.paranoid_checks = true;
 102   options.compression = leveldb::kSnappyCompression;
 103
 104   // For info about the troubles we've run into with this parameter, see:
 105   // https://code.google.com/p/chromium/issues/detail?id=227313#c11
 106   options.max_open_files = 80;
 107   options.env = env;
 108
 109   // ChromiumEnv assumes UTF8, converts back to FilePath before using.
 110   return leveldb::DB::Open(options, path.AsUTF8Unsafe(), db);
 111 }
 112
 113 leveldb::Status LevelDBDatabase::Destroy(const base::FilePath& file_name) {
 114   leveldb::Options options;
 115   options.env = leveldb::IDBEnv();
 116   // ChromiumEnv assumes UTF8, converts back to FilePath before using.
 117   return leveldb::DestroyDB(file_name.AsUTF8Unsafe(), options);
 118 }
 119
 120 namespace {
 121 class LockImpl : public LevelDBLock {
 122  public:
 123   explicit LockImpl(leveldb::Env* env, leveldb::FileLock* lock)
 124       : env_(env), lock_(lock) {}
 125   virtual ~LockImpl() { env_->UnlockFile(lock_); }
 126  private:
 127   leveldb::Env* env_;
 128   leveldb::FileLock* lock_;
 129 };
 130 }
 131
 132 scoped_ptr<LevelDBLock> LevelDBDatabase::LockForTesting(
 133     const base::FilePath& file_name) {
 134   leveldb::Env* env = leveldb::IDBEnv();
 135   base::FilePath lock_path = file_name.AppendASCII("LOCK");
 136   leveldb::FileLock* lock = NULL;
 137   leveldb::Status status = env->LockFile(lock_path.AsUTF8Unsafe(), &lock);
 138   if (!status.ok())
 139     return scoped_ptr<LevelDBLock>();
 140   DCHECK(lock);
 141   return scoped_ptr<LevelDBLock>(new LockImpl(env, lock));
 142 }
 143
 144 static int CheckFreeSpace(const char* const type,
 145                           const base::FilePath& file_name) {
 146   std::string name =
 147       std::string("WebCore.IndexedDB.LevelDB.Open") + type + "FreeDiskSpace";
 148   int64 free_disk_space_in_k_bytes =
 149       base::SysInfo::AmountOfFreeDiskSpace(file_name) / 1024;
 150   if (free_disk_space_in_k_bytes < 0) {
 151     base::Histogram::FactoryGet(
 152         "WebCore.IndexedDB.LevelDB.FreeDiskSpaceFailure",
 153         1,
 154         2 /*boundary*/,
 155         2 /*boundary*/ + 1,
 156         base::HistogramBase::kUmaTargetedHistogramFlag)->Add(1 /*sample*/);
 157     return -1;
 158   }
 159   int clamped_disk_space_k_bytes = free_disk_space_in_k_bytes > INT_MAX
 160                                        ? INT_MAX
 161                                        : free_disk_space_in_k_bytes;
 162   const uint64 histogram_max = static_cast<uint64>(1e9);
 163   COMPILE_ASSERT(histogram_max <= INT_MAX, histogram_max_too_big);
 164   base::Histogram::FactoryGet(name,
 165                               1,
 166                               histogram_max,
 167                               11 /*buckets*/,
 168                               base::HistogramBase::kUmaTargetedHistogramFlag)
 169       ->Add(clamped_disk_space_k_bytes);
 170   return clamped_disk_space_k_bytes;
 171 }
 172
 173 static void ParseAndHistogramIOErrorDetails(const std::string& histogram_name,
 174                                             const leveldb::Status& s) {
 175   leveldb_env::MethodID method;
 176   int error = -1;
 177   leveldb_env::ErrorParsingResult result =
 178       leveldb_env::ParseMethodAndError(s.ToString().c_str(), &method, &error);
 179   if (result == leveldb_env::NONE)
 180     return;
 181   std::string method_histogram_name(histogram_name);
 182   method_histogram_name.append(".EnvMethod");
 183   base::LinearHistogram::FactoryGet(
 184       method_histogram_name,
 185       1,
 186       leveldb_env::kNumEntries,
 187       leveldb_env::kNumEntries + 1,
 188       base::HistogramBase::kUmaTargetedHistogramFlag)->Add(method);
 189
 190   std::string error_histogram_name(histogram_name);
 191
 192   if (result == leveldb_env::METHOD_AND_PFE) {
 193     DCHECK(error < 0);
 194     error_histogram_name.append(std::string(".PFE.") +
 195                                 leveldb_env::MethodIDToString(method));
 196     base::LinearHistogram::FactoryGet(
 197         error_histogram_name,
 198         1,
 199         -base::File::FILE_ERROR_MAX,
 200         -base::File::FILE_ERROR_MAX + 1,
 201         base::HistogramBase::kUmaTargetedHistogramFlag)->Add(-error);
 202   } else if (result == leveldb_env::METHOD_AND_ERRNO) {
 203     error_histogram_name.append(std::string(".Errno.") +
 204                                 leveldb_env::MethodIDToString(method));
 205     base::LinearHistogram::FactoryGet(
 206         error_histogram_name,
 207         1,
 208         ERANGE + 1,
 209         ERANGE + 2,
 210         base::HistogramBase::kUmaTargetedHistogramFlag)->Add(error);
 211   }
 212 }
 213
 214 static void ParseAndHistogramCorruptionDetails(
 215     const std::string& histogram_name,
 216     const leveldb::Status& status) {
 217   int error = leveldb_env::GetCorruptionCode(status);
 218   DCHECK(error >= 0);
 219   std::string corruption_histogram_name(histogram_name);
 220   corruption_histogram_name.append(".Corruption");
 221   const int kNumPatterns = leveldb_env::GetNumCorruptionCodes();
 222   base::LinearHistogram::FactoryGet(
 223       corruption_histogram_name,
 224       1,
 225       kNumPatterns,
 226       kNumPatterns + 1,
 227       base::HistogramBase::kUmaTargetedHistogramFlag)->Add(error);
 228 }
 229
 230 static void HistogramLevelDBError(const std::string& histogram_name,
 231                                   const leveldb::Status& s) {
 232   if (s.ok()) {
 233     NOTREACHED();
 234     return;
 235   }
 236   enum {
 237     LEVEL_DB_NOT_FOUND,
 238     LEVEL_DB_CORRUPTION,
 239     LEVEL_DB_IO_ERROR,
 240     LEVEL_DB_OTHER,
 241     LEVEL_DB_MAX_ERROR
 242   };
 243   int leveldb_error = LEVEL_DB_OTHER;
 244   if (s.IsNotFound())
 245     leveldb_error = LEVEL_DB_NOT_FOUND;
 246   else if (s.IsCorruption())
 247     leveldb_error = LEVEL_DB_CORRUPTION;
 248   else if (s.IsIOError())
 249     leveldb_error = LEVEL_DB_IO_ERROR;
 250   base::Histogram::FactoryGet(histogram_name,
 251                               1,
 252                               LEVEL_DB_MAX_ERROR,
 253                               LEVEL_DB_MAX_ERROR + 1,
 254                               base::HistogramBase::kUmaTargetedHistogramFlag)
 255       ->Add(leveldb_error);
 256   if (s.IsIOError())
 257     ParseAndHistogramIOErrorDetails(histogram_name, s);
 258   else
 259     ParseAndHistogramCorruptionDetails(histogram_name, s);
 260 }
 261
 262 leveldb::Status LevelDBDatabase::Open(const base::FilePath& file_name,
 263                                       const LevelDBComparator* comparator,
 264                                       scoped_ptr<LevelDBDatabase>* result,
 265                                       bool* is_disk_full) {
 266   scoped_ptr<ComparatorAdapter> comparator_adapter(
 267       new ComparatorAdapter(comparator));
 268
 269   leveldb::DB* db;
 270   const leveldb::Status s =
 271       OpenDB(comparator_adapter.get(), leveldb::IDBEnv(), file_name, &db);
 272
 273   if (!s.ok()) {
 274     HistogramLevelDBError("WebCore.IndexedDB.LevelDBOpenErrors", s);
 275     int free_space_k_bytes = CheckFreeSpace("Failure", file_name);
 276     // Disks with <100k of free space almost never succeed in opening a
 277     // leveldb database.
 278     if (is_disk_full)
 279       *is_disk_full = free_space_k_bytes >= 0 && free_space_k_bytes < 100;
 280
 281     LOG(ERROR) << "Failed to open LevelDB database from "
 282                << file_name.AsUTF8Unsafe() << "," << s.ToString();
 283     return s;
 284   }
 285
 286   CheckFreeSpace("Success", file_name);
 287
 288   (*result).reset(new LevelDBDatabase);
 289   (*result)->db_ = make_scoped_ptr(db);
 290   (*result)->comparator_adapter_ = comparator_adapter.Pass();
 291   (*result)->comparator_ = comparator;
 292
 293   return s;
 294 }
 295
 296 scoped_ptr<LevelDBDatabase> LevelDBDatabase::OpenInMemory(
 297     const LevelDBComparator* comparator) {
 298   scoped_ptr<ComparatorAdapter> comparator_adapter(
 299       new ComparatorAdapter(comparator));
 300   scoped_ptr<leveldb::Env> in_memory_env(leveldb::NewMemEnv(leveldb::IDBEnv()));
 301
 302   leveldb::DB* db;
 303   const leveldb::Status s = OpenDB(
 304       comparator_adapter.get(), in_memory_env.get(), base::FilePath(), &db);
 305
 306   if (!s.ok()) {
 307     LOG(ERROR) << "Failed to open in-memory LevelDB database: " << s.ToString();
 308     return scoped_ptr<LevelDBDatabase>();
 309   }
 310
 311   scoped_ptr<LevelDBDatabase> result(new LevelDBDatabase);
 312   result->env_ = in_memory_env.Pass();
 313   result->db_ = make_scoped_ptr(db);
 314   result->comparator_adapter_ = comparator_adapter.Pass();
 315   result->comparator_ = comparator;
 316
 317   return result.Pass();
 318 }
 319
 320 leveldb::Status LevelDBDatabase::Put(const StringPiece& key,
 321                                      std::string* value) {
 322   leveldb::WriteOptions write_options;
 323   write_options.sync = kSyncWrites;
 324
 325   const leveldb::Status s =
 326       db_->Put(write_options, MakeSlice(key), MakeSlice(*value));
 327   if (!s.ok())
 328     LOG(ERROR) << "LevelDB put failed: " << s.ToString();
 329   return s;
 330 }
 331
 332 leveldb::Status LevelDBDatabase::Remove(const StringPiece& key) {
 333   leveldb::WriteOptions write_options;
 334   write_options.sync = kSyncWrites;
 335
 336   const leveldb::Status s = db_->Delete(write_options, MakeSlice(key));
 337   if (!s.IsNotFound())
 338     LOG(ERROR) << "LevelDB remove failed: " << s.ToString();
 339   return s;
 340 }
 341
 342 leveldb::Status LevelDBDatabase::Get(const StringPiece& key,
 343                                      std::string* value,
 344                                      bool* found,
 345                                      const LevelDBSnapshot* snapshot) {
 346   *found = false;
 347   leveldb::ReadOptions read_options;
 348   read_options.verify_checksums = true;  // TODO(jsbell): Disable this if the
 349                                          // performance impact is too great.
 350   read_options.snapshot = snapshot ? snapshot->snapshot_ : 0;
 351
 352   const leveldb::Status s = db_->Get(read_options, MakeSlice(key), value);
 353   if (s.ok()) {
 354     *found = true;
 355     return s;
 356   }
 357   if (s.IsNotFound())
 358     return leveldb::Status::OK();
 359   HistogramLevelDBError("WebCore.IndexedDB.LevelDBReadErrors", s);
 360   LOG(ERROR) << "LevelDB get failed: " << s.ToString();
 361   return s;
 362 }
 363
 364 leveldb::Status LevelDBDatabase::Write(const LevelDBWriteBatch& write_batch) {
 365   leveldb::WriteOptions write_options;
 366   write_options.sync = kSyncWrites;
 367
 368   const leveldb::Status s =
 369       db_->Write(write_options, write_batch.write_batch_.get());
 370   if (!s.ok()) {
 371     HistogramLevelDBError("WebCore.IndexedDB.LevelDBWriteErrors", s);
 372     LOG(ERROR) << "LevelDB write failed: " << s.ToString();
 373   }
 374   return s;
 375 }
 376
 377 namespace {
 378 class IteratorImpl : public LevelDBIterator {
 379  public:
 380   virtual ~IteratorImpl() {}
 381
 382   virtual bool IsValid() const OVERRIDE;
 383   virtual void SeekToLast() OVERRIDE;
 384   virtual void Seek(const StringPiece& target) OVERRIDE;
 385   virtual void Next() OVERRIDE;
 386   virtual void Prev() OVERRIDE;
 387   virtual StringPiece Key() const OVERRIDE;
 388   virtual StringPiece Value() const OVERRIDE;
 389
 390  private:
 391   friend class content::LevelDBDatabase;
 392   explicit IteratorImpl(scoped_ptr<leveldb::Iterator> iterator);
 393   void CheckStatus();
 394
 395   scoped_ptr<leveldb::Iterator> iterator_;
 396 };
 397 }
 398
 399 IteratorImpl::IteratorImpl(scoped_ptr<leveldb::Iterator> it)
 400     : iterator_(it.Pass()) {}
 401
 402 void IteratorImpl::CheckStatus() {
 403   const leveldb::Status s = iterator_->status();
 404   if (!s.ok())
 405     LOG(ERROR) << "LevelDB iterator error: " << s.ToString();
 406 }
 407
 408 bool IteratorImpl::IsValid() const { return iterator_->Valid(); }
 409
 410 void IteratorImpl::SeekToLast() {
 411   iterator_->SeekToLast();
 412   CheckStatus();
 413 }
 414
 415 void IteratorImpl::Seek(const StringPiece& target) {
 416   iterator_->Seek(MakeSlice(target));
 417   CheckStatus();
 418 }
 419
 420 void IteratorImpl::Next() {
 421   DCHECK(IsValid());
 422   iterator_->Next();
 423   CheckStatus();
 424 }
 425
 426 void IteratorImpl::Prev() {
 427   DCHECK(IsValid());
 428   iterator_->Prev();
 429   CheckStatus();
 430 }
 431
 432 StringPiece IteratorImpl::Key() const {
 433   DCHECK(IsValid());
 434   return MakeStringPiece(iterator_->key());
 435 }
 436
 437 StringPiece IteratorImpl::Value() const {
 438   DCHECK(IsValid());
 439   return MakeStringPiece(iterator_->value());
 440 }
 441
 442 scoped_ptr<LevelDBIterator> LevelDBDatabase::CreateIterator(
 443     const LevelDBSnapshot* snapshot) {
 444   leveldb::ReadOptions read_options;
 445   read_options.verify_checksums = true;  // TODO(jsbell): Disable this if the
 446                                          // performance impact is too great.
 447   read_options.snapshot = snapshot ? snapshot->snapshot_ : 0;
 448
 449   scoped_ptr<leveldb::Iterator> i(db_->NewIterator(read_options));
 450   return scoped_ptr<LevelDBIterator>(new IteratorImpl(i.Pass()));
 451 }
 452
 453 const LevelDBComparator* LevelDBDatabase::Comparator() const {
 454   return comparator_;
 455 }
 456
 457 void LevelDBDatabase::Compact(const base::StringPiece& start,
 458                               const base::StringPiece& stop) {
 459   const leveldb::Slice start_slice = MakeSlice(start);
 460   const leveldb::Slice stop_slice = MakeSlice(stop);
 461   db_->CompactRange(&start_slice, &stop_slice);
 462 }
 463
 464 }  // namespace content