content/browser/indexed_db/leveldb/leveldb_database.cc

   1 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "content/browser/indexed_db/leveldb/leveldb_database.h"
   6
   7 #include <cerrno>
   8
   9 #include "base/basictypes.h"
  10 #include "base/files/file.h"
  11 #include "base/logging.h"
  12 #include "base/memory/scoped_ptr.h"
  13 #include "base/metrics/histogram.h"
  14 #include "base/strings/string16.h"
  15 #include "base/strings/string_piece.h"
  16 #include "base/strings/stringprintf.h"
  17 #include "base/strings/utf_string_conversions.h"
  18 #include "base/sys_info.h"
  19 #include "content/browser/indexed_db/indexed_db_class_factory.h"
  20 #include "content/browser/indexed_db/leveldb/leveldb_comparator.h"
  21 #include "content/browser/indexed_db/leveldb/leveldb_iterator_impl.h"
  22 #include "content/browser/indexed_db/leveldb/leveldb_write_batch.h"
  23 #include "third_party/leveldatabase/env_chromium.h"
  24 #include "third_party/leveldatabase/env_idb.h"
  25 #include "third_party/leveldatabase/src/helpers/memenv/memenv.h"
  26 #include "third_party/leveldatabase/src/include/leveldb/db.h"
  27 #include "third_party/leveldatabase/src/include/leveldb/env.h"
  28 #include "third_party/leveldatabase/src/include/leveldb/slice.h"
  29
  30 using base::StringPiece;
  31
  32 namespace content {
  33
  34 // Forcing flushes to disk at the end of a transaction guarantees that the
  35 // data hit disk, but drastically impacts throughput when the filesystem is
  36 // busy with background compactions. Not syncing trades off reliability for
  37 // performance. Note that background compactions which move data from the
  38 // log to SSTs are always done with reliable writes.
  39 //
  40 // Sync writes are necessary on Windows for quota calculations; POSIX
  41 // calculates file sizes correctly even when not synced to disk.
  42 #if defined(OS_WIN)
  43 static const bool kSyncWrites = true;
  44 #else
  45 // TODO(dgrogan): Either remove the #if block or change this back to false.
  46 // See http://crbug.com/338385.
  47 static const bool kSyncWrites = true;
  48 #endif
  49
  50 static leveldb::Slice MakeSlice(const StringPiece& s) {
  51   return leveldb::Slice(s.begin(), s.size());
  52 }
  53
  54 static StringPiece MakeStringPiece(const leveldb::Slice& s) {
  55   return StringPiece(s.data(), s.size());
  56 }
  57
  58 LevelDBDatabase::ComparatorAdapter::ComparatorAdapter(
  59     const LevelDBComparator* comparator)
  60     : comparator_(comparator) {}
  61
  62 int LevelDBDatabase::ComparatorAdapter::Compare(const leveldb::Slice& a,
  63                                                 const leveldb::Slice& b) const {
  64   return comparator_->Compare(MakeStringPiece(a), MakeStringPiece(b));
  65 }
  66
  67 const char* LevelDBDatabase::ComparatorAdapter::Name() const {
  68   return comparator_->Name();
  69 }
  70
  71 // TODO(jsbell): Support the methods below in the future.
  72 void LevelDBDatabase::ComparatorAdapter::FindShortestSeparator(
  73     std::string* start,
  74     const leveldb::Slice& limit) const {}
  75
  76 void LevelDBDatabase::ComparatorAdapter::FindShortSuccessor(
  77     std::string* key) const {}
  78
  79 LevelDBSnapshot::LevelDBSnapshot(LevelDBDatabase* db)
  80     : db_(db->db_.get()), snapshot_(db_->GetSnapshot()) {}
  81
  82 LevelDBSnapshot::~LevelDBSnapshot() { db_->ReleaseSnapshot(snapshot_); }
  83
  84 LevelDBDatabase::LevelDBDatabase() {}
  85
  86 LevelDBDatabase::~LevelDBDatabase() {
  87   // db_'s destructor uses comparator_adapter_; order of deletion is important.
  88   db_.reset();
  89   comparator_adapter_.reset();
  90   env_.reset();
  91 }
  92
  93 static leveldb::Status OpenDB(leveldb::Comparator* comparator,
  94                               leveldb::Env* env,
  95                               const base::FilePath& path,
  96                               leveldb::DB** db) {
  97   leveldb::Options options;
  98   options.comparator = comparator;
  99   options.create_if_missing = true;
 100   options.paranoid_checks = true;
 101   options.compression = leveldb::kSnappyCompression;
 102
 103   // For info about the troubles we've run into with this parameter, see:
 104   // https://code.google.com/p/chromium/issues/detail?id=227313#c11
 105   options.max_open_files = 80;
 106   options.env = env;
 107
 108   // ChromiumEnv assumes UTF8, converts back to FilePath before using.
 109   return leveldb::DB::Open(options, path.AsUTF8Unsafe(), db);
 110 }
 111
 112 leveldb::Status LevelDBDatabase::Destroy(const base::FilePath& file_name) {
 113   leveldb::Options options;
 114   options.env = leveldb::IDBEnv();
 115   // ChromiumEnv assumes UTF8, converts back to FilePath before using.
 116   return leveldb::DestroyDB(file_name.AsUTF8Unsafe(), options);
 117 }
 118
 119 namespace {
 120 class LockImpl : public LevelDBLock {
 121  public:
 122   explicit LockImpl(leveldb::Env* env, leveldb::FileLock* lock)
 123       : env_(env), lock_(lock) {}
 124   virtual ~LockImpl() { env_->UnlockFile(lock_); }
 125  private:
 126   leveldb::Env* env_;
 127   leveldb::FileLock* lock_;
 128
 129   DISALLOW_COPY_AND_ASSIGN(LockImpl);
 130 };
 131 }  // namespace
 132
 133 scoped_ptr<LevelDBLock> LevelDBDatabase::LockForTesting(
 134     const base::FilePath& file_name) {
 135   leveldb::Env* env = leveldb::IDBEnv();
 136   base::FilePath lock_path = file_name.AppendASCII("LOCK");
 137   leveldb::FileLock* lock = NULL;
 138   leveldb::Status status = env->LockFile(lock_path.AsUTF8Unsafe(), &lock);
 139   if (!status.ok())
 140     return scoped_ptr<LevelDBLock>();
 141   DCHECK(lock);
 142   return scoped_ptr<LevelDBLock>(new LockImpl(env, lock));
 143 }
 144
 145 static int CheckFreeSpace(const char* const type,
 146                           const base::FilePath& file_name) {
 147   std::string name =
 148       std::string("WebCore.IndexedDB.LevelDB.Open") + type + "FreeDiskSpace";
 149   int64 free_disk_space_in_k_bytes =
 150       base::SysInfo::AmountOfFreeDiskSpace(file_name) / 1024;
 151   if (free_disk_space_in_k_bytes < 0) {
 152     base::Histogram::FactoryGet(
 153         "WebCore.IndexedDB.LevelDB.FreeDiskSpaceFailure",
 154         1,
 155         2 /*boundary*/,
 156         2 /*boundary*/ + 1,
 157         base::HistogramBase::kUmaTargetedHistogramFlag)->Add(1 /*sample*/);
 158     return -1;
 159   }
 160   int clamped_disk_space_k_bytes = free_disk_space_in_k_bytes > INT_MAX
 161                                        ? INT_MAX
 162                                        : free_disk_space_in_k_bytes;
 163   const uint64 histogram_max = static_cast<uint64>(1e9);
 164   COMPILE_ASSERT(histogram_max <= INT_MAX, histogram_max_too_big);
 165   base::Histogram::FactoryGet(name,
 166                               1,
 167                               histogram_max,
 168                               11 /*buckets*/,
 169                               base::HistogramBase::kUmaTargetedHistogramFlag)
 170       ->Add(clamped_disk_space_k_bytes);
 171   return clamped_disk_space_k_bytes;
 172 }
 173
 174 static void ParseAndHistogramIOErrorDetails(const std::string& histogram_name,
 175                                             const leveldb::Status& s) {
 176   leveldb_env::MethodID method;
 177   int error = -1;
 178   leveldb_env::ErrorParsingResult result =
 179       leveldb_env::ParseMethodAndError(s.ToString().c_str(), &method, &error);
 180   if (result == leveldb_env::NONE)
 181     return;
 182   std::string method_histogram_name(histogram_name);
 183   method_histogram_name.append(".EnvMethod");
 184   base::LinearHistogram::FactoryGet(
 185       method_histogram_name,
 186       1,
 187       leveldb_env::kNumEntries,
 188       leveldb_env::kNumEntries + 1,
 189       base::HistogramBase::kUmaTargetedHistogramFlag)->Add(method);
 190
 191   std::string error_histogram_name(histogram_name);
 192
 193   if (result == leveldb_env::METHOD_AND_PFE) {
 194     DCHECK_LT(error, 0);
 195     error_histogram_name.append(std::string(".PFE.") +
 196                                 leveldb_env::MethodIDToString(method));
 197     base::LinearHistogram::FactoryGet(
 198         error_histogram_name,
 199         1,
 200         -base::File::FILE_ERROR_MAX,
 201         -base::File::FILE_ERROR_MAX + 1,
 202         base::HistogramBase::kUmaTargetedHistogramFlag)->Add(-error);
 203   } else if (result == leveldb_env::METHOD_AND_ERRNO) {
 204     error_histogram_name.append(std::string(".Errno.") +
 205                                 leveldb_env::MethodIDToString(method));
 206     base::LinearHistogram::FactoryGet(
 207         error_histogram_name,
 208         1,
 209         ERANGE + 1,
 210         ERANGE + 2,
 211         base::HistogramBase::kUmaTargetedHistogramFlag)->Add(error);
 212   }
 213 }
 214
 215 static void ParseAndHistogramCorruptionDetails(
 216     const std::string& histogram_name,
 217     const leveldb::Status& status) {
 218   int error = leveldb_env::GetCorruptionCode(status);
 219   DCHECK_GE(error, 0);
 220   std::string corruption_histogram_name(histogram_name);
 221   corruption_histogram_name.append(".Corruption");
 222   const int kNumPatterns = leveldb_env::GetNumCorruptionCodes();
 223   base::LinearHistogram::FactoryGet(
 224       corruption_histogram_name,
 225       1,
 226       kNumPatterns,
 227       kNumPatterns + 1,
 228       base::HistogramBase::kUmaTargetedHistogramFlag)->Add(error);
 229 }
 230
 231 static void HistogramLevelDBError(const std::string& histogram_name,
 232                                   const leveldb::Status& s) {
 233   if (s.ok()) {
 234     NOTREACHED();
 235     return;
 236   }
 237   enum {
 238     LEVEL_DB_NOT_FOUND,
 239     LEVEL_DB_CORRUPTION,
 240     LEVEL_DB_IO_ERROR,
 241     LEVEL_DB_OTHER,
 242     LEVEL_DB_MAX_ERROR
 243   };
 244   int leveldb_error = LEVEL_DB_OTHER;
 245   if (s.IsNotFound())
 246     leveldb_error = LEVEL_DB_NOT_FOUND;
 247   else if (s.IsCorruption())
 248     leveldb_error = LEVEL_DB_CORRUPTION;
 249   else if (s.IsIOError())
 250     leveldb_error = LEVEL_DB_IO_ERROR;
 251   base::Histogram::FactoryGet(histogram_name,
 252                               1,
 253                               LEVEL_DB_MAX_ERROR,
 254                               LEVEL_DB_MAX_ERROR + 1,
 255                               base::HistogramBase::kUmaTargetedHistogramFlag)
 256       ->Add(leveldb_error);
 257   if (s.IsIOError())
 258     ParseAndHistogramIOErrorDetails(histogram_name, s);
 259   else
 260     ParseAndHistogramCorruptionDetails(histogram_name, s);
 261 }
 262
 263 leveldb::Status LevelDBDatabase::Open(const base::FilePath& file_name,
 264                                       const LevelDBComparator* comparator,
 265                                       scoped_ptr<LevelDBDatabase>* result,
 266                                       bool* is_disk_full) {
 267   base::TimeTicks begin_time = base::TimeTicks::Now();
 268
 269   scoped_ptr<ComparatorAdapter> comparator_adapter(
 270       new ComparatorAdapter(comparator));
 271
 272   leveldb::DB* db;
 273   const leveldb::Status s =
 274       OpenDB(comparator_adapter.get(), leveldb::IDBEnv(), file_name, &db);
 275
 276   if (!s.ok()) {
 277     HistogramLevelDBError("WebCore.IndexedDB.LevelDBOpenErrors", s);
 278     int free_space_k_bytes = CheckFreeSpace("Failure", file_name);
 279     // Disks with <100k of free space almost never succeed in opening a
 280     // leveldb database.
 281     if (is_disk_full)
 282       *is_disk_full = free_space_k_bytes >= 0 && free_space_k_bytes < 100;
 283
 284     LOG(ERROR) << "Failed to open LevelDB database from "
 285                << file_name.AsUTF8Unsafe() << "," << s.ToString();
 286     return s;
 287   }
 288
 289   UMA_HISTOGRAM_MEDIUM_TIMES("WebCore.IndexedDB.LevelDB.OpenTime",
 290                              base::TimeTicks::Now() - begin_time);
 291
 292   CheckFreeSpace("Success", file_name);
 293
 294   (*result).reset(new LevelDBDatabase);
 295   (*result)->db_ = make_scoped_ptr(db);
 296   (*result)->comparator_adapter_ = comparator_adapter.Pass();
 297   (*result)->comparator_ = comparator;
 298
 299   return s;
 300 }
 301
 302 scoped_ptr<LevelDBDatabase> LevelDBDatabase::OpenInMemory(
 303     const LevelDBComparator* comparator) {
 304   scoped_ptr<ComparatorAdapter> comparator_adapter(
 305       new ComparatorAdapter(comparator));
 306   scoped_ptr<leveldb::Env> in_memory_env(leveldb::NewMemEnv(leveldb::IDBEnv()));
 307
 308   leveldb::DB* db;
 309   const leveldb::Status s = OpenDB(
 310       comparator_adapter.get(), in_memory_env.get(), base::FilePath(), &db);
 311
 312   if (!s.ok()) {
 313     LOG(ERROR) << "Failed to open in-memory LevelDB database: " << s.ToString();
 314     return scoped_ptr<LevelDBDatabase>();
 315   }
 316
 317   scoped_ptr<LevelDBDatabase> result(new LevelDBDatabase);
 318   result->env_ = in_memory_env.Pass();
 319   result->db_ = make_scoped_ptr(db);
 320   result->comparator_adapter_ = comparator_adapter.Pass();
 321   result->comparator_ = comparator;
 322
 323   return result.Pass();
 324 }
 325
 326 leveldb::Status LevelDBDatabase::Put(const StringPiece& key,
 327                                      std::string* value) {
 328   base::TimeTicks begin_time = base::TimeTicks::Now();
 329
 330   leveldb::WriteOptions write_options;
 331   write_options.sync = kSyncWrites;
 332
 333   const leveldb::Status s =
 334       db_->Put(write_options, MakeSlice(key), MakeSlice(*value));
 335   if (!s.ok())
 336     LOG(ERROR) << "LevelDB put failed: " << s.ToString();
 337   else
 338     UMA_HISTOGRAM_TIMES("WebCore.IndexedDB.LevelDB.PutTime",
 339                         base::TimeTicks::Now() - begin_time);
 340   return s;
 341 }
 342
 343 leveldb::Status LevelDBDatabase::Remove(const StringPiece& key) {
 344   leveldb::WriteOptions write_options;
 345   write_options.sync = kSyncWrites;
 346
 347   const leveldb::Status s = db_->Delete(write_options, MakeSlice(key));
 348   if (!s.IsNotFound())
 349     LOG(ERROR) << "LevelDB remove failed: " << s.ToString();
 350   return s;
 351 }
 352
 353 leveldb::Status LevelDBDatabase::Get(const StringPiece& key,
 354                                      std::string* value,
 355                                      bool* found,
 356                                      const LevelDBSnapshot* snapshot) {
 357   *found = false;
 358   leveldb::ReadOptions read_options;
 359   read_options.verify_checksums = true;  // TODO(jsbell): Disable this if the
 360                                          // performance impact is too great.
 361   read_options.snapshot = snapshot ? snapshot->snapshot_ : 0;
 362
 363   const leveldb::Status s = db_->Get(read_options, MakeSlice(key), value);
 364   if (s.ok()) {
 365     *found = true;
 366     return s;
 367   }
 368   if (s.IsNotFound())
 369     return leveldb::Status::OK();
 370   HistogramLevelDBError("WebCore.IndexedDB.LevelDBReadErrors", s);
 371   LOG(ERROR) << "LevelDB get failed: " << s.ToString();
 372   return s;
 373 }
 374
 375 leveldb::Status LevelDBDatabase::Write(const LevelDBWriteBatch& write_batch) {
 376   base::TimeTicks begin_time = base::TimeTicks::Now();
 377   leveldb::WriteOptions write_options;
 378   write_options.sync = kSyncWrites;
 379
 380   const leveldb::Status s =
 381       db_->Write(write_options, write_batch.write_batch_.get());
 382   if (!s.ok()) {
 383     HistogramLevelDBError("WebCore.IndexedDB.LevelDBWriteErrors", s);
 384     LOG(ERROR) << "LevelDB write failed: " << s.ToString();
 385   } else {
 386     UMA_HISTOGRAM_TIMES("WebCore.IndexedDB.LevelDB.WriteTime",
 387                         base::TimeTicks::Now() - begin_time);
 388   }
 389   return s;
 390 }
 391
 392 scoped_ptr<LevelDBIterator> LevelDBDatabase::CreateIterator(
 393     const LevelDBSnapshot* snapshot) {
 394   leveldb::ReadOptions read_options;
 395   read_options.verify_checksums = true;  // TODO(jsbell): Disable this if the
 396                                          // performance impact is too great.
 397   read_options.snapshot = snapshot ? snapshot->snapshot_ : 0;
 398
 399   scoped_ptr<leveldb::Iterator> i(db_->NewIterator(read_options));
 400   return scoped_ptr<LevelDBIterator>(
 401       IndexedDBClassFactory::Get()->CreateIteratorImpl(i.Pass()));
 402 }
 403
 404 const LevelDBComparator* LevelDBDatabase::Comparator() const {
 405   return comparator_;
 406 }
 407
 408 void LevelDBDatabase::Compact(const base::StringPiece& start,
 409                               const base::StringPiece& stop) {
 410   const leveldb::Slice start_slice = MakeSlice(start);
 411   const leveldb::Slice stop_slice = MakeSlice(stop);
 412   // NULL batch means just wait for earlier writes to be done
 413   db_->Write(leveldb::WriteOptions(), NULL);
 414   db_->CompactRange(&start_slice, &stop_slice);
 415 }
 416
 417 void LevelDBDatabase::CompactAll() { db_->CompactRange(NULL, NULL); }
 418
 419 }  // namespace content