content/browser/indexed_db/leveldb/leveldb_database.cc

   1 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "content/browser/indexed_db/leveldb/leveldb_database.h"
   6
   7 #include <cerrno>
   8 #include <string>
   9
  10 #include "base/basictypes.h"
  11 #include "base/logging.h"
  12 #include "base/memory/scoped_ptr.h"
  13 #include "base/metrics/histogram.h"
  14 #include "base/strings/string16.h"
  15 #include "base/strings/stringprintf.h"
  16 #include "base/strings/utf_string_conversions.h"
  17 #include "base/sys_info.h"
  18 #include "content/browser/indexed_db/leveldb/leveldb_comparator.h"
  19 #include "content/browser/indexed_db/leveldb/leveldb_iterator.h"
  20 #include "content/browser/indexed_db/leveldb/leveldb_slice.h"
  21 #include "content/browser/indexed_db/leveldb/leveldb_write_batch.h"
  22 #include "third_party/leveldatabase/env_chromium.h"
  23 #include "third_party/leveldatabase/env_idb.h"
  24 #include "third_party/leveldatabase/src/helpers/memenv/memenv.h"
  25 #include "third_party/leveldatabase/src/include/leveldb/comparator.h"
  26 #include "third_party/leveldatabase/src/include/leveldb/db.h"
  27 #include "third_party/leveldatabase/src/include/leveldb/env.h"
  28 #include "third_party/leveldatabase/src/include/leveldb/slice.h"
  29
  30 namespace content {
  31
  32 static leveldb::Slice MakeSlice(const std::vector<char>& value) {
  33   DCHECK_GT(value.size(), static_cast<size_t>(0));
  34   return leveldb::Slice(&*value.begin(), value.size());
  35 }
  36
  37 static leveldb::Slice MakeSlice(const LevelDBSlice& s) {
  38   return leveldb::Slice(s.begin(), s.end() - s.begin());
  39 }
  40
  41 static LevelDBSlice MakeLevelDBSlice(const leveldb::Slice& s) {
  42   return LevelDBSlice(s.data(), s.data() + s.size());
  43 }
  44
  45 class ComparatorAdapter : public leveldb::Comparator {
  46  public:
  47   explicit ComparatorAdapter(const LevelDBComparator* comparator)
  48       : comparator_(comparator) {}
  49
  50   virtual int Compare(const leveldb::Slice& a, const leveldb::Slice& b) const
  51       OVERRIDE {
  52     return comparator_->Compare(MakeLevelDBSlice(a), MakeLevelDBSlice(b));
  53   }
  54
  55   virtual const char* Name() const OVERRIDE { return comparator_->Name(); }
  56
  57   // TODO(jsbell): Support the methods below in the future.
  58   virtual void FindShortestSeparator(std::string* start,
  59                                      const leveldb::Slice& limit) const
  60       OVERRIDE {}
  61   virtual void FindShortSuccessor(std::string* key) const OVERRIDE {}
  62
  63  private:
  64   const LevelDBComparator* comparator_;
  65 };
  66
  67 LevelDBSnapshot::LevelDBSnapshot(LevelDBDatabase* db)
  68     : db_(db->db_.get()), snapshot_(db_->GetSnapshot()) {}
  69
  70 LevelDBSnapshot::~LevelDBSnapshot() { db_->ReleaseSnapshot(snapshot_); }
  71
  72 LevelDBDatabase::LevelDBDatabase() {}
  73
  74 LevelDBDatabase::~LevelDBDatabase() {
  75   // db_'s destructor uses comparator_adapter_; order of deletion is important.
  76   db_.reset();
  77   comparator_adapter_.reset();
  78   env_.reset();
  79 }
  80
  81 static leveldb::Status OpenDB(leveldb::Comparator* comparator,
  82                               leveldb::Env* env,
  83                               const base::FilePath& path,
  84                               leveldb::DB** db) {
  85   leveldb::Options options;
  86   options.comparator = comparator;
  87   options.create_if_missing = true;
  88   options.paranoid_checks = true;
  89
  90   // Marking compression as explicitly off so snappy support can be
  91   // compiled in for other leveldb clients without implicitly enabling
  92   // it for IndexedDB. http://crbug.com/81384
  93   options.compression = leveldb::kNoCompression;
  94
  95   // For info about the troubles we've run into with this parameter, see:
  96   // https://code.google.com/p/chromium/issues/detail?id=227313#c11
  97   options.max_open_files = 80;
  98   options.env = env;
  99
 100   // ChromiumEnv assumes UTF8, converts back to FilePath before using.
 101   return leveldb::DB::Open(options, path.AsUTF8Unsafe(), db);
 102 }
 103
 104 bool LevelDBDatabase::Destroy(const base::FilePath& file_name) {
 105   leveldb::Options options;
 106   options.env = leveldb::IDBEnv();
 107   // ChromiumEnv assumes UTF8, converts back to FilePath before using.
 108   const leveldb::Status s =
 109       leveldb::DestroyDB(file_name.AsUTF8Unsafe(), options);
 110   return s.ok();
 111 }
 112
 113 static int CheckFreeSpace(const char* const type,
 114                           const base::FilePath& file_name) {
 115   std::string name =
 116       std::string("WebCore.IndexedDB.LevelDB.Open") + type + "FreeDiskSpace";
 117   int64 free_disk_space_in_k_bytes =
 118       base::SysInfo::AmountOfFreeDiskSpace(file_name) / 1024;
 119   if (free_disk_space_in_k_bytes < 0) {
 120     base::Histogram::FactoryGet(
 121         "WebCore.IndexedDB.LevelDB.FreeDiskSpaceFailure",
 122         1,
 123         2 /*boundary*/,
 124         2 /*boundary*/ + 1,
 125         base::HistogramBase::kUmaTargetedHistogramFlag)->Add(1 /*sample*/);
 126     return -1;
 127   }
 128   int clamped_disk_space_k_bytes =
 129       free_disk_space_in_k_bytes > INT_MAX ? INT_MAX
 130                                            : free_disk_space_in_k_bytes;
 131   const uint64 histogram_max = static_cast<uint64>(1e9);
 132   COMPILE_ASSERT(histogram_max <= INT_MAX, histogram_max_too_big);
 133   base::Histogram::FactoryGet(name,
 134                               1,
 135                               histogram_max,
 136                               11 /*buckets*/,
 137                               base::HistogramBase::kUmaTargetedHistogramFlag)
 138       ->Add(clamped_disk_space_k_bytes);
 139   return clamped_disk_space_k_bytes;
 140 }
 141
 142 static void ParseAndHistogramIOErrorDetails(const std::string& histogram_name,
 143                                             const leveldb::Status& s) {
 144   leveldb_env::MethodID method;
 145   int error = -1;
 146   leveldb_env::ErrorParsingResult result =
 147       leveldb_env::ParseMethodAndError(s.ToString().c_str(), &method, &error);
 148   if (result == leveldb_env::NONE)
 149     return;
 150   std::string method_histogram_name(histogram_name);
 151   method_histogram_name.append(".EnvMethod");
 152   base::LinearHistogram::FactoryGet(
 153       method_histogram_name,
 154       1,
 155       leveldb_env::kNumEntries,
 156       leveldb_env::kNumEntries + 1,
 157       base::HistogramBase::kUmaTargetedHistogramFlag)->Add(method);
 158
 159   std::string error_histogram_name(histogram_name);
 160
 161   if (result == leveldb_env::METHOD_AND_PFE) {
 162     DCHECK(error < 0);
 163     error_histogram_name.append(std::string(".PFE.") +
 164                                 leveldb_env::MethodIDToString(method));
 165     base::LinearHistogram::FactoryGet(
 166         error_histogram_name,
 167         1,
 168         -base::PLATFORM_FILE_ERROR_MAX,
 169         -base::PLATFORM_FILE_ERROR_MAX + 1,
 170         base::HistogramBase::kUmaTargetedHistogramFlag)->Add(-error);
 171   } else if (result == leveldb_env::METHOD_AND_ERRNO) {
 172     error_histogram_name.append(std::string(".Errno.") +
 173                                 leveldb_env::MethodIDToString(method));
 174     base::LinearHistogram::FactoryGet(
 175         error_histogram_name,
 176         1,
 177         ERANGE + 1,
 178         ERANGE + 2,
 179         base::HistogramBase::kUmaTargetedHistogramFlag)->Add(error);
 180   }
 181 }
 182
 183 static void ParseAndHistogramCorruptionDetails(
 184     const std::string& histogram_name,
 185     const leveldb::Status& status) {
 186   DCHECK(!status.IsIOError());
 187   DCHECK(!status.ok());
 188   const int kOtherError = 0;
 189   int error = kOtherError;
 190   const std::string& str_error = status.ToString();
 191   // Keep in sync with LevelDBCorruptionTypes in histograms.xml.
 192   const char* patterns[] = {
 193     "missing files",
 194     "log record too small",
 195     "corrupted internal key",
 196     "partial record",
 197     "missing start of fragmented record",
 198     "error in middle of record",
 199     "unknown record type",
 200     "truncated record at end",
 201     "bad record length",
 202     "VersionEdit",
 203     "FileReader invoked with unexpected value",
 204     "corrupted key",
 205     "CURRENT file does not end with newline",
 206     "no meta-nextfile entry",
 207     "no meta-lognumber entry",
 208     "no last-sequence-number entry",
 209     "malformed WriteBatch",
 210     "bad WriteBatch Put",
 211     "bad WriteBatch Delete",
 212     "unknown WriteBatch tag",
 213     "WriteBatch has wrong count",
 214     "bad entry in block",
 215     "bad block contents",
 216     "bad block handle",
 217     "truncated block read",
 218     "block checksum mismatch",
 219     "checksum mismatch",
 220     "corrupted compressed block contents",
 221     "bad block type",
 222     "bad magic number",
 223     "file is too short",
 224   };
 225   const size_t kNumPatterns = arraysize(patterns);
 226   for (size_t i = 0; i < kNumPatterns; ++i) {
 227     if (str_error.find(patterns[i]) != std::string::npos) {
 228       error = i + 1;
 229       break;
 230     }
 231   }
 232   DCHECK(error >= 0);
 233   std::string corruption_histogram_name(histogram_name);
 234   corruption_histogram_name.append(".Corruption");
 235   base::LinearHistogram::FactoryGet(
 236       corruption_histogram_name,
 237       1,
 238       kNumPatterns + 1,
 239       kNumPatterns + 2,
 240       base::HistogramBase::kUmaTargetedHistogramFlag)->Add(error);
 241 }
 242
 243 static void HistogramLevelDBError(const std::string& histogram_name,
 244                                   const leveldb::Status& s) {
 245   if (s.ok()) {
 246     NOTREACHED();
 247     return;
 248   }
 249   enum {
 250     LEVEL_DB_NOT_FOUND,
 251     LEVEL_DB_CORRUPTION,
 252     LEVEL_DB_IO_ERROR,
 253     LEVEL_DB_OTHER,
 254     LEVEL_DB_MAX_ERROR
 255   };
 256   int leveldb_error = LEVEL_DB_OTHER;
 257   if (s.IsNotFound())
 258     leveldb_error = LEVEL_DB_NOT_FOUND;
 259   else if (s.IsCorruption())
 260     leveldb_error = LEVEL_DB_CORRUPTION;
 261   else if (s.IsIOError())
 262     leveldb_error = LEVEL_DB_IO_ERROR;
 263   base::Histogram::FactoryGet(histogram_name,
 264                               1,
 265                               LEVEL_DB_MAX_ERROR,
 266                               LEVEL_DB_MAX_ERROR + 1,
 267                               base::HistogramBase::kUmaTargetedHistogramFlag)
 268       ->Add(leveldb_error);
 269   if (s.IsIOError())
 270     ParseAndHistogramIOErrorDetails(histogram_name, s);
 271   else
 272     ParseAndHistogramCorruptionDetails(histogram_name, s);
 273 }
 274
 275 scoped_ptr<LevelDBDatabase> LevelDBDatabase::Open(
 276     const base::FilePath& file_name,
 277     const LevelDBComparator* comparator,
 278     bool* is_disk_full) {
 279   scoped_ptr<ComparatorAdapter> comparator_adapter(
 280       new ComparatorAdapter(comparator));
 281
 282   leveldb::DB* db;
 283   const leveldb::Status s =
 284       OpenDB(comparator_adapter.get(), leveldb::IDBEnv(), file_name, &db);
 285
 286   if (!s.ok()) {
 287     HistogramLevelDBError("WebCore.IndexedDB.LevelDBOpenErrors", s);
 288     int free_space_k_bytes = CheckFreeSpace("Failure", file_name);
 289     // Disks with <100k of free space almost never succeed in opening a
 290     // leveldb database.
 291     if (is_disk_full)
 292       *is_disk_full = free_space_k_bytes >= 0 && free_space_k_bytes < 100;
 293
 294     LOG(ERROR) << "Failed to open LevelDB database from "
 295                << file_name.AsUTF8Unsafe() << "," << s.ToString();
 296     return scoped_ptr<LevelDBDatabase>();
 297   }
 298
 299   CheckFreeSpace("Success", file_name);
 300
 301   scoped_ptr<LevelDBDatabase> result(new LevelDBDatabase);
 302   result->db_ = make_scoped_ptr(db);
 303   result->comparator_adapter_ = comparator_adapter.Pass();
 304   result->comparator_ = comparator;
 305
 306   return result.Pass();
 307 }
 308
 309 scoped_ptr<LevelDBDatabase> LevelDBDatabase::OpenInMemory(
 310     const LevelDBComparator* comparator) {
 311   scoped_ptr<ComparatorAdapter> comparator_adapter(
 312       new ComparatorAdapter(comparator));
 313   scoped_ptr<leveldb::Env> in_memory_env(leveldb::NewMemEnv(leveldb::IDBEnv()));
 314
 315   leveldb::DB* db;
 316   const leveldb::Status s = OpenDB(
 317       comparator_adapter.get(), in_memory_env.get(), base::FilePath(), &db);
 318
 319   if (!s.ok()) {
 320     LOG(ERROR) << "Failed to open in-memory LevelDB database: " << s.ToString();
 321     return scoped_ptr<LevelDBDatabase>();
 322   }
 323
 324   scoped_ptr<LevelDBDatabase> result(new LevelDBDatabase);
 325   result->env_ = in_memory_env.Pass();
 326   result->db_ = make_scoped_ptr(db);
 327   result->comparator_adapter_ = comparator_adapter.Pass();
 328   result->comparator_ = comparator;
 329
 330   return result.Pass();
 331 }
 332
 333 bool LevelDBDatabase::Put(const LevelDBSlice& key,
 334                           std::vector<char>* value) {
 335   leveldb::WriteOptions write_options;
 336   write_options.sync = true;
 337
 338   const leveldb::Status s =
 339       db_->Put(write_options, MakeSlice(key), MakeSlice(*value));
 340   if (s.ok())
 341     return true;
 342   LOG(ERROR) << "LevelDB put failed: " << s.ToString();
 343   return false;
 344 }
 345
 346 bool LevelDBDatabase::Remove(const LevelDBSlice& key) {
 347   leveldb::WriteOptions write_options;
 348   write_options.sync = true;
 349
 350   const leveldb::Status s = db_->Delete(write_options, MakeSlice(key));
 351   if (s.ok())
 352     return true;
 353   if (s.IsNotFound())
 354     return false;
 355   LOG(ERROR) << "LevelDB remove failed: " << s.ToString();
 356   return false;
 357 }
 358
 359 bool LevelDBDatabase::Get(const LevelDBSlice& key,
 360                           std::string* value,
 361                           bool* found,
 362                           const LevelDBSnapshot* snapshot) {
 363   *found = false;
 364   leveldb::ReadOptions read_options;
 365   read_options.verify_checksums = true;  // TODO(jsbell): Disable this if the
 366                                          // performance impact is too great.
 367   read_options.snapshot = snapshot ? snapshot->snapshot_ : 0;
 368
 369   const leveldb::Status s = db_->Get(read_options, MakeSlice(key), value);
 370   if (s.ok()) {
 371     *found = true;
 372     return true;
 373   }
 374   if (s.IsNotFound())
 375     return true;
 376   LOG(ERROR) << "LevelDB get failed: " << s.ToString();
 377   return false;
 378 }
 379
 380 bool LevelDBDatabase::Write(const LevelDBWriteBatch& write_batch) {
 381   leveldb::WriteOptions write_options;
 382   write_options.sync = true;
 383
 384   const leveldb::Status s =
 385       db_->Write(write_options, write_batch.write_batch_.get());
 386   if (s.ok())
 387     return true;
 388   HistogramLevelDBError("WebCore.IndexedDB.LevelDBWriteErrors", s);
 389   LOG(ERROR) << "LevelDB write failed: " << s.ToString();
 390   return false;
 391 }
 392
 393 namespace {
 394 class IteratorImpl : public LevelDBIterator {
 395  public:
 396   virtual ~IteratorImpl() {}
 397
 398   virtual bool IsValid() const OVERRIDE;
 399   virtual void SeekToLast() OVERRIDE;
 400   virtual void Seek(const LevelDBSlice& target) OVERRIDE;
 401   virtual void Next() OVERRIDE;
 402   virtual void Prev() OVERRIDE;
 403   virtual LevelDBSlice Key() const OVERRIDE;
 404   virtual LevelDBSlice Value() const OVERRIDE;
 405
 406  private:
 407   friend class content::LevelDBDatabase;
 408   explicit IteratorImpl(scoped_ptr<leveldb::Iterator> iterator);
 409   void CheckStatus();
 410
 411   scoped_ptr<leveldb::Iterator> iterator_;
 412 };
 413 }
 414
 415 IteratorImpl::IteratorImpl(scoped_ptr<leveldb::Iterator> it)
 416     : iterator_(it.Pass()) {}
 417
 418 void IteratorImpl::CheckStatus() {
 419   const leveldb::Status s = iterator_->status();
 420   if (!s.ok())
 421     LOG(ERROR) << "LevelDB iterator error: " << s.ToString();
 422 }
 423
 424 bool IteratorImpl::IsValid() const { return iterator_->Valid(); }
 425
 426 void IteratorImpl::SeekToLast() {
 427   iterator_->SeekToLast();
 428   CheckStatus();
 429 }
 430
 431 void IteratorImpl::Seek(const LevelDBSlice& target) {
 432   iterator_->Seek(MakeSlice(target));
 433   CheckStatus();
 434 }
 435
 436 void IteratorImpl::Next() {
 437   DCHECK(IsValid());
 438   iterator_->Next();
 439   CheckStatus();
 440 }
 441
 442 void IteratorImpl::Prev() {
 443   DCHECK(IsValid());
 444   iterator_->Prev();
 445   CheckStatus();
 446 }
 447
 448 LevelDBSlice IteratorImpl::Key() const {
 449   DCHECK(IsValid());
 450   return MakeLevelDBSlice(iterator_->key());
 451 }
 452
 453 LevelDBSlice IteratorImpl::Value() const {
 454   DCHECK(IsValid());
 455   return MakeLevelDBSlice(iterator_->value());
 456 }
 457
 458 scoped_ptr<LevelDBIterator> LevelDBDatabase::CreateIterator(
 459     const LevelDBSnapshot* snapshot) {
 460   leveldb::ReadOptions read_options;
 461   read_options.verify_checksums = true;  // TODO(jsbell): Disable this if the
 462                                          // performance impact is too great.
 463   read_options.snapshot = snapshot ? snapshot->snapshot_ : 0;
 464
 465   scoped_ptr<leveldb::Iterator> i(db_->NewIterator(read_options));
 466   return scoped_ptr<LevelDBIterator>(new IteratorImpl(i.Pass()));
 467 }
 468
 469 const LevelDBComparator* LevelDBDatabase::Comparator() const {
 470   return comparator_;
 471 }
 472
 473 }  // namespace content