third_party/leveldatabase/env_chromium.cc

   1 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file. See the AUTHORS file for names of contributors.
   4
   5 #include <errno.h>
   6 #include <stdio.h>
   7
   8 #include <deque>
   9
  10 #include "base/at_exit.h"
  11 #include "base/debug/trace_event.h"
  12 #include "base/file_util.h"
  13 #include "base/files/file_enumerator.h"
  14 #include "base/files/file_path.h"
  15 #include "base/lazy_instance.h"
  16 #include "base/memory/ref_counted.h"
  17 #include "base/message_loop/message_loop.h"
  18 #include "base/metrics/histogram.h"
  19 #include "base/platform_file.h"
  20 #include "base/posix/eintr_wrapper.h"
  21 #include "base/strings/utf_string_conversions.h"
  22 #include "base/synchronization/lock.h"
  23 #include "base/sys_info.h"
  24 #include "base/threading/platform_thread.h"
  25 #include "base/threading/thread.h"
  26 #include "chromium_logger.h"
  27 #include "env_chromium.h"
  28 #include "leveldb/env.h"
  29 #include "leveldb/slice.h"
  30 #include "port/port.h"
  31 #include "third_party/re2/re2/re2.h"
  32 #include "util/logging.h"
  33
  34 #if defined(OS_WIN)
  35 #include <io.h>
  36 #include "base/win/win_util.h"
  37 #endif
  38
  39 #if defined(OS_POSIX)
  40 #include <dirent.h>
  41 #include <fcntl.h>
  42 #include <sys/resource.h>
  43 #include <sys/time.h>
  44 #endif
  45
  46 using namespace leveldb;
  47
  48 namespace leveldb_env {
  49
  50 namespace {
  51
  52 const base::FilePath::CharType backup_table_extension[] =
  53     FILE_PATH_LITERAL(".bak");
  54 const base::FilePath::CharType table_extension[] = FILE_PATH_LITERAL(".ldb");
  55
  56 #if (defined(OS_POSIX) && !defined(OS_LINUX)) || defined(OS_WIN)
  57 // The following are glibc-specific
  58
  59 size_t fread_unlocked(void *ptr, size_t size, size_t n, FILE *file) {
  60   return fread(ptr, size, n, file);
  61 }
  62
  63 size_t fwrite_unlocked(const void *ptr, size_t size, size_t n, FILE *file) {
  64   return fwrite(ptr, size, n, file);
  65 }
  66
  67 int fflush_unlocked(FILE *file) {
  68   return fflush(file);
  69 }
  70
  71 #if !defined(OS_ANDROID)
  72 int fdatasync(int fildes) {
  73 #if defined(OS_WIN)
  74   return _commit(fildes);
  75 #else
  76   return HANDLE_EINTR(fsync(fildes));
  77 #endif
  78 }
  79 #endif
  80
  81 #endif
  82
  83 // Wide-char safe fopen wrapper.
  84 FILE* fopen_internal(const char* fname, const char* mode) {
  85 #if defined(OS_WIN)
  86   return _wfopen(UTF8ToUTF16(fname).c_str(), ASCIIToUTF16(mode).c_str());
  87 #else
  88   return fopen(fname, mode);
  89 #endif
  90 }
  91
  92 base::FilePath CreateFilePath(const std::string& file_path) {
  93 #if defined(OS_WIN)
  94   return base::FilePath(UTF8ToUTF16(file_path));
  95 #else
  96   return base::FilePath(file_path);
  97 #endif
  98 }
  99
 100 static const base::FilePath::CharType kLevelDBTestDirectoryPrefix[]
 101     = FILE_PATH_LITERAL("leveldb-test-");
 102
 103 const char* PlatformFileErrorString(const ::base::PlatformFileError& error) {
 104   switch (error) {
 105     case ::base::PLATFORM_FILE_ERROR_FAILED:
 106       return "No further details.";
 107     case ::base::PLATFORM_FILE_ERROR_IN_USE:
 108       return "File currently in use.";
 109     case ::base::PLATFORM_FILE_ERROR_EXISTS:
 110       return "File already exists.";
 111     case ::base::PLATFORM_FILE_ERROR_NOT_FOUND:
 112       return "File not found.";
 113     case ::base::PLATFORM_FILE_ERROR_ACCESS_DENIED:
 114       return "Access denied.";
 115     case ::base::PLATFORM_FILE_ERROR_TOO_MANY_OPENED:
 116       return "Too many files open.";
 117     case ::base::PLATFORM_FILE_ERROR_NO_MEMORY:
 118       return "Out of memory.";
 119     case ::base::PLATFORM_FILE_ERROR_NO_SPACE:
 120       return "No space left on drive.";
 121     case ::base::PLATFORM_FILE_ERROR_NOT_A_DIRECTORY:
 122       return "Not a directory.";
 123     case ::base::PLATFORM_FILE_ERROR_INVALID_OPERATION:
 124       return "Invalid operation.";
 125     case ::base::PLATFORM_FILE_ERROR_SECURITY:
 126       return "Security error.";
 127     case ::base::PLATFORM_FILE_ERROR_ABORT:
 128       return "File operation aborted.";
 129     case ::base::PLATFORM_FILE_ERROR_NOT_A_FILE:
 130       return "The supplied path was not a file.";
 131     case ::base::PLATFORM_FILE_ERROR_NOT_EMPTY:
 132       return "The file was not empty.";
 133     case ::base::PLATFORM_FILE_ERROR_INVALID_URL:
 134       return "Invalid URL.";
 135     case ::base::PLATFORM_FILE_ERROR_IO:
 136       return "OS or hardware error.";
 137     case ::base::PLATFORM_FILE_OK:
 138       return "OK.";
 139     case ::base::PLATFORM_FILE_ERROR_MAX:
 140       NOTREACHED();
 141   }
 142   NOTIMPLEMENTED();
 143   return "Unknown error.";
 144 }
 145
 146 class ChromiumSequentialFile: public SequentialFile {
 147  private:
 148   std::string filename_;
 149   FILE* file_;
 150   const UMALogger* uma_logger_;
 151
 152  public:
 153   ChromiumSequentialFile(const std::string& fname, FILE* f,
 154                          const UMALogger* uma_logger)
 155       : filename_(fname), file_(f), uma_logger_(uma_logger) { }
 156   virtual ~ChromiumSequentialFile() { fclose(file_); }
 157
 158   virtual Status Read(size_t n, Slice* result, char* scratch) {
 159     Status s;
 160     size_t r = fread_unlocked(scratch, 1, n, file_);
 161     *result = Slice(scratch, r);
 162     if (r < n) {
 163       if (feof(file_)) {
 164         // We leave status as ok if we hit the end of the file
 165       } else {
 166         // A partial read with an error: return a non-ok status
 167         s = MakeIOError(filename_, strerror(errno), kSequentialFileRead, errno);
 168         uma_logger_->RecordErrorAt(kSequentialFileRead);
 169       }
 170     }
 171     return s;
 172   }
 173
 174   virtual Status Skip(uint64_t n) {
 175     if (fseek(file_, n, SEEK_CUR)) {
 176       int saved_errno = errno;
 177       uma_logger_->RecordErrorAt(kSequentialFileSkip);
 178       return MakeIOError(
 179           filename_, strerror(saved_errno), kSequentialFileSkip, saved_errno);
 180     }
 181     return Status::OK();
 182   }
 183 };
 184
 185 class ChromiumRandomAccessFile: public RandomAccessFile {
 186  private:
 187   std::string filename_;
 188   ::base::PlatformFile file_;
 189   const UMALogger* uma_logger_;
 190
 191  public:
 192   ChromiumRandomAccessFile(const std::string& fname, ::base::PlatformFile file,
 193                            const UMALogger* uma_logger)
 194       : filename_(fname), file_(file), uma_logger_(uma_logger) { }
 195   virtual ~ChromiumRandomAccessFile() { ::base::ClosePlatformFile(file_); }
 196
 197   virtual Status Read(uint64_t offset, size_t n, Slice* result,
 198                       char* scratch) const {
 199     Status s;
 200     int r = ::base::ReadPlatformFile(file_, offset, scratch, n);
 201     *result = Slice(scratch, (r < 0) ? 0 : r);
 202     if (r < 0) {
 203       // An error: return a non-ok status
 204       s = MakeIOError(
 205           filename_, "Could not perform read", kRandomAccessFileRead);
 206       uma_logger_->RecordErrorAt(kRandomAccessFileRead);
 207     }
 208     return s;
 209   }
 210 };
 211
 212 class ChromiumFileLock : public FileLock {
 213  public:
 214   ::base::PlatformFile file_;
 215   std::string name_;
 216 };
 217
 218 class Retrier {
 219  public:
 220   Retrier(MethodID method, RetrierProvider* provider)
 221       : start_(base::TimeTicks::Now()),
 222         limit_(start_ + base::TimeDelta::FromMilliseconds(
 223                             provider->MaxRetryTimeMillis())),
 224         last_(start_),
 225         time_to_sleep_(base::TimeDelta::FromMilliseconds(10)),
 226         success_(true),
 227         method_(method),
 228         last_error_(base::PLATFORM_FILE_OK),
 229         provider_(provider) {}
 230   ~Retrier() {
 231     if (success_) {
 232       provider_->GetRetryTimeHistogram(method_)->AddTime(last_ - start_);
 233       if (last_error_ != base::PLATFORM_FILE_OK) {
 234         DCHECK(last_error_ < 0);
 235         provider_->GetRecoveredFromErrorHistogram(method_)->Add(-last_error_);
 236       }
 237     }
 238   }
 239   bool ShouldKeepTrying(base::PlatformFileError last_error) {
 240     DCHECK_NE(last_error, base::PLATFORM_FILE_OK);
 241     last_error_ = last_error;
 242     if (last_ < limit_) {
 243       base::PlatformThread::Sleep(time_to_sleep_);
 244       last_ = base::TimeTicks::Now();
 245       return true;
 246     }
 247     success_ = false;
 248     return false;
 249   }
 250
 251  private:
 252   base::TimeTicks start_;
 253   base::TimeTicks limit_;
 254   base::TimeTicks last_;
 255   base::TimeDelta time_to_sleep_;
 256   bool success_;
 257   MethodID method_;
 258   base::PlatformFileError last_error_;
 259   RetrierProvider* provider_;
 260 };
 261
 262 class IDBEnv : public ChromiumEnv {
 263  public:
 264   IDBEnv() : ChromiumEnv() {
 265     name_ = "LevelDBEnv.IDB";
 266     make_backup_ = true;
 267   }
 268 };
 269
 270 ::base::LazyInstance<IDBEnv>::Leaky idb_env = LAZY_INSTANCE_INITIALIZER;
 271
 272 ::base::LazyInstance<ChromiumEnv>::Leaky default_env =
 273     LAZY_INSTANCE_INITIALIZER;
 274
 275 }  // unnamed namespace
 276
 277 const char* MethodIDToString(MethodID method) {
 278   switch (method) {
 279     case kSequentialFileRead:
 280       return "SequentialFileRead";
 281     case kSequentialFileSkip:
 282       return "SequentialFileSkip";
 283     case kRandomAccessFileRead:
 284       return "RandomAccessFileRead";
 285     case kWritableFileAppend:
 286       return "WritableFileAppend";
 287     case kWritableFileClose:
 288       return "WritableFileClose";
 289     case kWritableFileFlush:
 290       return "WritableFileFlush";
 291     case kWritableFileSync:
 292       return "WritableFileSync";
 293     case kNewSequentialFile:
 294       return "NewSequentialFile";
 295     case kNewRandomAccessFile:
 296       return "NewRandomAccessFile";
 297     case kNewWritableFile:
 298       return "NewWritableFile";
 299     case kDeleteFile:
 300       return "DeleteFile";
 301     case kCreateDir:
 302       return "CreateDir";
 303     case kDeleteDir:
 304       return "DeleteDir";
 305     case kGetFileSize:
 306       return "GetFileSize";
 307     case kRenameFile:
 308       return "RenameFile";
 309     case kLockFile:
 310       return "LockFile";
 311     case kUnlockFile:
 312       return "UnlockFile";
 313     case kGetTestDirectory:
 314       return "GetTestDirectory";
 315     case kNewLogger:
 316       return "NewLogger";
 317     case kSyncParent:
 318       return "SyncParent";
 319     case kGetChildren:
 320       return "GetChildren";
 321     case kNumEntries:
 322       NOTREACHED();
 323       return "kNumEntries";
 324   }
 325   NOTREACHED();
 326   return "Unknown";
 327 }
 328
 329 Status MakeIOError(Slice filename,
 330                    const char* message,
 331                    MethodID method,
 332                    int saved_errno) {
 333   char buf[512];
 334   snprintf(buf,
 335            sizeof(buf),
 336            "%s (ChromeMethodErrno: %d::%s::%d)",
 337            message,
 338            method,
 339            MethodIDToString(method),
 340            saved_errno);
 341   return Status::IOError(filename, buf);
 342 }
 343
 344 Status MakeIOError(Slice filename,
 345                    const char* message,
 346                    MethodID method,
 347                    base::PlatformFileError error) {
 348   DCHECK(error < 0);
 349   char buf[512];
 350   snprintf(buf,
 351            sizeof(buf),
 352            "%s (ChromeMethodPFE: %d::%s::%d)",
 353            message,
 354            method,
 355            MethodIDToString(method),
 356            -error);
 357   return Status::IOError(filename, buf);
 358 }
 359
 360 Status MakeIOError(Slice filename, const char* message, MethodID method) {
 361   char buf[512];
 362   snprintf(buf,
 363            sizeof(buf),
 364            "%s (ChromeMethodOnly: %d::%s)",
 365            message,
 366            method,
 367            MethodIDToString(method));
 368   return Status::IOError(filename, buf);
 369 }
 370
 371 ErrorParsingResult ParseMethodAndError(const char* string,
 372                                        MethodID* method_param,
 373                                        int* error) {
 374   int method;
 375   if (RE2::PartialMatch(string, "ChromeMethodOnly: (\\d+)", &method)) {
 376     *method_param = static_cast<MethodID>(method);
 377     return METHOD_ONLY;
 378   }
 379   if (RE2::PartialMatch(
 380           string, "ChromeMethodPFE: (\\d+)::.*::(\\d+)", &method, error)) {
 381     *error = -*error;
 382     *method_param = static_cast<MethodID>(method);
 383     return METHOD_AND_PFE;
 384   }
 385   if (RE2::PartialMatch(
 386           string, "ChromeMethodErrno: (\\d+)::.*::(\\d+)", &method, error)) {
 387     *method_param = static_cast<MethodID>(method);
 388     return METHOD_AND_ERRNO;
 389   }
 390   return NONE;
 391 }
 392
 393 bool IndicatesDiskFull(leveldb::Status status) {
 394   if (status.ok())
 395     return false;
 396   leveldb_env::MethodID method;
 397   int error = -1;
 398   leveldb_env::ErrorParsingResult result = leveldb_env::ParseMethodAndError(
 399       status.ToString().c_str(), &method, &error);
 400   return (result == leveldb_env::METHOD_AND_PFE &&
 401           static_cast<base::PlatformFileError>(error) ==
 402               base::PLATFORM_FILE_ERROR_NO_SPACE) ||
 403          (result == leveldb_env::METHOD_AND_ERRNO && error == ENOSPC);
 404 }
 405
 406 bool IsIOError(leveldb::Status status) {
 407   leveldb_env::MethodID method;
 408   int error = -1;
 409   leveldb_env::ErrorParsingResult result = leveldb_env::ParseMethodAndError(
 410       status.ToString().c_str(), &method, &error);
 411   return result != leveldb_env::NONE;
 412 }
 413
 414 std::string FilePathToString(const base::FilePath& file_path) {
 415 #if defined(OS_WIN)
 416   return UTF16ToUTF8(file_path.value());
 417 #else
 418   return file_path.value();
 419 #endif
 420 }
 421
 422 ChromiumWritableFile::ChromiumWritableFile(const std::string& fname,
 423                                            FILE* f,
 424                                            const UMALogger* uma_logger,
 425                                            WriteTracker* tracker,
 426                                            bool make_backup)
 427     : filename_(fname),
 428       file_(f),
 429       uma_logger_(uma_logger),
 430       tracker_(tracker),
 431       file_type_(kOther),
 432       make_backup_(make_backup) {
 433   base::FilePath path = base::FilePath::FromUTF8Unsafe(fname);
 434   if (FilePathToString(path.BaseName()).find("MANIFEST") == 0)
 435     file_type_ = kManifest;
 436   else if (path.MatchesExtension(table_extension))
 437     file_type_ = kTable;
 438   if (file_type_ != kManifest)
 439     tracker_->DidCreateNewFile(filename_);
 440   parent_dir_ = FilePathToString(CreateFilePath(fname).DirName());
 441 }
 442
 443 ChromiumWritableFile::~ChromiumWritableFile() {
 444   if (file_ != NULL) {
 445     // Ignoring any potential errors
 446     fclose(file_);
 447   }
 448 }
 449
 450 Status ChromiumWritableFile::SyncParent() {
 451   Status s;
 452 #if !defined(OS_WIN)
 453   TRACE_EVENT0("leveldb", "SyncParent");
 454
 455   int parent_fd =
 456       HANDLE_EINTR(open(parent_dir_.c_str(), O_RDONLY));
 457   if (parent_fd < 0) {
 458     int saved_errno = errno;
 459     return MakeIOError(
 460         parent_dir_, strerror(saved_errno), kSyncParent, saved_errno);
 461   }
 462   if (HANDLE_EINTR(fsync(parent_fd)) != 0) {
 463     int saved_errno = errno;
 464     s = MakeIOError(
 465         parent_dir_, strerror(saved_errno), kSyncParent, saved_errno);
 466   };
 467   HANDLE_EINTR(close(parent_fd));
 468 #endif
 469   return s;
 470 }
 471
 472 Status ChromiumWritableFile::Append(const Slice& data) {
 473   if (file_type_ == kManifest && tracker_->DoesDirNeedSync(filename_)) {
 474     Status s = SyncParent();
 475     if (!s.ok())
 476       return s;
 477     tracker_->DidSyncDir(filename_);
 478   }
 479
 480   size_t r = fwrite_unlocked(data.data(), 1, data.size(), file_);
 481   if (r != data.size()) {
 482     int saved_errno = errno;
 483     uma_logger_->RecordOSError(kWritableFileAppend, saved_errno);
 484     return MakeIOError(
 485         filename_, strerror(saved_errno), kWritableFileAppend, saved_errno);
 486   }
 487   return Status::OK();
 488 }
 489
 490 Status ChromiumWritableFile::Close() {
 491   Status result;
 492   if (fclose(file_) != 0) {
 493     result = MakeIOError(filename_, strerror(errno), kWritableFileClose, errno);
 494     uma_logger_->RecordErrorAt(kWritableFileClose);
 495   }
 496   file_ = NULL;
 497   return result;
 498 }
 499
 500 Status ChromiumWritableFile::Flush() {
 501   Status result;
 502   if (HANDLE_EINTR(fflush_unlocked(file_))) {
 503     int saved_errno = errno;
 504     result = MakeIOError(
 505         filename_, strerror(saved_errno), kWritableFileFlush, saved_errno);
 506     uma_logger_->RecordOSError(kWritableFileFlush, saved_errno);
 507   }
 508   return result;
 509 }
 510
 511 static bool MakeBackup(const std::string& fname) {
 512   base::FilePath original_table_name = CreateFilePath(fname);
 513   base::FilePath backup_table_name =
 514       original_table_name.ReplaceExtension(backup_table_extension);
 515   return base::CopyFile(original_table_name, backup_table_name);
 516 }
 517
 518 Status ChromiumWritableFile::Sync() {
 519   TRACE_EVENT0("leveldb", "ChromiumEnv::Sync");
 520   Status result;
 521   int error = 0;
 522
 523   if (HANDLE_EINTR(fflush_unlocked(file_)))
 524     error = errno;
 525   // Sync even if fflush gave an error; perhaps the data actually got out,
 526   // even though something went wrong.
 527   if (fdatasync(fileno(file_)) && !error)
 528     error = errno;
 529   // Report the first error we found.
 530   if (error) {
 531     result = MakeIOError(filename_, strerror(error), kWritableFileSync, error);
 532     uma_logger_->RecordErrorAt(kWritableFileSync);
 533   } else if (make_backup_ && file_type_ == kTable) {
 534     bool success = MakeBackup(filename_);
 535     uma_logger_->RecordBackupResult(success);
 536   }
 537   return result;
 538 }
 539
 540 ChromiumEnv::ChromiumEnv()
 541     : name_("LevelDBEnv"),
 542       make_backup_(false),
 543       bgsignal_(&mu_),
 544       started_bgthread_(false),
 545       kMaxRetryTimeMillis(1000) {
 546 }
 547
 548 ChromiumEnv::~ChromiumEnv() {
 549   // In chromium, ChromiumEnv is leaked. It'd be nice to add NOTREACHED here to
 550   // ensure that behavior isn't accidentally changed, but there's an instance in
 551   // a unit test that is deleted.
 552 }
 553
 554 Status ChromiumEnv::NewSequentialFile(const std::string& fname,
 555                                       SequentialFile** result) {
 556   FILE* f = fopen_internal(fname.c_str(), "rb");
 557   if (f == NULL) {
 558     *result = NULL;
 559     int saved_errno = errno;
 560     RecordOSError(kNewSequentialFile, saved_errno);
 561     return MakeIOError(
 562         fname, strerror(saved_errno), kNewSequentialFile, saved_errno);
 563   } else {
 564     *result = new ChromiumSequentialFile(fname, f, this);
 565     return Status::OK();
 566   }
 567 }
 568
 569 void ChromiumEnv::RecordOpenFilesLimit(const std::string& type) {
 570 #if defined(OS_POSIX)
 571   struct rlimit nofile;
 572   if (getrlimit(RLIMIT_NOFILE, &nofile))
 573     return;
 574   GetMaxFDHistogram(type)->Add(nofile.rlim_cur);
 575 #endif
 576 }
 577
 578 Status ChromiumEnv::NewRandomAccessFile(const std::string& fname,
 579                                         RandomAccessFile** result) {
 580   int flags = ::base::PLATFORM_FILE_READ | ::base::PLATFORM_FILE_OPEN;
 581   bool created;
 582   ::base::PlatformFileError error_code;
 583   ::base::PlatformFile file = ::base::CreatePlatformFile(
 584       CreateFilePath(fname), flags, &created, &error_code);
 585   if (error_code == ::base::PLATFORM_FILE_OK) {
 586     *result = new ChromiumRandomAccessFile(fname, file, this);
 587     RecordOpenFilesLimit("Success");
 588     return Status::OK();
 589   }
 590   if (error_code == ::base::PLATFORM_FILE_ERROR_TOO_MANY_OPENED)
 591     RecordOpenFilesLimit("TooManyOpened");
 592   else
 593     RecordOpenFilesLimit("OtherError");
 594   *result = NULL;
 595   RecordOSError(kNewRandomAccessFile, error_code);
 596   return MakeIOError(fname,
 597                      PlatformFileErrorString(error_code),
 598                      kNewRandomAccessFile,
 599                      error_code);
 600 }
 601
 602 Status ChromiumEnv::NewWritableFile(const std::string& fname,
 603                                     WritableFile** result) {
 604   *result = NULL;
 605   FILE* f = fopen_internal(fname.c_str(), "wb");
 606   if (f == NULL) {
 607     int saved_errno = errno;
 608     RecordErrorAt(kNewWritableFile);
 609     return MakeIOError(
 610         fname, strerror(saved_errno), kNewWritableFile, saved_errno);
 611   } else {
 612     *result = new ChromiumWritableFile(fname, f, this, this, make_backup_);
 613     return Status::OK();
 614   }
 615 }
 616
 617 bool ChromiumEnv::FileExists(const std::string& fname) {
 618   return ::base::PathExists(CreateFilePath(fname));
 619 }
 620
 621 base::FilePath ChromiumEnv::RestoreFromBackup(const base::FilePath& base_name) {
 622   base::FilePath table_name =
 623       base_name.AddExtension(table_extension);
 624   bool result = base::CopyFile(base_name.AddExtension(backup_table_extension),
 625                                table_name);
 626   std::string uma_name(name_);
 627   uma_name.append(".TableRestore");
 628   base::BooleanHistogram::FactoryGet(
 629       uma_name, base::Histogram::kUmaTargetedHistogramFlag)->AddBoolean(result);
 630   return table_name;
 631 }
 632
 633 void ChromiumEnv::RestoreIfNecessary(const std::string& dir,
 634                                      std::vector<std::string>* result) {
 635   std::set<base::FilePath> tables_found;
 636   std::set<base::FilePath> backups_found;
 637   for (std::vector<std::string>::iterator it = result->begin();
 638        it != result->end();
 639        ++it) {
 640     base::FilePath current = CreateFilePath(*it);
 641     if (current.MatchesExtension(table_extension))
 642       tables_found.insert(current.RemoveExtension());
 643     if (current.MatchesExtension(backup_table_extension))
 644       backups_found.insert(current.RemoveExtension());
 645   }
 646   std::set<base::FilePath> backups_only;
 647   std::set_difference(backups_found.begin(),
 648                       backups_found.end(),
 649                       tables_found.begin(),
 650                       tables_found.end(),
 651                       std::inserter(backups_only, backups_only.begin()));
 652   if (backups_only.size()) {
 653     std::string uma_name(name_);
 654     uma_name.append(".MissingFiles");
 655     int num_missing_files =
 656         backups_only.size() > INT_MAX ? INT_MAX : backups_only.size();
 657     base::Histogram::FactoryGet(uma_name,
 658                                 1 /*min*/,
 659                                 100 /*max*/,
 660                                 8 /*num_buckets*/,
 661                                 base::Histogram::kUmaTargetedHistogramFlag)
 662         ->Add(num_missing_files);
 663   }
 664   base::FilePath dir_filepath = base::FilePath::FromUTF8Unsafe(dir);
 665   for (std::set<base::FilePath>::iterator it = backups_only.begin();
 666        it != backups_only.end();
 667        ++it) {
 668     base::FilePath restored_table_name =
 669         RestoreFromBackup(dir_filepath.Append(*it));
 670     result->push_back(FilePathToString(restored_table_name.BaseName()));
 671   }
 672 }
 673
 674 namespace {
 675 #if defined(OS_WIN)
 676 static base::PlatformFileError GetDirectoryEntries(
 677     const base::FilePath& dir_param,
 678     std::vector<base::FilePath>* result) {
 679   result->clear();
 680   base::FilePath dir_filepath = dir_param.Append(FILE_PATH_LITERAL("*"));
 681   WIN32_FIND_DATA find_data;
 682   HANDLE find_handle = FindFirstFile(dir_filepath.value().c_str(), &find_data);
 683   if (find_handle == INVALID_HANDLE_VALUE) {
 684     DWORD last_error = GetLastError();
 685     if (last_error == ERROR_FILE_NOT_FOUND)
 686       return base::PLATFORM_FILE_OK;
 687     return base::LastErrorToPlatformFileError(last_error);
 688   }
 689   do {
 690     base::FilePath filepath(find_data.cFileName);
 691     base::FilePath::StringType basename = filepath.BaseName().value();
 692     if (basename == FILE_PATH_LITERAL(".") ||
 693         basename == FILE_PATH_LITERAL(".."))
 694       continue;
 695     result->push_back(filepath.BaseName());
 696   } while (FindNextFile(find_handle, &find_data));
 697   DWORD last_error = GetLastError();
 698   base::PlatformFileError return_value = base::PLATFORM_FILE_OK;
 699   if (last_error != ERROR_NO_MORE_FILES)
 700     return_value = base::LastErrorToPlatformFileError(last_error);
 701   FindClose(find_handle);
 702   return return_value;
 703 }
 704 #else
 705 static base::PlatformFileError GetDirectoryEntries(
 706     const base::FilePath& dir_filepath,
 707     std::vector<base::FilePath>* result) {
 708   const std::string dir_string = FilePathToString(dir_filepath);
 709   result->clear();
 710   DIR* dir = opendir(dir_string.c_str());
 711   if (!dir)
 712     return base::ErrnoToPlatformFileError(errno);
 713   struct dirent dent_buf;
 714   struct dirent* dent;
 715   int readdir_result;
 716   while ((readdir_result = readdir_r(dir, &dent_buf, &dent)) == 0 && dent)
 717     result->push_back(CreateFilePath(dent->d_name));
 718   int saved_errno = errno;
 719   closedir(dir);
 720   if (readdir_result != 0)
 721     return base::ErrnoToPlatformFileError(saved_errno);
 722   return base::PLATFORM_FILE_OK;
 723 }
 724 #endif
 725 }
 726
 727 Status ChromiumEnv::GetChildren(const std::string& dir_string,
 728                                 std::vector<std::string>* result) {
 729   std::vector<base::FilePath> entries;
 730   base::PlatformFileError error =
 731       GetDirectoryEntries(CreateFilePath(dir_string), &entries);
 732   if (error != base::PLATFORM_FILE_OK) {
 733     RecordOSError(kGetChildren, error);
 734     return MakeIOError(
 735         dir_string, "Could not open/read directory", kGetChildren, error);
 736   }
 737   for (std::vector<base::FilePath>::iterator it = entries.begin();
 738        it != entries.end();
 739        ++it) {
 740     result->push_back(FilePathToString(*it));
 741   }
 742
 743   if (make_backup_)
 744     RestoreIfNecessary(dir_string, result);
 745   return Status::OK();
 746 }
 747
 748 Status ChromiumEnv::DeleteFile(const std::string& fname) {
 749   Status result;
 750   base::FilePath fname_filepath = CreateFilePath(fname);
 751   // TODO(jorlow): Should we assert this is a file?
 752   if (!::base::DeleteFile(fname_filepath, false)) {
 753     result = MakeIOError(fname, "Could not delete file.", kDeleteFile);
 754     RecordErrorAt(kDeleteFile);
 755   }
 756   if (make_backup_ && fname_filepath.MatchesExtension(table_extension)) {
 757     base::DeleteFile(fname_filepath.ReplaceExtension(backup_table_extension),
 758                      false);
 759   }
 760   return result;
 761 }
 762
 763 Status ChromiumEnv::CreateDir(const std::string& name) {
 764   Status result;
 765   base::PlatformFileError error = base::PLATFORM_FILE_OK;
 766   Retrier retrier(kCreateDir, this);
 767   do {
 768     if (::file_util::CreateDirectoryAndGetError(CreateFilePath(name), &error))
 769       return result;
 770   } while (retrier.ShouldKeepTrying(error));
 771   result = MakeIOError(name, "Could not create directory.", kCreateDir, error);
 772   RecordOSError(kCreateDir, error);
 773   return result;
 774 }
 775
 776 Status ChromiumEnv::DeleteDir(const std::string& name) {
 777   Status result;
 778   // TODO(jorlow): Should we assert this is a directory?
 779   if (!::base::DeleteFile(CreateFilePath(name), false)) {
 780     result = MakeIOError(name, "Could not delete directory.", kDeleteDir);
 781     RecordErrorAt(kDeleteDir);
 782   }
 783   return result;
 784 }
 785
 786 Status ChromiumEnv::GetFileSize(const std::string& fname, uint64_t* size) {
 787   Status s;
 788   int64_t signed_size;
 789   if (!::file_util::GetFileSize(CreateFilePath(fname), &signed_size)) {
 790     *size = 0;
 791     s = MakeIOError(fname, "Could not determine file size.", kGetFileSize);
 792     RecordErrorAt(kGetFileSize);
 793   } else {
 794     *size = static_cast<uint64_t>(signed_size);
 795   }
 796   return s;
 797 }
 798
 799 Status ChromiumEnv::RenameFile(const std::string& src, const std::string& dst) {
 800   Status result;
 801   base::FilePath src_file_path = CreateFilePath(src);
 802   if (!::base::PathExists(src_file_path))
 803     return result;
 804   base::FilePath destination = CreateFilePath(dst);
 805
 806   Retrier retrier(kRenameFile, this);
 807   base::PlatformFileError error = base::PLATFORM_FILE_OK;
 808   do {
 809     if (base::ReplaceFile(src_file_path, destination, &error))
 810       return result;
 811   } while (retrier.ShouldKeepTrying(error));
 812
 813   DCHECK(error != base::PLATFORM_FILE_OK);
 814   RecordOSError(kRenameFile, error);
 815   char buf[100];
 816   snprintf(buf,
 817            sizeof(buf),
 818            "Could not rename file: %s",
 819            PlatformFileErrorString(error));
 820   return MakeIOError(src, buf, kRenameFile, error);
 821 }
 822
 823 Status ChromiumEnv::LockFile(const std::string& fname, FileLock** lock) {
 824   *lock = NULL;
 825   Status result;
 826   int flags = ::base::PLATFORM_FILE_OPEN_ALWAYS |
 827               ::base::PLATFORM_FILE_READ |
 828               ::base::PLATFORM_FILE_WRITE;
 829   bool created;
 830   ::base::PlatformFileError error_code;
 831   ::base::PlatformFile file;
 832   Retrier retrier(kLockFile, this);
 833   do {
 834     file = ::base::CreatePlatformFile(
 835         CreateFilePath(fname), flags, &created, &error_code);
 836   } while (error_code != ::base::PLATFORM_FILE_OK &&
 837            retrier.ShouldKeepTrying(error_code));
 838
 839   if (error_code == ::base::PLATFORM_FILE_ERROR_NOT_FOUND) {
 840     ::base::FilePath parent = CreateFilePath(fname).DirName();
 841     ::base::FilePath last_parent;
 842     int num_missing_ancestors = 0;
 843     do {
 844       if (base::DirectoryExists(parent))
 845         break;
 846       ++num_missing_ancestors;
 847       last_parent = parent;
 848       parent = parent.DirName();
 849     } while (parent != last_parent);
 850     RecordLockFileAncestors(num_missing_ancestors);
 851   }
 852
 853   if (error_code != ::base::PLATFORM_FILE_OK) {
 854     result = MakeIOError(
 855         fname, PlatformFileErrorString(error_code), kLockFile, error_code);
 856     RecordOSError(kLockFile, error_code);
 857     return result;
 858   }
 859
 860   if (!locks_.Insert(fname)) {
 861     result = MakeIOError(fname, "Lock file already locked.", kLockFile);
 862     ::base::ClosePlatformFile(file);
 863     return result;
 864   }
 865
 866   Retrier lock_retrier = Retrier(kLockFile, this);
 867   do {
 868     error_code = ::base::LockPlatformFile(file);
 869   } while (error_code != ::base::PLATFORM_FILE_OK &&
 870            retrier.ShouldKeepTrying(error_code));
 871
 872   if (error_code != ::base::PLATFORM_FILE_OK) {
 873     ::base::ClosePlatformFile(file);
 874     locks_.Remove(fname);
 875     result = MakeIOError(
 876         fname, PlatformFileErrorString(error_code), kLockFile, error_code);
 877     RecordOSError(kLockFile, error_code);
 878     return result;
 879   }
 880
 881   ChromiumFileLock* my_lock = new ChromiumFileLock;
 882   my_lock->file_ = file;
 883   my_lock->name_ = fname;
 884   *lock = my_lock;
 885   return result;
 886 }
 887
 888 Status ChromiumEnv::UnlockFile(FileLock* lock) {
 889   ChromiumFileLock* my_lock = reinterpret_cast<ChromiumFileLock*>(lock);
 890   Status result;
 891
 892   ::base::PlatformFileError error_code =
 893       ::base::UnlockPlatformFile(my_lock->file_);
 894   if (error_code != ::base::PLATFORM_FILE_OK) {
 895     result =
 896         MakeIOError(my_lock->name_, "Could not unlock lock file.", kUnlockFile);
 897     RecordOSError(kUnlockFile, error_code);
 898     ::base::ClosePlatformFile(my_lock->file_);
 899   } else if (!::base::ClosePlatformFile(my_lock->file_)) {
 900     result =
 901         MakeIOError(my_lock->name_, "Could not close lock file.", kUnlockFile);
 902     RecordErrorAt(kUnlockFile);
 903   }
 904   bool removed = locks_.Remove(my_lock->name_);
 905   DCHECK(removed);
 906   delete my_lock;
 907   return result;
 908 }
 909
 910 Status ChromiumEnv::GetTestDirectory(std::string* path) {
 911   mu_.Acquire();
 912   if (test_directory_.empty()) {
 913     if (!::file_util::CreateNewTempDirectory(kLevelDBTestDirectoryPrefix,
 914                                              &test_directory_)) {
 915       mu_.Release();
 916       RecordErrorAt(kGetTestDirectory);
 917       return MakeIOError(
 918           "Could not create temp directory.", "", kGetTestDirectory);
 919     }
 920   }
 921   *path = FilePathToString(test_directory_);
 922   mu_.Release();
 923   return Status::OK();
 924 }
 925
 926 Status ChromiumEnv::NewLogger(const std::string& fname, Logger** result) {
 927   FILE* f = fopen_internal(fname.c_str(), "w");
 928   if (f == NULL) {
 929     *result = NULL;
 930     int saved_errno = errno;
 931     RecordOSError(kNewLogger, saved_errno);
 932     return MakeIOError(fname, strerror(saved_errno), kNewLogger, saved_errno);
 933   } else {
 934     *result = new ChromiumLogger(f);
 935     return Status::OK();
 936   }
 937 }
 938
 939 uint64_t ChromiumEnv::NowMicros() {
 940   return ::base::TimeTicks::Now().ToInternalValue();
 941 }
 942
 943 void ChromiumEnv::SleepForMicroseconds(int micros) {
 944   // Round up to the next millisecond.
 945   ::base::PlatformThread::Sleep(::base::TimeDelta::FromMicroseconds(micros));
 946 }
 947
 948 void ChromiumEnv::RecordErrorAt(MethodID method) const {
 949   GetMethodIOErrorHistogram()->Add(method);
 950 }
 951
 952 void ChromiumEnv::RecordLockFileAncestors(int num_missing_ancestors) const {
 953   GetLockFileAncestorHistogram()->Add(num_missing_ancestors);
 954 }
 955
 956 void ChromiumEnv::RecordOSError(MethodID method,
 957                                 base::PlatformFileError error) const {
 958   DCHECK(error < 0);
 959   RecordErrorAt(method);
 960   GetOSErrorHistogram(method, -base::PLATFORM_FILE_ERROR_MAX)->Add(-error);
 961 }
 962
 963 void ChromiumEnv::RecordOSError(MethodID method, int error) const {
 964   DCHECK(error > 0);
 965   RecordErrorAt(method);
 966   GetOSErrorHistogram(method, ERANGE + 1)->Add(error);
 967 }
 968
 969 void ChromiumEnv::RecordBackupResult(bool result) const {
 970   std::string uma_name(name_);
 971   uma_name.append(".TableBackup");
 972   base::BooleanHistogram::FactoryGet(
 973       uma_name, base::Histogram::kUmaTargetedHistogramFlag)->AddBoolean(result);
 974 }
 975
 976 base::HistogramBase* ChromiumEnv::GetOSErrorHistogram(MethodID method,
 977                                                       int limit) const {
 978   std::string uma_name(name_);
 979   // TODO(dgrogan): This is probably not the best way to concatenate strings.
 980   uma_name.append(".IOError.").append(MethodIDToString(method));
 981   return base::LinearHistogram::FactoryGet(uma_name, 1, limit, limit + 1,
 982       base::Histogram::kUmaTargetedHistogramFlag);
 983 }
 984
 985 base::HistogramBase* ChromiumEnv::GetMethodIOErrorHistogram() const {
 986   std::string uma_name(name_);
 987   uma_name.append(".IOError");
 988   return base::LinearHistogram::FactoryGet(uma_name, 1, kNumEntries,
 989       kNumEntries + 1, base::Histogram::kUmaTargetedHistogramFlag);
 990 }
 991
 992 base::HistogramBase* ChromiumEnv::GetMaxFDHistogram(
 993     const std::string& type) const {
 994   std::string uma_name(name_);
 995   uma_name.append(".MaxFDs.").append(type);
 996   // These numbers make each bucket twice as large as the previous bucket.
 997   const int kFirstEntry = 1;
 998   const int kLastEntry = 65536;
 999   const int kNumBuckets = 18;
1000   return base::Histogram::FactoryGet(
1001       uma_name, kFirstEntry, kLastEntry, kNumBuckets,
1002       base::Histogram::kUmaTargetedHistogramFlag);
1003 }
1004
1005 base::HistogramBase* ChromiumEnv::GetLockFileAncestorHistogram() const {
1006   std::string uma_name(name_);
1007   uma_name.append(".LockFileAncestorsNotFound");
1008   const int kMin = 1;
1009   const int kMax = 10;
1010   const int kNumBuckets = 11;
1011   return base::LinearHistogram::FactoryGet(
1012       uma_name, kMin, kMax, kNumBuckets,
1013       base::Histogram::kUmaTargetedHistogramFlag);
1014 }
1015
1016 base::HistogramBase* ChromiumEnv::GetRetryTimeHistogram(MethodID method) const {
1017   std::string uma_name(name_);
1018   // TODO(dgrogan): This is probably not the best way to concatenate strings.
1019   uma_name.append(".TimeUntilSuccessFor").append(MethodIDToString(method));
1020
1021   const int kBucketSizeMillis = 25;
1022   // Add 2, 1 for each of the buckets <1 and >max.
1023   const int kNumBuckets = kMaxRetryTimeMillis / kBucketSizeMillis + 2;
1024   return base::Histogram::FactoryTimeGet(
1025       uma_name, base::TimeDelta::FromMilliseconds(1),
1026       base::TimeDelta::FromMilliseconds(kMaxRetryTimeMillis + 1),
1027       kNumBuckets,
1028       base::Histogram::kUmaTargetedHistogramFlag);
1029 }
1030
1031 base::HistogramBase* ChromiumEnv::GetRecoveredFromErrorHistogram(
1032     MethodID method) const {
1033   std::string uma_name(name_);
1034   uma_name.append(".RetryRecoveredFromErrorIn")
1035       .append(MethodIDToString(method));
1036   return base::LinearHistogram::FactoryGet(uma_name, 1, kNumEntries,
1037       kNumEntries + 1, base::Histogram::kUmaTargetedHistogramFlag);
1038 }
1039
1040 class Thread : public ::base::PlatformThread::Delegate {
1041  public:
1042   Thread(void (*function)(void* arg), void* arg)
1043       : function_(function), arg_(arg) {
1044     ::base::PlatformThreadHandle handle;
1045     bool success = ::base::PlatformThread::Create(0, this, &handle);
1046     DCHECK(success);
1047   }
1048   virtual ~Thread() {}
1049   virtual void ThreadMain() {
1050     (*function_)(arg_);
1051     delete this;
1052   }
1053
1054  private:
1055   void (*function_)(void* arg);
1056   void* arg_;
1057 };
1058
1059 void ChromiumEnv::Schedule(void (*function)(void*), void* arg) {
1060   mu_.Acquire();
1061
1062   // Start background thread if necessary
1063   if (!started_bgthread_) {
1064     started_bgthread_ = true;
1065     StartThread(&ChromiumEnv::BGThreadWrapper, this);
1066   }
1067
1068   // If the queue is currently empty, the background thread may currently be
1069   // waiting.
1070   if (queue_.empty()) {
1071     bgsignal_.Signal();
1072   }
1073
1074   // Add to priority queue
1075   queue_.push_back(BGItem());
1076   queue_.back().function = function;
1077   queue_.back().arg = arg;
1078
1079   mu_.Release();
1080 }
1081
1082 void ChromiumEnv::BGThread() {
1083   base::PlatformThread::SetName(name_.c_str());
1084
1085   while (true) {
1086     // Wait until there is an item that is ready to run
1087     mu_.Acquire();
1088     while (queue_.empty()) {
1089       bgsignal_.Wait();
1090     }
1091
1092     void (*function)(void*) = queue_.front().function;
1093     void* arg = queue_.front().arg;
1094     queue_.pop_front();
1095
1096     mu_.Release();
1097     TRACE_EVENT0("leveldb", "ChromiumEnv::BGThread-Task");
1098     (*function)(arg);
1099   }
1100 }
1101
1102 void ChromiumEnv::StartThread(void (*function)(void* arg), void* arg) {
1103   new Thread(function, arg); // Will self-delete.
1104 }
1105
1106 static std::string GetDirName(const std::string& filename) {
1107   base::FilePath file = base::FilePath::FromUTF8Unsafe(filename);
1108   return FilePathToString(file.DirName());
1109 }
1110
1111 void ChromiumEnv::DidCreateNewFile(const std::string& filename) {
1112   base::AutoLock auto_lock(map_lock_);
1113   needs_sync_map_[GetDirName(filename)] = true;
1114 }
1115
1116 bool ChromiumEnv::DoesDirNeedSync(const std::string& filename) {
1117   base::AutoLock auto_lock(map_lock_);
1118   return needs_sync_map_.find(GetDirName(filename)) != needs_sync_map_.end();
1119 }
1120
1121 void ChromiumEnv::DidSyncDir(const std::string& filename) {
1122   base::AutoLock auto_lock(map_lock_);
1123   needs_sync_map_.erase(GetDirName(filename));
1124 }
1125
1126 }  // namespace leveldb_env
1127
1128 namespace leveldb {
1129
1130 Env* IDBEnv() {
1131   return leveldb_env::idb_env.Pointer();
1132 }
1133
1134 Env* Env::Default() {
1135   return leveldb_env::default_env.Pointer();
1136 }
1137
1138 }  // namespace leveldb
1139