Merge #11107: Fix races in AppInitMain and others with lock and atomic bools
[bitcoinplatinum.git] / src / wallet / db.cpp
blobd66ba48421fe39e5db5383f0d175d4909c220900
1 // Copyright (c) 2009-2010 Satoshi Nakamoto
2 // Copyright (c) 2009-2016 The Bitcoin Core developers
3 // Distributed under the MIT software license, see the accompanying
4 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
6 #include "db.h"
8 #include "addrman.h"
9 #include "fs.h"
10 #include "hash.h"
11 #include "protocol.h"
12 #include "util.h"
13 #include "utilstrencodings.h"
15 #include <stdint.h>
17 #ifndef WIN32
18 #include <sys/stat.h>
19 #endif
21 #include <boost/thread.hpp>
24 // CDB
27 CDBEnv bitdb;
29 void CDBEnv::EnvShutdown()
31 if (!fDbEnvInit)
32 return;
34 fDbEnvInit = false;
35 int ret = dbenv->close(0);
36 if (ret != 0)
37 LogPrintf("CDBEnv::EnvShutdown: Error %d shutting down database environment: %s\n", ret, DbEnv::strerror(ret));
38 if (!fMockDb)
39 DbEnv((u_int32_t)0).remove(strPath.c_str(), 0);
42 void CDBEnv::Reset()
44 delete dbenv;
45 dbenv = new DbEnv(DB_CXX_NO_EXCEPTIONS);
46 fDbEnvInit = false;
47 fMockDb = false;
50 CDBEnv::CDBEnv() : dbenv(nullptr)
52 Reset();
55 CDBEnv::~CDBEnv()
57 EnvShutdown();
58 delete dbenv;
59 dbenv = nullptr;
62 void CDBEnv::Close()
64 EnvShutdown();
67 bool CDBEnv::Open(const fs::path& pathIn)
69 if (fDbEnvInit)
70 return true;
72 boost::this_thread::interruption_point();
74 strPath = pathIn.string();
75 fs::path pathLogDir = pathIn / "database";
76 TryCreateDirectories(pathLogDir);
77 fs::path pathErrorFile = pathIn / "db.log";
78 LogPrintf("CDBEnv::Open: LogDir=%s ErrorFile=%s\n", pathLogDir.string(), pathErrorFile.string());
80 unsigned int nEnvFlags = 0;
81 if (gArgs.GetBoolArg("-privdb", DEFAULT_WALLET_PRIVDB))
82 nEnvFlags |= DB_PRIVATE;
84 dbenv->set_lg_dir(pathLogDir.string().c_str());
85 dbenv->set_cachesize(0, 0x100000, 1); // 1 MiB should be enough for just the wallet
86 dbenv->set_lg_bsize(0x10000);
87 dbenv->set_lg_max(1048576);
88 dbenv->set_lk_max_locks(40000);
89 dbenv->set_lk_max_objects(40000);
90 dbenv->set_errfile(fsbridge::fopen(pathErrorFile, "a")); /// debug
91 dbenv->set_flags(DB_AUTO_COMMIT, 1);
92 dbenv->set_flags(DB_TXN_WRITE_NOSYNC, 1);
93 dbenv->log_set_config(DB_LOG_AUTO_REMOVE, 1);
94 int ret = dbenv->open(strPath.c_str(),
95 DB_CREATE |
96 DB_INIT_LOCK |
97 DB_INIT_LOG |
98 DB_INIT_MPOOL |
99 DB_INIT_TXN |
100 DB_THREAD |
101 DB_RECOVER |
102 nEnvFlags,
103 S_IRUSR | S_IWUSR);
104 if (ret != 0) {
105 dbenv->close(0);
106 return error("CDBEnv::Open: Error %d opening database environment: %s\n", ret, DbEnv::strerror(ret));
109 fDbEnvInit = true;
110 fMockDb = false;
111 return true;
114 void CDBEnv::MakeMock()
116 if (fDbEnvInit)
117 throw std::runtime_error("CDBEnv::MakeMock: Already initialized");
119 boost::this_thread::interruption_point();
121 LogPrint(BCLog::DB, "CDBEnv::MakeMock\n");
123 dbenv->set_cachesize(1, 0, 1);
124 dbenv->set_lg_bsize(10485760 * 4);
125 dbenv->set_lg_max(10485760);
126 dbenv->set_lk_max_locks(10000);
127 dbenv->set_lk_max_objects(10000);
128 dbenv->set_flags(DB_AUTO_COMMIT, 1);
129 dbenv->log_set_config(DB_LOG_IN_MEMORY, 1);
130 int ret = dbenv->open(nullptr,
131 DB_CREATE |
132 DB_INIT_LOCK |
133 DB_INIT_LOG |
134 DB_INIT_MPOOL |
135 DB_INIT_TXN |
136 DB_THREAD |
137 DB_PRIVATE,
138 S_IRUSR | S_IWUSR);
139 if (ret > 0)
140 throw std::runtime_error(strprintf("CDBEnv::MakeMock: Error %d opening database environment.", ret));
142 fDbEnvInit = true;
143 fMockDb = true;
146 CDBEnv::VerifyResult CDBEnv::Verify(const std::string& strFile, recoverFunc_type recoverFunc, std::string& out_backup_filename)
148 LOCK(cs_db);
149 assert(mapFileUseCount.count(strFile) == 0);
151 Db db(dbenv, 0);
152 int result = db.verify(strFile.c_str(), nullptr, nullptr, 0);
153 if (result == 0)
154 return VERIFY_OK;
155 else if (recoverFunc == nullptr)
156 return RECOVER_FAIL;
158 // Try to recover:
159 bool fRecovered = (*recoverFunc)(strFile, out_backup_filename);
160 return (fRecovered ? RECOVER_OK : RECOVER_FAIL);
163 bool CDB::Recover(const std::string& filename, void *callbackDataIn, bool (*recoverKVcallback)(void* callbackData, CDataStream ssKey, CDataStream ssValue), std::string& newFilename)
165 // Recovery procedure:
166 // move wallet file to walletfilename.timestamp.bak
167 // Call Salvage with fAggressive=true to
168 // get as much data as possible.
169 // Rewrite salvaged data to fresh wallet file
170 // Set -rescan so any missing transactions will be
171 // found.
172 int64_t now = GetTime();
173 newFilename = strprintf("%s.%d.bak", filename, now);
175 int result = bitdb.dbenv->dbrename(nullptr, filename.c_str(), nullptr,
176 newFilename.c_str(), DB_AUTO_COMMIT);
177 if (result == 0)
178 LogPrintf("Renamed %s to %s\n", filename, newFilename);
179 else
181 LogPrintf("Failed to rename %s to %s\n", filename, newFilename);
182 return false;
185 std::vector<CDBEnv::KeyValPair> salvagedData;
186 bool fSuccess = bitdb.Salvage(newFilename, true, salvagedData);
187 if (salvagedData.empty())
189 LogPrintf("Salvage(aggressive) found no records in %s.\n", newFilename);
190 return false;
192 LogPrintf("Salvage(aggressive) found %u records\n", salvagedData.size());
194 std::unique_ptr<Db> pdbCopy(new Db(bitdb.dbenv, 0));
195 int ret = pdbCopy->open(nullptr, // Txn pointer
196 filename.c_str(), // Filename
197 "main", // Logical db name
198 DB_BTREE, // Database type
199 DB_CREATE, // Flags
201 if (ret > 0) {
202 LogPrintf("Cannot create database file %s\n", filename);
203 pdbCopy->close(0);
204 return false;
207 DbTxn* ptxn = bitdb.TxnBegin();
208 for (CDBEnv::KeyValPair& row : salvagedData)
210 if (recoverKVcallback)
212 CDataStream ssKey(row.first, SER_DISK, CLIENT_VERSION);
213 CDataStream ssValue(row.second, SER_DISK, CLIENT_VERSION);
214 if (!(*recoverKVcallback)(callbackDataIn, ssKey, ssValue))
215 continue;
217 Dbt datKey(&row.first[0], row.first.size());
218 Dbt datValue(&row.second[0], row.second.size());
219 int ret2 = pdbCopy->put(ptxn, &datKey, &datValue, DB_NOOVERWRITE);
220 if (ret2 > 0)
221 fSuccess = false;
223 ptxn->commit(0);
224 pdbCopy->close(0);
226 return fSuccess;
229 bool CDB::VerifyEnvironment(const std::string& walletFile, const fs::path& dataDir, std::string& errorStr)
231 LogPrintf("Using BerkeleyDB version %s\n", DbEnv::version(0, 0, 0));
232 LogPrintf("Using wallet %s\n", walletFile);
234 // Wallet file must be a plain filename without a directory
235 if (walletFile != fs::basename(walletFile) + fs::extension(walletFile))
237 errorStr = strprintf(_("Wallet %s resides outside data directory %s"), walletFile, dataDir.string());
238 return false;
241 if (!bitdb.Open(dataDir))
243 // try moving the database env out of the way
244 fs::path pathDatabase = dataDir / "database";
245 fs::path pathDatabaseBak = dataDir / strprintf("database.%d.bak", GetTime());
246 try {
247 fs::rename(pathDatabase, pathDatabaseBak);
248 LogPrintf("Moved old %s to %s. Retrying.\n", pathDatabase.string(), pathDatabaseBak.string());
249 } catch (const fs::filesystem_error&) {
250 // failure is ok (well, not really, but it's not worse than what we started with)
253 // try again
254 if (!bitdb.Open(dataDir)) {
255 // if it still fails, it probably means we can't even create the database env
256 errorStr = strprintf(_("Error initializing wallet database environment %s!"), GetDataDir());
257 return false;
260 return true;
263 bool CDB::VerifyDatabaseFile(const std::string& walletFile, const fs::path& dataDir, std::string& warningStr, std::string& errorStr, CDBEnv::recoverFunc_type recoverFunc)
265 if (fs::exists(dataDir / walletFile))
267 std::string backup_filename;
268 CDBEnv::VerifyResult r = bitdb.Verify(walletFile, recoverFunc, backup_filename);
269 if (r == CDBEnv::RECOVER_OK)
271 warningStr = strprintf(_("Warning: Wallet file corrupt, data salvaged!"
272 " Original %s saved as %s in %s; if"
273 " your balance or transactions are incorrect you should"
274 " restore from a backup."),
275 walletFile, backup_filename, dataDir);
277 if (r == CDBEnv::RECOVER_FAIL)
279 errorStr = strprintf(_("%s corrupt, salvage failed"), walletFile);
280 return false;
283 // also return true if files does not exists
284 return true;
287 /* End of headers, beginning of key/value data */
288 static const char *HEADER_END = "HEADER=END";
289 /* End of key/value data */
290 static const char *DATA_END = "DATA=END";
292 bool CDBEnv::Salvage(const std::string& strFile, bool fAggressive, std::vector<CDBEnv::KeyValPair>& vResult)
294 LOCK(cs_db);
295 assert(mapFileUseCount.count(strFile) == 0);
297 u_int32_t flags = DB_SALVAGE;
298 if (fAggressive)
299 flags |= DB_AGGRESSIVE;
301 std::stringstream strDump;
303 Db db(dbenv, 0);
304 int result = db.verify(strFile.c_str(), nullptr, &strDump, flags);
305 if (result == DB_VERIFY_BAD) {
306 LogPrintf("CDBEnv::Salvage: Database salvage found errors, all data may not be recoverable.\n");
307 if (!fAggressive) {
308 LogPrintf("CDBEnv::Salvage: Rerun with aggressive mode to ignore errors and continue.\n");
309 return false;
312 if (result != 0 && result != DB_VERIFY_BAD) {
313 LogPrintf("CDBEnv::Salvage: Database salvage failed with result %d.\n", result);
314 return false;
317 // Format of bdb dump is ascii lines:
318 // header lines...
319 // HEADER=END
320 // hexadecimal key
321 // hexadecimal value
322 // ... repeated
323 // DATA=END
325 std::string strLine;
326 while (!strDump.eof() && strLine != HEADER_END)
327 getline(strDump, strLine); // Skip past header
329 std::string keyHex, valueHex;
330 while (!strDump.eof() && keyHex != DATA_END) {
331 getline(strDump, keyHex);
332 if (keyHex != DATA_END) {
333 if (strDump.eof())
334 break;
335 getline(strDump, valueHex);
336 if (valueHex == DATA_END) {
337 LogPrintf("CDBEnv::Salvage: WARNING: Number of keys in data does not match number of values.\n");
338 break;
340 vResult.push_back(make_pair(ParseHex(keyHex), ParseHex(valueHex)));
344 if (keyHex != DATA_END) {
345 LogPrintf("CDBEnv::Salvage: WARNING: Unexpected end of file while reading salvage output.\n");
346 return false;
349 return (result == 0);
353 void CDBEnv::CheckpointLSN(const std::string& strFile)
355 dbenv->txn_checkpoint(0, 0, 0);
356 if (fMockDb)
357 return;
358 dbenv->lsn_reset(strFile.c_str(), 0);
362 CDB::CDB(CWalletDBWrapper& dbw, const char* pszMode, bool fFlushOnCloseIn) : pdb(nullptr), activeTxn(nullptr)
364 fReadOnly = (!strchr(pszMode, '+') && !strchr(pszMode, 'w'));
365 fFlushOnClose = fFlushOnCloseIn;
366 env = dbw.env;
367 if (dbw.IsDummy()) {
368 return;
370 const std::string &strFilename = dbw.strFile;
372 bool fCreate = strchr(pszMode, 'c') != nullptr;
373 unsigned int nFlags = DB_THREAD;
374 if (fCreate)
375 nFlags |= DB_CREATE;
378 LOCK(env->cs_db);
379 if (!env->Open(GetDataDir()))
380 throw std::runtime_error("CDB: Failed to open database environment.");
382 strFile = strFilename;
383 ++env->mapFileUseCount[strFile];
384 pdb = env->mapDb[strFile];
385 if (pdb == nullptr) {
386 int ret;
387 pdb = new Db(env->dbenv, 0);
389 bool fMockDb = env->IsMock();
390 if (fMockDb) {
391 DbMpoolFile* mpf = pdb->get_mpf();
392 ret = mpf->set_flags(DB_MPOOL_NOFILE, 1);
393 if (ret != 0)
394 throw std::runtime_error(strprintf("CDB: Failed to configure for no temp file backing for database %s", strFile));
397 ret = pdb->open(nullptr, // Txn pointer
398 fMockDb ? nullptr : strFile.c_str(), // Filename
399 fMockDb ? strFile.c_str() : "main", // Logical db name
400 DB_BTREE, // Database type
401 nFlags, // Flags
404 if (ret != 0) {
405 delete pdb;
406 pdb = nullptr;
407 --env->mapFileUseCount[strFile];
408 strFile = "";
409 throw std::runtime_error(strprintf("CDB: Error %d, can't open database %s", ret, strFilename));
412 if (fCreate && !Exists(std::string("version"))) {
413 bool fTmp = fReadOnly;
414 fReadOnly = false;
415 WriteVersion(CLIENT_VERSION);
416 fReadOnly = fTmp;
419 env->mapDb[strFile] = pdb;
424 void CDB::Flush()
426 if (activeTxn)
427 return;
429 // Flush database activity from memory pool to disk log
430 unsigned int nMinutes = 0;
431 if (fReadOnly)
432 nMinutes = 1;
434 env->dbenv->txn_checkpoint(nMinutes ? gArgs.GetArg("-dblogsize", DEFAULT_WALLET_DBLOGSIZE) * 1024 : 0, nMinutes, 0);
437 void CWalletDBWrapper::IncrementUpdateCounter()
439 ++nUpdateCounter;
442 void CDB::Close()
444 if (!pdb)
445 return;
446 if (activeTxn)
447 activeTxn->abort();
448 activeTxn = nullptr;
449 pdb = nullptr;
451 if (fFlushOnClose)
452 Flush();
455 LOCK(env->cs_db);
456 --env->mapFileUseCount[strFile];
460 void CDBEnv::CloseDb(const std::string& strFile)
463 LOCK(cs_db);
464 if (mapDb[strFile] != nullptr) {
465 // Close the database handle
466 Db* pdb = mapDb[strFile];
467 pdb->close(0);
468 delete pdb;
469 mapDb[strFile] = nullptr;
474 bool CDB::Rewrite(CWalletDBWrapper& dbw, const char* pszSkip)
476 if (dbw.IsDummy()) {
477 return true;
479 CDBEnv *env = dbw.env;
480 const std::string& strFile = dbw.strFile;
481 while (true) {
483 LOCK(env->cs_db);
484 if (!env->mapFileUseCount.count(strFile) || env->mapFileUseCount[strFile] == 0) {
485 // Flush log data to the dat file
486 env->CloseDb(strFile);
487 env->CheckpointLSN(strFile);
488 env->mapFileUseCount.erase(strFile);
490 bool fSuccess = true;
491 LogPrintf("CDB::Rewrite: Rewriting %s...\n", strFile);
492 std::string strFileRes = strFile + ".rewrite";
493 { // surround usage of db with extra {}
494 CDB db(dbw, "r");
495 Db* pdbCopy = new Db(env->dbenv, 0);
497 int ret = pdbCopy->open(nullptr, // Txn pointer
498 strFileRes.c_str(), // Filename
499 "main", // Logical db name
500 DB_BTREE, // Database type
501 DB_CREATE, // Flags
503 if (ret > 0) {
504 LogPrintf("CDB::Rewrite: Can't create database file %s\n", strFileRes);
505 fSuccess = false;
508 Dbc* pcursor = db.GetCursor();
509 if (pcursor)
510 while (fSuccess) {
511 CDataStream ssKey(SER_DISK, CLIENT_VERSION);
512 CDataStream ssValue(SER_DISK, CLIENT_VERSION);
513 int ret1 = db.ReadAtCursor(pcursor, ssKey, ssValue);
514 if (ret1 == DB_NOTFOUND) {
515 pcursor->close();
516 break;
517 } else if (ret1 != 0) {
518 pcursor->close();
519 fSuccess = false;
520 break;
522 if (pszSkip &&
523 strncmp(ssKey.data(), pszSkip, std::min(ssKey.size(), strlen(pszSkip))) == 0)
524 continue;
525 if (strncmp(ssKey.data(), "\x07version", 8) == 0) {
526 // Update version:
527 ssValue.clear();
528 ssValue << CLIENT_VERSION;
530 Dbt datKey(ssKey.data(), ssKey.size());
531 Dbt datValue(ssValue.data(), ssValue.size());
532 int ret2 = pdbCopy->put(nullptr, &datKey, &datValue, DB_NOOVERWRITE);
533 if (ret2 > 0)
534 fSuccess = false;
536 if (fSuccess) {
537 db.Close();
538 env->CloseDb(strFile);
539 if (pdbCopy->close(0))
540 fSuccess = false;
541 } else {
542 pdbCopy->close(0);
544 delete pdbCopy;
546 if (fSuccess) {
547 Db dbA(env->dbenv, 0);
548 if (dbA.remove(strFile.c_str(), nullptr, 0))
549 fSuccess = false;
550 Db dbB(env->dbenv, 0);
551 if (dbB.rename(strFileRes.c_str(), nullptr, strFile.c_str(), 0))
552 fSuccess = false;
554 if (!fSuccess)
555 LogPrintf("CDB::Rewrite: Failed to rewrite database file %s\n", strFileRes);
556 return fSuccess;
559 MilliSleep(100);
564 void CDBEnv::Flush(bool fShutdown)
566 int64_t nStart = GetTimeMillis();
567 // Flush log data to the actual data file on all files that are not in use
568 LogPrint(BCLog::DB, "CDBEnv::Flush: Flush(%s)%s\n", fShutdown ? "true" : "false", fDbEnvInit ? "" : " database not started");
569 if (!fDbEnvInit)
570 return;
572 LOCK(cs_db);
573 std::map<std::string, int>::iterator mi = mapFileUseCount.begin();
574 while (mi != mapFileUseCount.end()) {
575 std::string strFile = (*mi).first;
576 int nRefCount = (*mi).second;
577 LogPrint(BCLog::DB, "CDBEnv::Flush: Flushing %s (refcount = %d)...\n", strFile, nRefCount);
578 if (nRefCount == 0) {
579 // Move log data to the dat file
580 CloseDb(strFile);
581 LogPrint(BCLog::DB, "CDBEnv::Flush: %s checkpoint\n", strFile);
582 dbenv->txn_checkpoint(0, 0, 0);
583 LogPrint(BCLog::DB, "CDBEnv::Flush: %s detach\n", strFile);
584 if (!fMockDb)
585 dbenv->lsn_reset(strFile.c_str(), 0);
586 LogPrint(BCLog::DB, "CDBEnv::Flush: %s closed\n", strFile);
587 mapFileUseCount.erase(mi++);
588 } else
589 mi++;
591 LogPrint(BCLog::DB, "CDBEnv::Flush: Flush(%s)%s took %15dms\n", fShutdown ? "true" : "false", fDbEnvInit ? "" : " database not started", GetTimeMillis() - nStart);
592 if (fShutdown) {
593 char** listp;
594 if (mapFileUseCount.empty()) {
595 dbenv->log_archive(&listp, DB_ARCH_REMOVE);
596 Close();
597 if (!fMockDb)
598 fs::remove_all(fs::path(strPath) / "database");
604 bool CDB::PeriodicFlush(CWalletDBWrapper& dbw)
606 if (dbw.IsDummy()) {
607 return true;
609 bool ret = false;
610 CDBEnv *env = dbw.env;
611 const std::string& strFile = dbw.strFile;
612 TRY_LOCK(bitdb.cs_db,lockDb);
613 if (lockDb)
615 // Don't do this if any databases are in use
616 int nRefCount = 0;
617 std::map<std::string, int>::iterator mit = env->mapFileUseCount.begin();
618 while (mit != env->mapFileUseCount.end())
620 nRefCount += (*mit).second;
621 mit++;
624 if (nRefCount == 0)
626 boost::this_thread::interruption_point();
627 std::map<std::string, int>::iterator mi = env->mapFileUseCount.find(strFile);
628 if (mi != env->mapFileUseCount.end())
630 LogPrint(BCLog::DB, "Flushing %s\n", strFile);
631 int64_t nStart = GetTimeMillis();
633 // Flush wallet file so it's self contained
634 env->CloseDb(strFile);
635 env->CheckpointLSN(strFile);
637 env->mapFileUseCount.erase(mi++);
638 LogPrint(BCLog::DB, "Flushed %s %dms\n", strFile, GetTimeMillis() - nStart);
639 ret = true;
644 return ret;
647 bool CWalletDBWrapper::Rewrite(const char* pszSkip)
649 return CDB::Rewrite(*this, pszSkip);
652 bool CWalletDBWrapper::Backup(const std::string& strDest)
654 if (IsDummy()) {
655 return false;
657 while (true)
660 LOCK(env->cs_db);
661 if (!env->mapFileUseCount.count(strFile) || env->mapFileUseCount[strFile] == 0)
663 // Flush log data to the dat file
664 env->CloseDb(strFile);
665 env->CheckpointLSN(strFile);
666 env->mapFileUseCount.erase(strFile);
668 // Copy wallet file
669 fs::path pathSrc = GetDataDir() / strFile;
670 fs::path pathDest(strDest);
671 if (fs::is_directory(pathDest))
672 pathDest /= strFile;
674 try {
675 fs::copy_file(pathSrc, pathDest, fs::copy_option::overwrite_if_exists);
676 LogPrintf("copied %s to %s\n", strFile, pathDest.string());
677 return true;
678 } catch (const fs::filesystem_error& e) {
679 LogPrintf("error copying %s to %s - %s\n", strFile, pathDest.string(), e.what());
680 return false;
684 MilliSleep(100);
688 void CWalletDBWrapper::Flush(bool shutdown)
690 if (!IsDummy()) {
691 env->Flush(shutdown);