Line data Source code
1 : // Copyright (c) 2009-2010 Satoshi Nakamoto
2 : // Copyright (c) 2009-2021 The Bitcoin developers
3 : // Copyright (c) 2019-2021 The PIVX Core developers
4 : // Distributed under the MIT/X11 software license, see the accompanying
5 : // file COPYING or http://www.opensource.org/licenses/mit-license.php.
6 :
7 : #include "db.h"
8 :
9 : #include "addrman.h"
10 : #include "guiinterfaceutil.h"
11 : #include "hash.h"
12 : #include "protocol.h"
13 : #include "util/system.h"
14 : #include "utilstrencodings.h"
15 : #include "wallet/walletutil.h"
16 :
17 : #include <stdint.h>
18 :
19 : #ifndef WIN32
20 : #include <sys/stat.h>
21 : #endif
22 :
23 : #include <boost/thread.hpp>
24 :
25 :
26 : namespace {
27 :
28 : //! Make sure database has a unique fileid within the environment. If it
29 : //! doesn't, throw an error. BDB caches do not work properly when more than one
30 : //! open database has the same fileid (values written to one database may show
31 : //! up in reads to other databases).
32 : //!
33 : //! BerkeleyDB generates unique fileids by default
34 : //! (https://docs.oracle.com/cd/E17275_01/html/programmer_reference/program_copy.html),
35 : //! so bitcoin should never create different databases with the same fileid, but
36 : //! this error can be triggered if users manually copy database files.
37 1440 : void CheckUniqueFileid(const BerkeleyEnvironment& env, const std::string& filename, Db& db, WalletDatabaseFileId& fileid)
38 : {
39 1440 : if (env.IsMock()) return;
40 :
41 1401 : int ret = db.get_mpf()->get_fileid(fileid.value);
42 1401 : if (ret != 0) {
43 0 : throw std::runtime_error(strprintf("BerkeleyBatch: Can't open database %s (get_fileid failed with %d)", filename, ret));
44 : }
45 :
46 2746 : for (const auto& item : env.m_fileids) {
47 1346 : if (fileid == item.second && &fileid != &item.second) {
48 2 : throw std::runtime_error(strprintf("BerkeleyBatch: Can't open database %s (duplicates fileid %s from %s)", filename,
49 3 : HexStr(item.second.value), item.first));
50 : }
51 : }
52 : }
53 :
54 : RecursiveMutex cs_db;
55 : std::map<std::string, BerkeleyEnvironment> g_dbenvs; //!< Map from directory name to open db environment.
56 : } // namespace
57 :
58 1346 : bool WalletDatabaseFileId::operator==(const WalletDatabaseFileId& rhs) const
59 : {
60 1346 : return memcmp(value, &rhs.value, sizeof(value)) == 0;
61 : }
62 :
63 1160 : BerkeleyEnvironment* GetWalletEnv(const fs::path& wallet_path, std::string& database_filename)
64 : {
65 1160 : fs::path env_directory;
66 1160 : if (fs::is_regular_file(wallet_path)) {
67 : // Special case for backwards compatibility: if wallet path points to an
68 : // existing file, treat it as the path to a BDB data file in a parent
69 : // directory that also contains BDB log files.
70 24 : env_directory = wallet_path.parent_path();
71 12 : database_filename = wallet_path.filename().string();
72 : } else {
73 : // Normal case: Interpret wallet path as a directory path containing
74 : // data and log files.
75 1148 : env_directory = wallet_path;
76 1148 : database_filename = "wallet.dat";
77 : }
78 2320 : LOCK(cs_db);
79 : // Note: An unused temporary BerkeleyEnvironment object may be created inside the
80 : // emplace function if the key already exists. This is a little inefficient,
81 : // but not a big concern since the map will be changed in the future to hold
82 : // pointers instead of objects, anyway.
83 2320 : return &g_dbenvs.emplace(std::piecewise_construct, std::forward_as_tuple(env_directory.string()), std::forward_as_tuple(env_directory)).first->second;
84 : }
85 :
86 : //
87 : // BerkeleyBatch
88 : //
89 :
90 1568 : void BerkeleyEnvironment::Close()
91 : {
92 1568 : if (!fDbEnvInit)
93 1149 : return;
94 :
95 419 : fDbEnvInit = false;
96 :
97 830 : for (auto& db : mapDb) {
98 411 : auto count = mapFileUseCount.find(db.first);
99 411 : assert(count == mapFileUseCount.end() || count->second == 0);
100 411 : if (db.second) {
101 39 : db.second->close(0);
102 39 : delete db.second;
103 39 : db.second = nullptr;
104 : }
105 : }
106 :
107 419 : int ret = dbenv->close(0);
108 419 : if (ret != 0)
109 0 : LogPrintf("%s: Error %d closing database environment: %s\n", __func__, ret, DbEnv::strerror(ret));
110 419 : if (!fMockDb)
111 380 : DbEnv((u_int32_t)0).remove(strPath.c_str(), 0);
112 : }
113 :
114 1206 : void BerkeleyEnvironment::Reset()
115 : {
116 1206 : dbenv.reset(new DbEnv(DB_CXX_NO_EXCEPTIONS));
117 1206 : fDbEnvInit = false;
118 1206 : fMockDb = false;
119 1206 : }
120 :
121 1160 : BerkeleyEnvironment::BerkeleyEnvironment(const fs::path& dir_path) : strPath(dir_path.string())
122 : {
123 1160 : Reset();
124 1160 : }
125 :
126 2320 : BerkeleyEnvironment::~BerkeleyEnvironment()
127 : {
128 1160 : Close();
129 1160 : }
130 :
131 444229 : bool BerkeleyEnvironment::Open(bool retry)
132 : {
133 444229 : if (fDbEnvInit)
134 : return true;
135 :
136 383 : boost::this_thread::interruption_point();
137 :
138 444611 : fs::path pathIn = strPath;
139 383 : TryCreateDirectories(pathIn);
140 382 : if (!LockDirectory(pathIn, ".walletlock")) {
141 2 : LogPrintf("Cannot obtain a lock on wallet directory %s. Another instance of PIVX may be using it.\n", strPath);
142 : return false;
143 : }
144 :
145 763 : fs::path pathLogDir = pathIn / "database";
146 380 : TryCreateDirectories(pathLogDir);
147 760 : fs::path pathErrorFile = pathIn / "db.log";
148 380 : LogPrintf("BerkeleyEnvironment::Open: LogDir=%s ErrorFile=%s\n", pathLogDir.string(), pathErrorFile.string());
149 :
150 380 : unsigned int nEnvFlags = 0;
151 380 : if (gArgs.GetBoolArg("-privdb", DEFAULT_WALLET_PRIVDB))
152 380 : nEnvFlags |= DB_PRIVATE;
153 :
154 380 : dbenv->set_lg_dir(pathLogDir.string().c_str());
155 380 : dbenv->set_cachesize(0, 0x100000, 1); // 1 MiB should be enough for just the wallet
156 380 : dbenv->set_lg_bsize(0x10000);
157 380 : dbenv->set_lg_max(1048576);
158 380 : dbenv->set_lk_max_locks(40000);
159 380 : dbenv->set_lk_max_objects(40000);
160 380 : dbenv->set_errfile(fsbridge::fopen(pathErrorFile, "a")); /// debug
161 380 : dbenv->set_flags(DB_AUTO_COMMIT, 1);
162 380 : dbenv->set_flags(DB_TXN_WRITE_NOSYNC, 1);
163 380 : dbenv->log_set_config(DB_LOG_AUTO_REMOVE, 1);
164 380 : int ret = dbenv->open(strPath.c_str(),
165 : DB_CREATE |
166 : DB_INIT_LOCK |
167 : DB_INIT_LOG |
168 : DB_INIT_MPOOL |
169 : DB_INIT_TXN |
170 : DB_THREAD |
171 : DB_RECOVER |
172 : nEnvFlags,
173 380 : S_IRUSR | S_IWUSR);
174 380 : if (ret != 0) {
175 0 : LogPrintf("%s: Error %d opening database environment: %s\n", __func__, ret, DbEnv::strerror(ret));
176 0 : int ret2 = dbenv->close(0);
177 0 : if (ret2 != 0) {
178 0 : LogPrintf("%s: Error %d closing failed database environment: %s\n", __func__, ret2, DbEnv::strerror(ret2));
179 : }
180 0 : Reset();
181 0 : if (retry) {
182 : // try moving the database env out of the way
183 0 : fs::path pathDatabaseBak = pathIn / strprintf("database.%d.bak", GetTime());
184 0 : try {
185 0 : fs::rename(pathLogDir, pathDatabaseBak);
186 0 : LogPrintf("Moved old %s to %s. Retrying.\n", pathLogDir.string(), pathDatabaseBak.string());
187 0 : } catch (const fs::filesystem_error&) {
188 : // failure is ok (well, not really, but it's not worse than what we started with)
189 : }
190 : // try opening it again one more time
191 0 : if (!Open(false /* retry */)) {
192 : // if it still fails, it probably means we can't even create the database env
193 0 : return false;
194 : }
195 : } else {
196 : return false;
197 : }
198 : }
199 :
200 380 : fDbEnvInit = true;
201 380 : fMockDb = false;
202 380 : return true;
203 : }
204 :
205 39 : void BerkeleyEnvironment::MakeMock()
206 : {
207 39 : if (fDbEnvInit)
208 0 : throw std::runtime_error("BerkeleyEnvironment::MakeMock : Already initialized");
209 :
210 39 : boost::this_thread::interruption_point();
211 :
212 39 : LogPrint(BCLog::DB, "BerkeleyEnvironment::MakeMock\n");
213 :
214 39 : dbenv->set_cachesize(1, 0, 1);
215 39 : dbenv->set_lg_bsize(10485760 * 4);
216 39 : dbenv->set_lg_max(10485760);
217 39 : dbenv->set_lk_max_locks(10000);
218 39 : dbenv->set_lk_max_objects(10000);
219 39 : dbenv->set_flags(DB_AUTO_COMMIT, 1);
220 39 : dbenv->log_set_config(DB_LOG_IN_MEMORY, 1);
221 39 : int ret = dbenv->open(nullptr,
222 : DB_CREATE |
223 : DB_INIT_LOCK |
224 : DB_INIT_LOG |
225 : DB_INIT_MPOOL |
226 : DB_INIT_TXN |
227 : DB_THREAD |
228 : DB_PRIVATE,
229 39 : S_IRUSR | S_IWUSR);
230 39 : if (ret > 0)
231 0 : throw std::runtime_error(strprintf("BerkeleyEnvironment::MakeMock : Error %d opening database environment.", ret));
232 :
233 39 : fDbEnvInit = true;
234 39 : fMockDb = true;
235 39 : }
236 :
237 173 : BerkeleyEnvironment::VerifyResult BerkeleyEnvironment::Verify(const std::string& strFile, recoverFunc_type recoverFunc, std::string& out_backup_filename)
238 : {
239 346 : LOCK(cs_db);
240 173 : assert(mapFileUseCount.count(strFile) == 0);
241 :
242 346 : Db db(dbenv.get(), 0);
243 173 : int result = db.verify(strFile.c_str(), nullptr, nullptr, 0);
244 173 : if (result == 0)
245 : return VERIFY_OK;
246 0 : else if (recoverFunc == nullptr)
247 : return RECOVER_FAIL;
248 :
249 : // Try to recover:
250 0 : bool fRecovered = (*recoverFunc)(fs::path(strPath) / strFile, out_backup_filename);
251 0 : return (fRecovered ? RECOVER_OK : RECOVER_FAIL);
252 : }
253 :
254 0 : bool BerkeleyBatch::Recover(const fs::path& file_path, void *callbackDataIn, bool (*recoverKVcallback)(void* callbackData, CDataStream ssKey, CDataStream ssValue), std::string& newFilename)
255 : {
256 0 : std::string filename;
257 0 : BerkeleyEnvironment* env = GetWalletEnv(file_path, filename);
258 :
259 : // Recovery procedure:
260 : // move wallet file to walletfilename.timestamp.bak
261 : // Call Salvage with fAggressive=true to
262 : // get as much data as possible.
263 : // Rewrite salvaged data to fresh wallet file
264 : // Set -rescan so any missing transactions will be
265 : // found.
266 0 : int64_t now = GetTime();
267 0 : newFilename = strprintf("%s.%d.bak", filename, now);
268 :
269 0 : int result = env->dbenv->dbrename(nullptr, filename.c_str(), nullptr,
270 0 : newFilename.c_str(), DB_AUTO_COMMIT);
271 0 : if (result == 0) {
272 0 : LogPrintf("Renamed %s to %s\n", filename, newFilename);
273 : } else {
274 0 : LogPrintf("Failed to rename %s to %s\n", filename, newFilename);
275 : return false;
276 : }
277 :
278 0 : std::vector<BerkeleyEnvironment::KeyValPair> salvagedData;
279 0 : bool fSuccess = env->Salvage(newFilename, true, salvagedData);
280 0 : if (salvagedData.empty())
281 : {
282 0 : LogPrintf("Salvage(aggressive) found no records in %s.\n", newFilename);
283 : return false;
284 : }
285 0 : LogPrintf("Salvage(aggressive) found %u records\n", salvagedData.size());
286 :
287 0 : std::unique_ptr<Db> pdbCopy = std::make_unique<Db>(env->dbenv.get(), 0);
288 0 : int ret = pdbCopy->open(nullptr, // Txn pointer
289 : filename.c_str(), // Filename
290 : "main", // Logical db name
291 : DB_BTREE, // Database type
292 : DB_CREATE, // Flags
293 0 : 0);
294 0 : if (ret > 0) {
295 0 : LogPrintf("Cannot create database file %s\n", filename);
296 0 : pdbCopy->close(0);
297 : return false;
298 : }
299 :
300 0 : DbTxn* ptxn = env->TxnBegin();
301 0 : for (BerkeleyEnvironment::KeyValPair& row : salvagedData)
302 : {
303 0 : if (recoverKVcallback)
304 : {
305 0 : CDataStream ssKey(row.first, SER_DISK, CLIENT_VERSION);
306 0 : CDataStream ssValue(row.second, SER_DISK, CLIENT_VERSION);
307 0 : if (!(*recoverKVcallback)(callbackDataIn, ssKey, ssValue))
308 0 : continue;
309 : }
310 0 : Dbt datKey(&row.first[0], row.first.size());
311 0 : Dbt datValue(&row.second[0], row.second.size());
312 0 : int ret2 = pdbCopy->put(ptxn, &datKey, &datValue, DB_NOOVERWRITE);
313 0 : if (ret2 > 0)
314 0 : fSuccess = false;
315 : }
316 0 : ptxn->commit(0);
317 0 : pdbCopy->close(0);
318 :
319 : return fSuccess;
320 : }
321 :
322 377 : bool BerkeleyBatch::VerifyEnvironment(const fs::path& file_path, std::string& errorStr)
323 : {
324 753 : std::string walletFile;
325 377 : BerkeleyEnvironment* env = GetWalletEnv(file_path, walletFile);
326 754 : fs::path walletDir = env->Directory();
327 :
328 378 : LogPrintf("Using BerkeleyDB version %s\n", DbEnv::version(0, 0, 0));
329 377 : LogPrintf("Using wallet %s\n", walletFile);
330 :
331 : // Wallet file must be a plain filename without a directory
332 754 : fs::path wallet_file_path(walletFile);
333 377 : if (walletFile != wallet_file_path.filename().string())
334 0 : return UIError(strprintf(_("Wallet %s resides outside data directory %s"), walletFile, walletDir.string()));
335 :
336 377 : if (!env->Open(true /* retry */)) {
337 3 : errorStr = strprintf(_("Error initializing wallet database environment %s!"), walletDir);
338 2 : return false;
339 : }
340 :
341 : return true;
342 : }
343 :
344 374 : bool BerkeleyBatch::VerifyDatabaseFile(const fs::path& file_path, std::string& warningStr, std::string& errorStr, BerkeleyEnvironment::recoverFunc_type recoverFunc)
345 : {
346 748 : std::string walletFile;
347 374 : BerkeleyEnvironment* env = GetWalletEnv(file_path, walletFile);
348 748 : fs::path walletDir = env->Directory();
349 :
350 1496 : if (fs::exists(walletDir / walletFile))
351 : {
352 346 : std::string backup_filename;
353 173 : BerkeleyEnvironment::VerifyResult r = env->Verify(walletFile, recoverFunc, backup_filename);
354 173 : if (r == BerkeleyEnvironment::RECOVER_OK)
355 : {
356 0 : warningStr = strprintf(_("Warning: Wallet file corrupt, data salvaged!"
357 : " Original %s saved as %s in %s; if"
358 : " your balance or transactions are incorrect you should"
359 : " restore from a backup."),
360 0 : walletFile, backup_filename, walletDir);
361 : }
362 173 : if (r == BerkeleyEnvironment::RECOVER_FAIL) {
363 0 : errorStr = strprintf(_("%s corrupt, salvage failed"), walletFile);
364 0 : return false;
365 : }
366 : }
367 : // also return true if files does not exists
368 : return true;
369 : }
370 :
371 : /* End of headers, beginning of key/value data */
372 : static const char *HEADER_END = "HEADER=END";
373 : /* End of key/value data */
374 : static const char *DATA_END = "DATA=END";
375 :
376 0 : bool BerkeleyEnvironment::Salvage(const std::string& strFile, bool fAggressive, std::vector<BerkeleyEnvironment::KeyValPair>& vResult)
377 : {
378 0 : LOCK(cs_db);
379 0 : assert(mapFileUseCount.count(strFile) == 0);
380 :
381 0 : u_int32_t flags = DB_SALVAGE;
382 0 : if (fAggressive)
383 0 : flags |= DB_AGGRESSIVE;
384 :
385 0 : std::stringstream strDump;
386 :
387 0 : Db db(dbenv.get(), 0);
388 0 : int result = db.verify(strFile.c_str(), nullptr, &strDump, flags);
389 0 : if (result == DB_VERIFY_BAD) {
390 0 : LogPrintf("BerkeleyEnvironment::Salvage : Database salvage found errors, all data may not be recoverable.\n");
391 0 : if (!fAggressive) {
392 0 : LogPrintf("BerkeleyEnvironment::Salvage : Rerun with aggressive mode to ignore errors and continue.\n");
393 : return false;
394 : }
395 : }
396 0 : if (result != 0 && result != DB_VERIFY_BAD) {
397 0 : LogPrintf("BerkeleyEnvironment::Salvage : Database salvage failed with result %d.\n", result);
398 : return false;
399 : }
400 :
401 : // Format of bdb dump is ascii lines:
402 : // header lines...
403 : // HEADER=END
404 : // hexadecimal key
405 : // hexadecimal value
406 : // ... repeated
407 : // DATA=END
408 :
409 0 : std::string strLine;
410 0 : while (!strDump.eof() && strLine != HEADER_END)
411 0 : getline(strDump, strLine); // Skip past header
412 :
413 0 : std::string keyHex, valueHex;
414 0 : while (!strDump.eof() && keyHex != DATA_END) {
415 0 : getline(strDump, keyHex);
416 0 : if (keyHex != DATA_END) {
417 0 : if (strDump.eof())
418 : break;
419 0 : getline(strDump, valueHex);
420 0 : if (valueHex == DATA_END) {
421 0 : LogPrintf("BerkeleyEnvironment::Salvage: WARNING: Number of keys in data does not match number of values.\n");
422 : break;
423 : }
424 0 : vResult.emplace_back(ParseHex(keyHex), ParseHex(valueHex));
425 : }
426 : }
427 :
428 0 : if (keyHex != DATA_END) {
429 0 : LogPrintf("BerkeleyEnvironment::Salvage: WARNING: Unexpected end of file while reading salvage output.\n");
430 : return false;
431 : }
432 :
433 0 : return (result == 0);
434 : }
435 :
436 :
437 1107 : void BerkeleyEnvironment::CheckpointLSN(const std::string& strFile)
438 : {
439 1107 : dbenv->txn_checkpoint(0, 0, 0);
440 1107 : if (fMockDb)
441 : return;
442 1107 : dbenv->lsn_reset(strFile.c_str(), 0);
443 : }
444 :
445 :
446 443975 : BerkeleyBatch::BerkeleyBatch(BerkeleyDatabase& database, const char* pszMode, bool fFlushOnCloseIn) : pdb(nullptr), activeTxn(nullptr)
447 : {
448 443974 : fReadOnly = (!strchr(pszMode, '+') && !strchr(pszMode, 'w'));
449 443974 : fFlushOnClose = fFlushOnCloseIn;
450 443974 : env = database.env;
451 443974 : if (database.IsDummy()) {
452 : return;
453 : }
454 443845 : const std::string &strFilename = database.strFile;
455 :
456 443845 : bool fCreate = strchr(pszMode, 'c') != nullptr;
457 443845 : unsigned int nFlags = DB_THREAD;
458 443845 : if (fCreate)
459 409 : nFlags |= DB_CREATE;
460 :
461 443845 : {
462 887690 : LOCK(cs_db);
463 443845 : if (!env->Open(false /* retry */))
464 0 : throw std::runtime_error("BerkeleyBatch: Failed to open database environment.");
465 :
466 443845 : pdb = env->mapDb[strFilename];
467 443845 : if (pdb == nullptr) {
468 1309 : int ret;
469 2618 : std::unique_ptr<Db> pdb_temp = std::make_unique<Db>(env->dbenv.get(), 0);
470 :
471 1309 : bool fMockDb = env->IsMock();
472 1309 : if (fMockDb) {
473 39 : DbMpoolFile* mpf = pdb_temp->get_mpf();
474 39 : ret = mpf->set_flags(DB_MPOOL_NOFILE, 1);
475 39 : if (ret != 0) {
476 0 : throw std::runtime_error(strprintf("BerkeleyBatch: Failed to configure for no temp file backing for database %s", strFilename));
477 : }
478 : }
479 :
480 2618 : ret = pdb_temp->open(nullptr, // Txn pointer
481 1270 : fMockDb ? nullptr : strFilename.c_str(), // Filename
482 39 : fMockDb ? strFilename.c_str() : "main", // Logical db name
483 : DB_BTREE, // Database type
484 : nFlags, // Flags
485 1309 : 0);
486 :
487 1309 : if (ret != 0) {
488 0 : throw std::runtime_error(strprintf("BerkeleyBatch: Error %d, can't open database %s", ret, strFilename));
489 : }
490 :
491 : // Call CheckUniqueFileid on the containing BDB environment to
492 : // avoid BDB data consistency bugs that happen when different data
493 : // files in the same environment have the same fileid.
494 : //
495 : // Also call CheckUniqueFileid on all the other g_dbenvs to prevent
496 : // PIVX from opening the same data file through another
497 : // environment when the file is referenced through equivalent but
498 : // not obviously identical symlinked or hard linked or bind mounted
499 : // paths. In the future a more relaxed check for equal inode and
500 : // device ids could be done instead, which would allow opening
501 : // different backup copies of a wallet at the same time. Maybe even
502 : // more ideally, an exclusive lock for accessing the database could
503 : // be implemented, so no equality checks are needed at all. (Newer
504 : // versions of BDB have an set_lk_exclusive method for this
505 : // purpose, but the older version we use does not.)
506 2748 : for (const auto& env : g_dbenvs) {
507 1440 : CheckUniqueFileid(env.second, strFilename, *pdb_temp, this->env->m_fileids[strFilename]);
508 : }
509 :
510 1308 : pdb = pdb_temp.release();
511 1308 : env->mapDb[strFilename] = pdb;
512 :
513 1874 : if (fCreate && !Exists(std::string("version"))) {
514 241 : bool fTmp = fReadOnly;
515 241 : fReadOnly = false;
516 241 : WriteVersion(CLIENT_VERSION);
517 241 : fReadOnly = fTmp;
518 : }
519 : }
520 443844 : ++env->mapFileUseCount[strFilename];
521 443844 : strFile = strFilename;
522 : }
523 : }
524 :
525 47657 : void BerkeleyBatch::Flush()
526 : {
527 47657 : if (activeTxn)
528 : return;
529 :
530 : // Flush database activity from memory pool to disk log
531 47657 : unsigned int nMinutes = 0;
532 47657 : if (fReadOnly)
533 7 : nMinutes = 1;
534 :
535 47664 : env->dbenv->txn_checkpoint(nMinutes ? gArgs.GetArg("-dblogsize", 100) * 1024 : 0, nMinutes, 0);
536 : }
537 :
538 531015 : void BerkeleyDatabase::IncrementUpdateCounter()
539 : {
540 531015 : ++nUpdateCounter;
541 531015 : }
542 :
543 443980 : void BerkeleyBatch::Close()
544 : {
545 443980 : if (!pdb)
546 : return;
547 443844 : if (activeTxn)
548 0 : activeTxn->abort();
549 443844 : activeTxn = nullptr;
550 443844 : pdb = nullptr;
551 :
552 443844 : if (fFlushOnClose)
553 47657 : Flush();
554 :
555 443844 : {
556 443844 : LOCK(cs_db);
557 443844 : --env->mapFileUseCount[strFile];
558 : }
559 443844 : env->m_db_in_use.notify_all();
560 : }
561 :
562 1286 : void BerkeleyEnvironment::CloseDb(const std::string& strFile)
563 : {
564 1286 : {
565 1286 : LOCK(cs_db);
566 1286 : if (mapDb[strFile] != nullptr) {
567 : // Close the database handle
568 1269 : Db* pdb = mapDb[strFile];
569 1269 : pdb->close(0);
570 1269 : delete pdb;
571 1269 : mapDb[strFile] = nullptr;
572 : }
573 : }
574 1286 : }
575 :
576 7 : void BerkeleyEnvironment::ReloadDbEnv()
577 : {
578 : // Make sure that no Db's are in use
579 7 : AssertLockNotHeld(cs_db);
580 7 : std::unique_lock<RecursiveMutex> lock(cs_db);
581 7 : m_db_in_use.wait(lock, [this](){
582 14 : for (auto& count : mapFileUseCount) {
583 7 : if (count.second > 0) return false;
584 : }
585 7 : return true;
586 : });
587 :
588 14 : std::vector<std::string> filenames;
589 14 : for (auto it : mapDb) {
590 7 : filenames.push_back(it.first);
591 : }
592 : // Close the individual Db's
593 14 : for (const std::string& filename : filenames) {
594 7 : CloseDb(filename);
595 : }
596 : // Reset the environment
597 7 : Flush(true); // This will flush and close the environment
598 7 : Reset();
599 7 : Open(true);
600 7 : }
601 :
602 7 : bool BerkeleyBatch::Rewrite(BerkeleyDatabase& database, const char* pszSkip)
603 : {
604 7 : if (database.IsDummy()) {
605 : return true;
606 : }
607 7 : BerkeleyEnvironment *env = database.env;
608 7 : const std::string& strFile = database.strFile;
609 7 : while (true) {
610 7 : {
611 7 : LOCK(cs_db);
612 14 : if (!env->mapFileUseCount.count(strFile) || env->mapFileUseCount[strFile] == 0) {
613 : // Flush log data to the dat file
614 7 : env->CloseDb(strFile);
615 7 : env->CheckpointLSN(strFile);
616 7 : env->mapFileUseCount.erase(strFile);
617 :
618 7 : bool fSuccess = true;
619 7 : LogPrintf("BerkeleyBatch::Rewrite : Rewriting %s...\n", strFile);
620 14 : std::string strFileRes = strFile + ".rewrite";
621 7 : { // surround usage of db with extra {}
622 7 : BerkeleyBatch db(database, "r");
623 14 : std::unique_ptr<Db> pdbCopy = std::make_unique<Db>(env->dbenv.get(), 0);
624 :
625 7 : int ret = pdbCopy->open(nullptr, // Txn pointer
626 : strFileRes.c_str(), // Filename
627 : "main", // Logical db name
628 : DB_BTREE, // Database type
629 : DB_CREATE, // Flags
630 7 : 0);
631 7 : if (ret > 0) {
632 0 : LogPrintf("BerkeleyBatch::Rewrite : Can't create database file %s\n", strFileRes);
633 : fSuccess = false;
634 : }
635 :
636 7 : Dbc* pcursor = db.GetCursor();
637 7 : if (pcursor)
638 4044 : while (fSuccess) {
639 8081 : CDataStream ssKey(SER_DISK, CLIENT_VERSION);
640 4037 : CDataStream ssValue(SER_DISK, CLIENT_VERSION);
641 4044 : int ret1 = db.ReadAtCursor(pcursor, ssKey, ssValue);
642 4044 : if (ret1 == DB_NOTFOUND) {
643 7 : pcursor->close();
644 7 : break;
645 4037 : } else if (ret1 != 0) {
646 0 : pcursor->close();
647 : fSuccess = false;
648 : break;
649 : }
650 4037 : if (pszSkip &&
651 0 : strncmp(ssKey.data(), pszSkip, std::min(ssKey.size(), strlen(pszSkip))) == 0)
652 0 : continue;
653 4037 : if (strncmp(ssKey.data(), "\x07version", 8) == 0) {
654 : // Update version:
655 7 : ssValue.clear();
656 7 : ssValue << CLIENT_VERSION;
657 : }
658 8074 : Dbt datKey(ssKey.data(), ssKey.size());
659 8074 : Dbt datValue(ssValue.data(), ssValue.size());
660 4037 : int ret2 = pdbCopy->put(nullptr, &datKey, &datValue, DB_NOOVERWRITE);
661 4037 : if (ret2 > 0)
662 0 : fSuccess = false;
663 : }
664 7 : if (fSuccess) {
665 7 : db.Close();
666 7 : env->CloseDb(strFile);
667 7 : if (pdbCopy->close(0))
668 0 : fSuccess = false;
669 : } else {
670 0 : pdbCopy->close(0);
671 : }
672 : }
673 7 : if (fSuccess) {
674 14 : Db dbA(env->dbenv.get(), 0);
675 7 : if (dbA.remove(strFile.c_str(), nullptr, 0))
676 0 : fSuccess = false;
677 14 : Db dbB(env->dbenv.get(), 0);
678 7 : if (dbB.rename(strFileRes.c_str(), nullptr, strFile.c_str(), 0))
679 0 : fSuccess = false;
680 : }
681 7 : if (!fSuccess)
682 0 : LogPrintf("BerkeleyBatch::Rewrite : Failed to rewrite database file %s\n", strFileRes);
683 7 : return fSuccess;
684 : }
685 : }
686 0 : MilliSleep(100);
687 0 : }
688 : }
689 :
690 :
691 733 : void BerkeleyEnvironment::Flush(bool fShutdown)
692 : {
693 733 : int64_t nStart = GetTimeMillis();
694 : // Flush log data to the actual data file on all files that are not in use
695 1093 : LogPrint(BCLog::DB, "BerkeleyEnvironment::Flush : Flush(%s)%s\n", fShutdown ? "true" : "false", fDbEnvInit ? "" : " database not started");
696 733 : if (!fDbEnvInit)
697 : return;
698 731 : {
699 1462 : LOCK(cs_db);
700 731 : std::map<std::string, int>::iterator mi = mapFileUseCount.begin();
701 896 : while (mi != mapFileUseCount.end()) {
702 330 : std::string strFile = (*mi).first;
703 165 : int nRefCount = (*mi).second;
704 165 : LogPrint(BCLog::DB, "BerkeleyEnvironment::Flush : Flushing %s (refcount = %d)...\n", strFile, nRefCount);
705 165 : if (nRefCount == 0) {
706 : // Move log data to the dat file
707 165 : CloseDb(strFile);
708 165 : LogPrint(BCLog::DB, "BerkeleyEnvironment::Flush: %s checkpoint\n", strFile);
709 165 : dbenv->txn_checkpoint(0, 0, 0);
710 165 : LogPrint(BCLog::DB, "BerkeleyEnvironment::Flush: %s detach\n", strFile);
711 165 : if (!fMockDb)
712 165 : dbenv->lsn_reset(strFile.c_str(), 0);
713 165 : LogPrint(BCLog::DB, "BerkeleyEnvironment::Flush: %s closed\n", strFile);
714 165 : mapFileUseCount.erase(mi++);
715 : } else
716 165 : mi++;
717 : }
718 1089 : LogPrint(BCLog::DB, "BerkeleyEnvironment::Flush : Flush(%s)%s took %15dms\n", fShutdown ? "true" : "false", fDbEnvInit ? "" : " database not started", GetTimeMillis() - nStart);
719 731 : if (fShutdown) {
720 369 : char** listp;
721 369 : if (mapFileUseCount.empty()) {
722 369 : dbenv->log_archive(&listp, DB_ARCH_REMOVE);
723 369 : Close();
724 369 : if (!fMockDb) {
725 1476 : fs::remove_all(fs::path(strPath) / "database");
726 : }
727 : }
728 : }
729 : }
730 : }
731 :
732 19004 : bool BerkeleyBatch::PeriodicFlush(BerkeleyDatabase& database)
733 : {
734 19004 : if (database.IsDummy()) {
735 : return true;
736 : }
737 19004 : bool ret = false;
738 19004 : BerkeleyEnvironment *env = database.env;
739 19004 : const std::string& strFile = database.strFile;
740 38008 : TRY_LOCK(cs_db, lockDb);
741 19004 : if (lockDb)
742 : {
743 : // Don't do this if any databases are in use
744 19004 : int nRefCount = 0;
745 19004 : std::map<std::string, int>::iterator mi = env->mapFileUseCount.begin();
746 19808 : while (mi != env->mapFileUseCount.end()) {
747 804 : nRefCount += (*mi).second;
748 19808 : mi++;
749 : }
750 :
751 19004 : if (nRefCount == 0) {
752 19004 : boost::this_thread::interruption_point();
753 19004 : std::map<std::string, int>::iterator _mi = env->mapFileUseCount.find(strFile);
754 19004 : if (_mi != env->mapFileUseCount.end()) {
755 804 : LogPrint(BCLog::DB, "Flushing %s\n", strFile);
756 804 : int64_t nStart = GetTimeMillis();
757 :
758 : // Flush wallet file so it's self contained
759 804 : env->CloseDb(strFile);
760 804 : env->CheckpointLSN(strFile);
761 :
762 804 : env->mapFileUseCount.erase(_mi++);
763 804 : LogPrint(BCLog::DB, "Flushed %s %dms\n", strFile, GetTimeMillis() - nStart);
764 : ret = true;
765 : }
766 : }
767 : }
768 :
769 19004 : return ret;
770 : }
771 :
772 7 : bool BerkeleyDatabase::Rewrite(const char* pszSkip)
773 : {
774 7 : return BerkeleyBatch::Rewrite(*this, pszSkip);
775 : }
776 :
777 296 : bool BerkeleyDatabase::Backup(const std::string& strDest)
778 : {
779 296 : if (IsDummy()) {
780 : return false;
781 : }
782 296 : while (true)
783 : {
784 296 : {
785 296 : LOCK(cs_db);
786 592 : if (!env->mapFileUseCount.count(strFile) || env->mapFileUseCount[strFile] == 0)
787 : {
788 : // Flush log data to the dat file
789 296 : env->CloseDb(strFile);
790 296 : env->CheckpointLSN(strFile);
791 296 : env->mapFileUseCount.erase(strFile);
792 :
793 : // Copy wallet file
794 1480 : fs::path pathSrc = env->Directory() / strFile;
795 592 : fs::path pathDest(strDest);
796 296 : if (fs::is_directory(pathDest))
797 2 : pathDest /= strFile;
798 :
799 296 : try {
800 296 : if (fs::equivalent(pathSrc, pathDest)) {
801 4 : LogPrintf("cannot backup to wallet source file %s\n", pathDest.string());
802 : return false;
803 : }
804 :
805 : #if BOOST_VERSION >= 107400
806 292 : fs::copy_file(pathSrc.c_str(), pathDest, fs::copy_options::overwrite_existing);
807 : #elif BOOST_VERSION >= 105800 /* BOOST_LIB_VERSION 1_58 */
808 : fs::copy_file(pathSrc.c_str(), pathDest, fs::copy_option::overwrite_if_exists);
809 : #endif
810 292 : LogPrintf("copied %s to %s\n", strFile, pathDest.string());
811 : return true;
812 0 : } catch (const fs::filesystem_error& e) {
813 0 : LogPrintf("error copying %s to %s - %s\n", strFile, pathDest.string(), fsbridge::get_filesystem_error_message(e));
814 0 : return false;
815 : }
816 : }
817 : }
818 0 : MilliSleep(100);
819 0 : }
820 : }
821 :
822 726 : void BerkeleyDatabase::Flush(bool shutdown)
823 : {
824 726 : if (!IsDummy()) {
825 726 : env->Flush(shutdown);
826 726 : if (shutdown) {
827 728 : LOCK(cs_db);
828 728 : g_dbenvs.erase(env->Directory().string());
829 364 : env = nullptr;
830 : } else {
831 : // TODO: To avoid g_dbenvs.erase erasing the environment prematurely after the
832 : // first database shutdown when multiple databases are open in the same
833 : // environment, should replace raw database `env` pointers with shared or weak
834 : // pointers, or else separate the database and environment shutdowns so
835 : // environments can be shut down after databases.
836 362 : env->m_fileids.erase(strFile);
837 : }
838 : }
839 726 : }
840 :
841 7 : void BerkeleyDatabase::ReloadDbEnv()
842 : {
843 7 : if (!IsDummy()) {
844 7 : env->ReloadDbEnv();
845 : }
846 7 : }
|
