Compare commits

..

1 Commits

Author SHA1 Message Date
Denis Angell
d9a8ef5aba featureLMDB 2024-10-04 06:32:14 +02:00
53 changed files with 3062 additions and 251 deletions

View File

@@ -417,6 +417,7 @@ target_sources (rippled PRIVATE
src/ripple/app/paths/impl/XRPEndpointStep.cpp
src/ripple/app/rdb/backend/detail/impl/Node.cpp
src/ripple/app/rdb/backend/detail/impl/Shard.cpp
src/ripple/app/rdb/backend/impl/LMDBDatabase.cpp
src/ripple/app/rdb/backend/impl/PostgresDatabase.cpp
src/ripple/app/rdb/backend/impl/SQLiteDatabase.cpp
src/ripple/app/rdb/impl/Download.cpp
@@ -927,6 +928,7 @@ if (tests)
src/test/nodestore/Backend_test.cpp
src/test/nodestore/Basics_test.cpp
src/test/nodestore/DatabaseShard_test.cpp
src/test/nodestore/LMDB_test.cpp
src/test/nodestore/Database_test.cpp
src/test/nodestore/Timing_test.cpp
src/test/nodestore/import_test.cpp

View File

@@ -44,7 +44,7 @@ else()
endif()
# TBD:
# Boost_USE_DEBUG_RUNTIME: When ON, uses Boost libraries linked against the
find_package(Boost 1.86 REQUIRED
find_package(Boost 1.70 REQUIRED
COMPONENTS
chrono
container

View File

@@ -0,0 +1,32 @@
include(FetchContent)
FetchContent_Declare(
lmdb
GIT_REPOSITORY https://github.com/LMDB/lmdb.git
GIT_TAG LMDB_0.9.31
)
FetchContent_MakeAvailable(lmdb)
add_library(lmdb STATIC
${lmdb_SOURCE_DIR}/libraries/liblmdb/mdb.c
${lmdb_SOURCE_DIR}/libraries/liblmdb/midl.c
)
target_include_directories(lmdb PUBLIC ${lmdb_SOURCE_DIR}/libraries/liblmdb)
install(TARGETS lmdb
EXPORT lmdbTargets
ARCHIVE DESTINATION lib
LIBRARY DESTINATION lib
RUNTIME DESTINATION bin
INCLUDES DESTINATION include
)
install(DIRECTORY ${lmdb_SOURCE_DIR}/libraries/liblmdb/
DESTINATION include
FILES_MATCHING PATTERN "*.h"
)
add_library (NIH::lmdb ALIAS lmdb)
target_link_libraries (ripple_libs INTERFACE NIH::lmdb)

View File

@@ -75,6 +75,7 @@ include(deps/gRPC)
include(deps/cassandra)
include(deps/Postgres)
include(deps/WasmEdge)
include(deps/LMDB)
###

View File

@@ -90,11 +90,11 @@ echo "-- Install Cmake 3.23.1 --" &&
pwd &&
( wget -nc -q https://github.com/Kitware/CMake/releases/download/v3.23.1/cmake-3.23.1-linux-x86_64.tar.gz; echo "" ) &&
tar -xzf cmake-3.23.1-linux-x86_64.tar.gz -C /hbb/ &&
echo "-- Install Boost 1.86.0 --" &&
echo "-- Install Boost 1.75.0 --" &&
pwd &&
( wget -nc -q https://boostorg.jfrog.io/artifactory/main/release/1.86.0/source/boost_1_86_0.tar.gz; echo "" ) &&
tar -xzf boost_1_86_0.tar.gz &&
cd boost_1_86_0 && ./bootstrap.sh && ./b2 link=static -j$3 && ./b2 install &&
( wget -nc -q https://boostorg.jfrog.io/artifactory/main/release/1.75.0/source/boost_1_75_0.tar.gz; echo "" ) &&
tar -xzf boost_1_75_0.tar.gz &&
cd boost_1_75_0 && ./bootstrap.sh && ./b2 link=static -j$3 && ./b2 install &&
cd ../ &&
echo "-- Install Protobuf 3.20.0 --" &&
pwd &&
@@ -127,9 +127,9 @@ echo "-- Build WasmEdge --" &&
cd WasmEdge-0.11.2 &&
( mkdir build; echo "" ) &&
cd build &&
export BOOST_ROOT="/usr/local/src/boost_1_86_0" &&
export BOOST_ROOT="/usr/local/src/boost_1_75_0" &&
export Boost_LIBRARY_DIRS="/usr/local/lib" &&
export BOOST_INCLUDEDIR="/usr/local/src/boost_1_86_0" &&
export BOOST_INCLUDEDIR="/usr/local/src/boost_1_75_0" &&
export PATH=`echo $PATH | sed -E "s/devtoolset-7/devtoolset-9/g"` &&
cmake .. \
-DCMAKE_BUILD_TYPE=Release \

View File

@@ -144,12 +144,4 @@ D686F2538F410C9D0D856788E98E3579595DAF7B38D38887F81ECAC934B06040 HooksUpdate1
86E83A7D2ECE3AD5FA87AB2195AE015C950469ABF0B72EAACED318F74886AE90 CryptoConditionsSuite
3C43D9A973AA4443EF3FC38E42DD306160FBFFDAB901CD8BAA15D09F2597EB87 NonFungibleTokensV1
0285B7E5E08E1A8E4C15636F0591D87F73CB6A7B6452A932AD72BBC8E5D1CBE3 fixNFTokenDirV1
36799EA497B1369B170805C078AEFE6188345F9B3E324C21E9CA3FF574E3C3D6 fixNFTokenNegOffer
4C499D17719BB365B69010A436B64FD1A82AAB199FC1CEB06962EBD01059FB09 fixXahauV1
215181D23BF5C173314B5FDB9C872C92DE6CC918483727DE037C0C13E7E6EE9D fixXahauV2
0D8BF22FF7570D58598D1EF19EBB6E142AD46E59A223FD3816262FBB69345BEA Remit
7CA0426E7F411D39BB014E57CD9E08F61DE1750F0D41FCD428D9FB80BB7596B0 ZeroB2M
4B8466415FAB32FFA89D9DCBE166A42340115771DF611A7160F8D7439C87ECD8 fixNSDelete
EDB4EE4C524E16BDD91D9A529332DED08DCAAA51CC6DC897ACFA1A0ED131C5B6 fix240819
8063140E9260799D6716756B891CEC3E7006C4E4F277AB84670663A88F94B9C4 fixPageCap
88693F108C3CD8A967F3F4253A32DEF5E35F9406ACD2A11B88B11D90865763A9 fix240911
36799EA497B1369B170805C078AEFE6188345F9B3E324C21E9CA3FF574E3C3D6 fixNFTokenNegOffer

View File

@@ -29,6 +29,7 @@
#include <ripple/app/misc/HashRouter.h>
#include <ripple/app/misc/LoadFeeTrack.h>
#include <ripple/app/misc/NetworkOPs.h>
#include <ripple/app/rdb/backend/LMDBDatabase.h>
#include <ripple/app/rdb/backend/PostgresDatabase.h>
#include <ripple/app/rdb/backend/SQLiteDatabase.h>
#include <ripple/basics/Log.h>
@@ -1010,14 +1011,28 @@ saveValidatedLedger(
return true;
}
auto const db = dynamic_cast<SQLiteDatabase*>(&app.getRelationalDatabase());
if (app.config().RELATIONAL_DB == 0)
{
auto const db = dynamic_cast<SQLiteDatabase*>(&app.getRelationalDatabase());
if (!db)
Throw<std::runtime_error>("Failed to get relational database");
auto const res = db->saveValidatedLedger(ledger, current);
// Clients can now trust the database for
// information about this ledger sequence.
app.pendingSaves().finishWork(seq);
return res;
}
auto const db = dynamic_cast<LMDBDatabase*>(&app.getRelationalDatabase());
if (!db)
Throw<std::runtime_error>("Failed to get relational database");
auto const res = db->saveValidatedLedger(ledger, current);
// Clients can now trust the database for
// information about this ledger sequence.
app.pendingSaves().finishWork(seq);
return res;
}

View File

@@ -41,6 +41,7 @@ inline constexpr std::uint32_t SQLITE_TUNING_CUTOFF = 10'000'000;
// Ledger database holds ledgers and ledger confirmations
inline constexpr auto LgrDBName{"ledger.db"};
inline constexpr auto LMDBLgrDBName{"ledger"};
inline constexpr std::array<char const*, 1> LgrDBPragma{
{"PRAGMA journal_size_limit=1582080;"}};
@@ -71,6 +72,7 @@ inline constexpr std::array<char const*, 5> LgrDBInit{
// Transaction database holds transactions and public keys
inline constexpr auto TxDBName{"transaction.db"};
inline constexpr auto LMDBTxDBName{"transaction"};
// In C++17 omitting the explicit template parameters caused
// a crash
@@ -215,6 +217,7 @@ inline constexpr std::array<char const*, 2> FinalShardDBPragma{
////////////////////////////////////////////////////////////////////////////////
inline constexpr auto WalletDBName{"wallet.db"};
inline constexpr auto LMDBWalletDBName{"wallet"};
inline constexpr std::array<char const*, 6> WalletDBInit{
{"BEGIN TRANSACTION;",

View File

@@ -58,12 +58,12 @@ getNodeIdentity(
return {publicKey, secretKey};
}
auto db = app.getWalletDB().checkoutDb();
auto db = app.getWalletDB().checkoutLMDB();
if (cmdline.count("newnodeid") != 0)
clearNodeIdentity(*db);
clearNodeIdentity(db.get());
return getNodeIdentity(*db);
return getNodeIdentity(db.get());
}
} // namespace ripple

View File

@@ -38,6 +38,7 @@
#include <ripple/app/misc/ValidatorKeys.h>
#include <ripple/app/misc/ValidatorList.h>
#include <ripple/app/misc/impl/AccountTxPaging.h>
#include <ripple/app/rdb/backend/LMDBDatabase.h>
#include <ripple/app/rdb/backend/PostgresDatabase.h>
#include <ripple/app/rdb/backend/SQLiteDatabase.h>
#include <ripple/app/reporting/ReportingETL.h>
@@ -3542,7 +3543,7 @@ NetworkOPsImp::unsubAccountInternal(
void
NetworkOPsImp::addAccountHistoryJob(SubAccountHistoryInfoWeak subInfo)
{
enum DatabaseType { Postgres, Sqlite, None };
enum DatabaseType { Postgres, Sqlite, LMDB, None };
static const auto databaseType = [&]() -> DatabaseType {
#ifdef RIPPLED_REPORTING
if (app_.config().reporting())

View File

@@ -22,6 +22,7 @@
#include <ripple/app/ledger/TransactionMaster.h>
#include <ripple/app/misc/NetworkOPs.h>
#include <ripple/app/rdb/State.h>
#include <ripple/app/rdb/backend/LMDBDatabase.h>
#include <ripple/app/rdb/backend/SQLiteDatabase.h>
#include <ripple/beast/core/CurrentThreadName.h>
#include <ripple/core/ConfigSections.h>
@@ -634,8 +635,53 @@ SHAMapStoreImp::clearPrior(LedgerIndex lastRotated)
if (healthWait() == stopping)
return;
SQLiteDatabase* const db =
dynamic_cast<SQLiteDatabase*>(&app_.getRelationalDatabase());
if (app_.config().RELATIONAL_DB == 0)
{
SQLiteDatabase* const db =
dynamic_cast<SQLiteDatabase*>(&app_.getRelationalDatabase());
if (!db)
Throw<std::runtime_error>("Failed to get relational database");
clearSql(
lastRotated,
"Ledgers",
[db]() -> std::optional<LedgerIndex> { return db->getMinLedgerSeq(); },
[db](LedgerIndex min) -> void { db->deleteBeforeLedgerSeq(min); });
if (healthWait() == stopping)
return;
if (!app_.config().useTxTables())
return;
clearSql(
lastRotated,
"Transactions",
[&db]() -> std::optional<LedgerIndex> {
return db->getTransactionsMinLedgerSeq();
},
[&db](LedgerIndex min) -> void {
db->deleteTransactionsBeforeLedgerSeq(min);
});
if (healthWait() == stopping)
return;
clearSql(
lastRotated,
"AccountTransactions",
[&db]() -> std::optional<LedgerIndex> {
return db->getAccountTransactionsMinLedgerSeq();
},
[&db](LedgerIndex min) -> void {
db->deleteAccountTransactionsBeforeLedgerSeq(min);
});
if (healthWait() == stopping)
return;
}
LMDBDatabase* const db =
dynamic_cast<LMDBDatabase*>(&app_.getRelationalDatabase());
if (!db)
Throw<std::runtime_error>("Failed to get relational database");

View File

@@ -111,7 +111,7 @@ public:
std::uint32_t minimumTxnInLedgerSA = 1000;
/// Number of transactions per ledger that fee escalation "works
/// towards".
std::uint32_t targetTxnInLedger = 1000;
std::uint32_t targetTxnInLedger = 256;
/** Optional maximum allowed value of transactions per ledger before
fee escalation kicks in. By default, the maximum is an emergent
property of network, validator, and consensus performance. This

View File

@@ -328,8 +328,8 @@ AmendmentTableImpl::AmendmentTableImpl(
// Find out if the FeatureVotes table exists in WalletDB
bool const featureVotesExist = [this]() {
auto db = db_.checkoutDb();
return createFeatureVotes(*db);
auto db = db_.checkoutLMDB();
return createFeatureVotes(db.get());
}();
// Parse supported amendments
@@ -404,9 +404,9 @@ AmendmentTableImpl::AmendmentTableImpl(
}
// Read amendment votes from wallet.db
auto db = db_.checkoutDb();
auto db = db_.checkoutLMDB();
readAmendments(
*db,
db.get(),
[&](boost::optional<std::string> amendment_hash,
boost::optional<std::string> amendment_name,
boost::optional<AmendmentVote> vote) {
@@ -506,8 +506,8 @@ AmendmentTableImpl::persistVote(
AmendmentVote vote) const
{
assert(vote != AmendmentVote::obsolete);
auto db = db_.checkoutDb();
voteAmendment(*db, amendment, name, vote);
auto db = db_.checkoutLMDB();
voteAmendment(db.get(), amendment, name, vote);
}
bool

View File

@@ -512,8 +512,8 @@ ManifestCache::applyManifest(Manifest m)
void
ManifestCache::load(DatabaseCon& dbCon, std::string const& dbTable)
{
auto db = dbCon.checkoutDb();
ripple::getManifests(*db, dbTable, *this, j_);
auto db = dbCon.checkoutLMDB();
ripple::getManifests(db.get(), dbTable, *this, j_);
}
bool
@@ -580,8 +580,8 @@ ManifestCache::save(
std::function<bool(PublicKey const&)> const& isTrusted)
{
std::shared_lock lock{mutex_};
auto db = dbCon.checkoutDb();
auto db = dbCon.checkoutLMDB();
saveManifests(*db, dbTable, isTrusted, map_, j_);
saveManifests(db.get(), dbTable, isTrusted, map_, j_);
}
} // namespace ripple

View File

@@ -21,6 +21,7 @@
#include <ripple/app/main/Application.h>
#include <ripple/app/misc/HashRouter.h>
#include <ripple/app/misc/Transaction.h>
#include <ripple/app/rdb/backend/LMDBDatabase.h>
#include <ripple/app/rdb/backend/PostgresDatabase.h>
#include <ripple/app/rdb/backend/SQLiteDatabase.h>
#include <ripple/app/tx/apply.h>
@@ -162,7 +163,20 @@ Transaction::load(
std::optional<ClosedInterval<uint32_t>> const& range,
error_code_i& ec)
{
auto const db = dynamic_cast<SQLiteDatabase*>(&app.getRelationalDatabase());
if (app.config().RELATIONAL_DB == 0)
{
auto const db = dynamic_cast<SQLiteDatabase*>(&app.getRelationalDatabase());
if (!db)
{
Throw<std::runtime_error>("Failed to get relational database");
}
return db->getTransaction(id, range, ec);
}
auto const db = dynamic_cast<LMDBDatabase*>(&app.getRelationalDatabase());
if (!db)
{

View File

@@ -26,12 +26,14 @@
#include <ripple/core/DatabaseCon.h>
#include <ripple/overlay/PeerReservationTable.h>
#include <ripple/peerfinder/impl/Store.h>
#include <lmdb.h>
namespace ripple {
/**
* @brief makeWalletDB Opens the wallet database and returns it.
* @param setup Path to the database and other opening parameters.
* @param j Journal.
* @return Unique pointer to the database descriptor.
*/
std::unique_ptr<DatabaseCon>
@@ -41,6 +43,7 @@ makeWalletDB(DatabaseCon::Setup const& setup);
* @brief makeTestWalletDB Opens a test wallet database with an arbitrary name.
* @param setup Path to the database and other opening parameters.
* @param dbname Name of the database.
* @param j Journal.
* @return Unique pointer to the database descriptor.
*/
std::unique_ptr<DatabaseCon>
@@ -56,6 +59,12 @@ makeTestWalletDB(DatabaseCon::Setup const& setup, std::string const& dbname);
* @param j Journal.
*/
void
getManifests(
MDB_env* env,
std::string const& dbTable,
ManifestCache& mCache,
beast::Journal j);
void
getManifests(
soci::session& session,
std::string const& dbTable,
@@ -71,6 +80,13 @@ getManifests(
* @param j Journal.
*/
void
saveManifests(
MDB_env* env,
std::string const& dbTable,
std::function<bool(PublicKey const&)> const& isTrusted,
hash_map<PublicKey, Manifest> const& map,
beast::Journal j);
void
saveManifests(
soci::session& session,
std::string const& dbTable,
@@ -85,10 +101,14 @@ saveManifests(
* @param serialized Manifest of the validator in raw format.
*/
void
addValidatorManifest(MDB_env* env, std::string const& serialized);
void
addValidatorManifest(soci::session& session, std::string const& serialized);
/** Delete any saved public/private key associated with this node. */
void
clearNodeIdentity(MDB_env* env);
void
clearNodeIdentity(soci::session& session);
/** Returns a stable public and private key for this node.
@@ -102,6 +122,8 @@ clearNodeIdentity(soci::session& session);
@return Pair of public and private secp256k1 keys.
*/
std::pair<PublicKey, SecretKey>
getNodeIdentity(MDB_env* env);
std::pair<PublicKey, SecretKey>
getNodeIdentity(soci::session& session);
/**
@@ -111,6 +133,8 @@ getNodeIdentity(soci::session& session);
* @return Peer reservation hash table.
*/
std::unordered_set<PeerReservation, beast::uhash<>, KeyEqual>
getPeerReservationTable(MDB_env* env, beast::Journal j);
std::unordered_set<PeerReservation, beast::uhash<>, KeyEqual>
getPeerReservationTable(soci::session& session, beast::Journal j);
/**
@@ -120,6 +144,11 @@ getPeerReservationTable(soci::session& session, beast::Journal j);
* @param description Description of the node.
*/
void
insertPeerReservation(
MDB_env* env,
PublicKey const& nodeId,
std::string const& description);
void
insertPeerReservation(
soci::session& session,
PublicKey const& nodeId,
@@ -132,6 +161,8 @@ insertPeerReservation(
* @param nodeId Public key of the node to remove.
*/
void
deletePeerReservation(MDB_env* env, PublicKey const& nodeId);
void
deletePeerReservation(soci::session& session, PublicKey const& nodeId);
/**
@@ -140,6 +171,8 @@ deletePeerReservation(soci::session& session, PublicKey const& nodeId);
* @return true if the table already exists
*/
bool
createFeatureVotes(MDB_env* env);
bool
createFeatureVotes(soci::session& session);
// For historical reasons the up-vote and down-vote integer representations
@@ -153,6 +186,13 @@ enum class AmendmentVote : int { obsolete = -1, up = 0, down = 1 };
* optionally a flag denoting whether the amendment should be vetoed.
*/
void
readAmendments(
MDB_env* env,
std::function<void(
boost::optional<std::string> amendment_hash,
boost::optional<std::string> amendment_name,
boost::optional<AmendmentVote> vote)> const& callback);
void
readAmendments(
soci::session& session,
std::function<void(
@@ -168,6 +208,12 @@ readAmendments(
* @param vote Whether to vote in favor of this amendment.
*/
void
voteAmendment(
MDB_env* env,
uint256 const& amendment,
std::string const& name,
AmendmentVote vote);
void
voteAmendment(
soci::session& session,
uint256 const& amendment,

View File

@@ -0,0 +1,313 @@
//------------------------------------------------------------------------------
/*
This file is part of rippled: https://github.com/ripple/rippled
Copyright (c) 2020 Ripple Labs Inc.
Permission to use, copy, modify, and/or distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#ifndef RIPPLE_APP_RDB_BACKEND_LMDBDATABASE_H_INCLUDED
#define RIPPLE_APP_RDB_BACKEND_LMDBDATABASE_H_INCLUDED
#include <ripple/app/rdb/RelationalDatabase.h>
namespace ripple {
class LMDBDatabase : public RelationalDatabase
{
public:
/**
* @brief getTransactionsMinLedgerSeq Returns the minimum ledger sequence
* stored in the Transactions table.
* @return Ledger sequence or no value if no ledgers exist.
*/
virtual std::optional<LedgerIndex>
getTransactionsMinLedgerSeq() = 0;
/**
* @brief getAccountTransactionsMinLedgerSeq Returns the minimum ledger
* sequence stored in the AccountTransactions table.
* @return Ledger sequence or no value if no ledgers exist.
*/
virtual std::optional<LedgerIndex>
getAccountTransactionsMinLedgerSeq() = 0;
/**
* @brief deleteTransactionByLedgerSeq Deletes transactions from the ledger
* with the given sequence.
* @param ledgerSeq Ledger sequence.
*/
virtual void
deleteTransactionByLedgerSeq(LedgerIndex ledgerSeq) = 0;
/**
* @brief deleteBeforeLedgerSeq Deletes all ledgers with a sequence number
* less than or equal to the given ledger sequence.
* @param ledgerSeq Ledger sequence.
*/
virtual void
deleteBeforeLedgerSeq(LedgerIndex ledgerSeq) = 0;
/**
* @brief deleteTransactionsBeforeLedgerSeq Deletes all transactions with
* a sequence number less than or equal to the given ledger
* sequence.
* @param ledgerSeq Ledger sequence.
*/
virtual void
deleteTransactionsBeforeLedgerSeq(LedgerIndex ledgerSeq) = 0;
/**
* @brief deleteAccountTransactionsBeforeLedgerSeq Deletes all account
* transactions with a sequence number less than or equal to the
* given ledger sequence.
* @param ledgerSeq Ledger sequence.
*/
virtual void
deleteAccountTransactionsBeforeLedgerSeq(LedgerIndex ledgerSeq) = 0;
/**
* @brief getTransactionCount Returns the number of transactions.
* @return Number of transactions.
*/
virtual std::size_t
getTransactionCount() = 0;
/**
* @brief getAccountTransactionCount Returns the number of account
* transactions.
* @return Number of account transactions.
*/
virtual std::size_t
getAccountTransactionCount() = 0;
/**
* @brief getLedgerCountMinMax Returns the minimum ledger sequence,
* maximum ledger sequence and total number of saved ledgers.
* @return Struct CountMinMax which contains the minimum sequence,
* maximum sequence and number of ledgers.
*/
virtual struct CountMinMax
getLedgerCountMinMax() = 0;
/**
* @brief saveValidatedLedger Saves a ledger into the database.
* @param ledger The ledger.
* @param current True if the ledger is current.
* @return True if saving was successful.
*/
virtual bool
saveValidatedLedger(
std::shared_ptr<Ledger const> const& ledger,
bool current) = 0;
/**
* @brief getLimitedOldestLedgerInfo Returns the info of the oldest ledger
* whose sequence number is greater than or equal to the given
* sequence number.
* @param ledgerFirstIndex Minimum ledger sequence.
* @return Ledger info if found, otherwise no value.
*/
virtual std::optional<LedgerInfo>
getLimitedOldestLedgerInfo(LedgerIndex ledgerFirstIndex) = 0;
/**
* @brief getLimitedNewestLedgerInfo Returns the info of the newest ledger
* whose sequence number is greater than or equal to the given
* sequence number.
* @param ledgerFirstIndex Minimum ledger sequence.
* @return Ledger info if found, otherwise no value.
*/
virtual std::optional<LedgerInfo>
getLimitedNewestLedgerInfo(LedgerIndex ledgerFirstIndex) = 0;
/**
* @brief getOldestAccountTxs Returns the oldest transactions for the
* account that matches the given criteria starting from the provided
* offset.
* @param options Struct AccountTxOptions which contains the criteria to
* match: the account, ledger search range, the offset of the first
* entry to return, the number of transactions to return, a flag if
* this number is unlimited.
* @return Vector of pairs of found transactions and their metadata
* sorted in ascending order by account sequence.
*/
virtual AccountTxs
getOldestAccountTxs(AccountTxOptions const& options) = 0;
/**
* @brief getNewestAccountTxs Returns the newest transactions for the
* account that matches the given criteria starting from the provided
* offset.
* @param options Struct AccountTxOptions which contains the criteria to
* match: the account, the ledger search range, the offset of the
* first entry to return, the number of transactions to return, a
* flag if this number unlimited.
* @return Vector of pairs of found transactions and their metadata
* sorted in descending order by account sequence.
*/
virtual AccountTxs
getNewestAccountTxs(AccountTxOptions const& options) = 0;
/**
* @brief getOldestAccountTxsB Returns the oldest transactions in binary
* form for the account that matches the given criteria starting from
* the provided offset.
* @param options Struct AccountTxOptions which contains the criteria to
* match: the account, the ledger search range, the offset of the
* first entry to return, the number of transactions to return, a
* flag if this number unlimited.
* @return Vector of tuples of found transactions, their metadata and
* account sequences sorted in ascending order by account sequence.
*/
virtual MetaTxsList
getOldestAccountTxsB(AccountTxOptions const& options) = 0;
/**
* @brief getNewestAccountTxsB Returns the newest transactions in binary
* form for the account that matches the given criteria starting from
* the provided offset.
* @param options Struct AccountTxOptions which contains the criteria to
* match: the account, the ledger search range, the offset of the
* first entry to return, the number of transactions to return, a
* flag if this number is unlimited.
* @return Vector of tuples of found transactions, their metadata and
* account sequences sorted in descending order by account
* sequence.
*/
virtual MetaTxsList
getNewestAccountTxsB(AccountTxOptions const& options) = 0;
/**
* @brief oldestAccountTxPage Returns the oldest transactions for the
* account that matches the given criteria starting from the
* provided marker.
* @param options Struct AccountTxPageOptions which contains the criteria to
* match: the account, the ledger search range, the marker of first
* returned entry, the number of transactions to return, a flag if
* this number is unlimited.
* @return Vector of pairs of found transactions and their metadata
* sorted in ascending order by account sequence and a marker
* for the next search if the search was not finished.
*/
virtual std::pair<AccountTxs, std::optional<AccountTxMarker>>
oldestAccountTxPage(AccountTxPageOptions const& options) = 0;
/**
* @brief newestAccountTxPage Returns the newest transactions for the
* account that matches the given criteria starting from the provided
* marker.
* @param options Struct AccountTxPageOptions which contains the criteria to
* match: the account, the ledger search range, the marker of the
* first returned entry, the number of transactions to return, a flag
* if this number unlimited.
* @return Vector of pairs of found transactions and their metadata
* sorted in descending order by account sequence and a marker
* for the next search if the search was not finished.
*/
virtual std::pair<AccountTxs, std::optional<AccountTxMarker>>
newestAccountTxPage(AccountTxPageOptions const& options) = 0;
/**
* @brief oldestAccountTxPageB Returns the oldest transactions in binary
* form for the account that matches the given criteria starting from
* the provided marker.
* @param options Struct AccountTxPageOptions which contains criteria to
* match: the account, the ledger search range, the marker of the
* first returned entry, the number of transactions to return, a flag
* if this number unlimited.
* @return Vector of tuples of found transactions, their metadata and
* account sequences sorted in ascending order by account
* sequence and a marker for the next search if the search was not
* finished.
*/
virtual std::pair<MetaTxsList, std::optional<AccountTxMarker>>
oldestAccountTxPageB(AccountTxPageOptions const& options) = 0;
/**
* @brief newestAccountTxPageB Returns the newest transactions in binary
* form for the account that matches the given criteria starting from
* the provided marker.
* @param options Struct AccountTxPageOptions which contains the criteria to
* match: the account, the ledger search range, the marker of the
* first returned entry, the number of transactions to return, a flag
* if this number is unlimited.
* @return Vector of tuples of found transactions, their metadata and
* account sequences sorted in descending order by account
* sequence and a marker for the next search if the search was not
* finished.
*/
virtual std::pair<MetaTxsList, std::optional<AccountTxMarker>>
newestAccountTxPageB(AccountTxPageOptions const& options) = 0;
/**
* @brief getTransaction Returns the transaction with the given hash. If a
* range is provided but the transaction is not found, then check if
* all ledgers in the range are present in the database.
* @param id Hash of the transaction.
* @param range Range of ledgers to check, if present.
* @param ec Default error code value.
* @return Transaction and its metadata if found, otherwise TxSearched::all
* if a range is provided and all ledgers from the range are present
* in the database, TxSearched::some if a range is provided and not
* all ledgers are present, TxSearched::unknown if the range is not
* provided or a deserializing error occurred. In the last case the
* error code is returned via the ec parameter, in other cases the
* default error code is not changed.
*/
virtual std::variant<AccountTx, TxSearched>
getTransaction(
uint256 const& id,
std::optional<ClosedInterval<uint32_t>> const& range,
error_code_i& ec) = 0;
/**
* @brief getKBUsedAll Returns the amount of space used by all databases.
* @return Space in kilobytes.
*/
virtual uint32_t
getKBUsedAll() = 0;
/**
* @brief getKBUsedLedger Returns the amount of space space used by the
* ledger database.
* @return Space in kilobytes.
*/
virtual uint32_t
getKBUsedLedger() = 0;
/**
* @brief getKBUsedTransaction Returns the amount of space used by the
* transaction database.
* @return Space in kilobytes.
*/
virtual uint32_t
getKBUsedTransaction() = 0;
/**
* @brief Closes the ledger database
*/
virtual void
closeLedgerDB() = 0;
/**
* @brief Closes the transaction database
*/
virtual void
closeTransactionDB() = 0;
};
} // namespace ripple
#endif

View File

@@ -27,6 +27,7 @@
#include <ripple/overlay/PeerReservationTable.h>
#include <ripple/peerfinder/impl/Store.h>
#include <boost/filesystem.hpp>
#include <lmdb.h>
namespace ripple {
namespace detail {
@@ -63,6 +64,8 @@ makeLedgerDBs(
* @return Ledger sequence or none if no ledgers exist.
*/
std::optional<LedgerIndex>
getMinLedgerSeq(MDB_env* env, TableType type);
std::optional<LedgerIndex>
getMinLedgerSeq(soci::session& session, TableType type);
/**
@@ -72,6 +75,8 @@ getMinLedgerSeq(soci::session& session, TableType type);
* @return Ledger sequence or none if no ledgers exist.
*/
std::optional<LedgerIndex>
getMaxLedgerSeq(MDB_env* env, TableType type);
std::optional<LedgerIndex>
getMaxLedgerSeq(soci::session& session, TableType type);
/**
@@ -82,6 +87,8 @@ getMaxLedgerSeq(soci::session& session, TableType type);
* @param ledgerSeq Ledger sequence.
*/
void
deleteByLedgerSeq(MDB_env* env, TableType type, LedgerIndex ledgerSeq);
void
deleteByLedgerSeq(
soci::session& session,
TableType type,
@@ -95,6 +102,8 @@ deleteByLedgerSeq(
* @param ledgerSeq Ledger sequence.
*/
void
deleteBeforeLedgerSeq(MDB_env* env, TableType type, LedgerIndex ledgerSeq);
void
deleteBeforeLedgerSeq(
soci::session& session,
TableType type,
@@ -107,6 +116,8 @@ deleteBeforeLedgerSeq(
* @return Number of rows.
*/
std::size_t
getRows(MDB_env* env, TableType type);
std::size_t
getRows(soci::session& session, TableType type);
/**
@@ -118,6 +129,8 @@ getRows(soci::session& session, TableType type);
* maximum sequence and number of rows.
*/
RelationalDatabase::CountMinMax
getRowsMinMax(MDB_env* env, TableType type);
RelationalDatabase::CountMinMax
getRowsMinMax(soci::session& session, TableType type);
/**
@@ -145,6 +158,8 @@ saveValidatedLedger(
* @return Ledger or none if ledger not found.
*/
std::optional<LedgerInfo>
getLedgerInfoByIndex(MDB_env* env, LedgerIndex ledgerSeq, beast::Journal j);
std::optional<LedgerInfo>
getLedgerInfoByIndex(
soci::session& session,
LedgerIndex ledgerSeq,
@@ -157,6 +172,8 @@ getLedgerInfoByIndex(
* @return Ledger info or none if ledger not found.
*/
std::optional<LedgerInfo>
getNewestLedgerInfo(MDB_env* env, beast::Journal j);
std::optional<LedgerInfo>
getNewestLedgerInfo(soci::session& session, beast::Journal j);
/**
@@ -168,6 +185,11 @@ getNewestLedgerInfo(soci::session& session, beast::Journal j);
* @return Ledger info or none if ledger not found.
*/
std::optional<LedgerInfo>
getLimitedOldestLedgerInfo(
MDB_env* env,
LedgerIndex ledgerFirstIndex,
beast::Journal j);
std::optional<LedgerInfo>
getLimitedOldestLedgerInfo(
soci::session& session,
LedgerIndex ledgerFirstIndex,
@@ -182,6 +204,11 @@ getLimitedOldestLedgerInfo(
* @return Ledger info or none if ledger not found.
*/
std::optional<LedgerInfo>
getLimitedNewestLedgerInfo(
MDB_env* env,
LedgerIndex ledgerFirstIndex,
beast::Journal j);
std::optional<LedgerInfo>
getLimitedNewestLedgerInfo(
soci::session& session,
LedgerIndex ledgerFirstIndex,
@@ -195,6 +222,8 @@ getLimitedNewestLedgerInfo(
* @return Ledger or none if ledger not found.
*/
std::optional<LedgerInfo>
getLedgerInfoByHash(MDB_env* env, uint256 const& ledgerHash, beast::Journal j);
std::optional<LedgerInfo>
getLedgerInfoByHash(
soci::session& session,
uint256 const& ledgerHash,
@@ -207,6 +236,8 @@ getLedgerInfoByHash(
* @return Hash of the ledger.
*/
uint256
getHashByIndex(MDB_env* env, LedgerIndex ledgerIndex);
uint256
getHashByIndex(soci::session& session, LedgerIndex ledgerIndex);
/**
@@ -219,6 +250,8 @@ getHashByIndex(soci::session& session, LedgerIndex ledgerIndex);
* its parent ledger.
*/
std::optional<LedgerHashPair>
getHashesByIndex(MDB_env* env, LedgerIndex ledgerIndex, beast::Journal j);
std::optional<LedgerHashPair>
getHashesByIndex(
soci::session& session,
LedgerIndex ledgerIndex,
@@ -236,6 +269,12 @@ getHashesByIndex(
* LedgerHashPair which contains ledger hash and its parent hash.
*/
std::map<LedgerIndex, LedgerHashPair>
getHashesByIndex(
MDB_env* env,
LedgerIndex minSeq,
LedgerIndex maxSeq,
beast::Journal j);
std::map<LedgerIndex, LedgerHashPair>
getHashesByIndex(
soci::session& session,
LedgerIndex minSeq,
@@ -463,6 +502,13 @@ newestAccountTxPage(
* parameter, in other cases default error code remained.
*/
std::variant<RelationalDatabase::AccountTx, TxSearched>
getTransaction(
MDB_env* env,
Application& app,
uint256 const& id,
std::optional<ClosedInterval<uint32_t>> const& range,
error_code_i& ec);
std::variant<RelationalDatabase::AccountTx, TxSearched>
getTransaction(
soci::session& session,
Application& app,
@@ -478,6 +524,8 @@ getTransaction(
* @return True if space is available.
*/
bool
dbHasSpace(MDB_env* env, Config const& config, beast::Journal j);
bool
dbHasSpace(soci::session& session, Config const& config, beast::Journal j);
} // namespace detail

View File

@@ -64,7 +64,7 @@ to_string(TableType type)
}
DatabasePairValid
makeLedgerDBs(
makeSQLLedgerDBs(
Config const& config,
DatabaseCon::Setup const& setup,
DatabaseCon::CheckpointerSetup const& checkpointerSetup)
@@ -119,6 +119,145 @@ makeLedgerDBs(
return {std::move(lgr), {}, true};
}
DatabasePairValid
makeLMDBLedgerDBs(
Config const& config,
DatabaseCon::Setup const& setup,
DatabaseCon::CheckpointerSetup const& checkpointerSetup)
{
// ledger database
auto lgr{
std::make_unique<DatabaseCon>(setup, LMDBLgrDBName, checkpointerSetup)};
lgr->getLMDB();
if (config.useTxTables())
{
// transaction database
auto tx{std::make_unique<DatabaseCon>(
setup, LMDBTxDBName, checkpointerSetup)};
tx->getLMDB();
return {std::move(lgr), std::move(tx), true};
}
else
return {std::move(lgr), {}, true};
}
DatabasePairValid
makeLedgerDBs(
Config const& config,
DatabaseCon::Setup const& setup,
DatabaseCon::CheckpointerSetup const& checkpointerSetup)
{
if (setup.sqlite3)
{
return makeSQLLedgerDBs(config, setup, checkpointerSetup);
}
return makeLMDBLedgerDBs(config, setup, checkpointerSetup);
}
std::optional<LedgerIndex>
getMinLedgerSeq(MDB_env* env, TableType type)
{
MDB_txn* txn;
MDB_dbi dbi;
MDB_val key, data;
std::optional<LedgerIndex> minLedgerSeq;
mdb_txn_begin(env, nullptr, MDB_RDONLY, &txn);
mdb_dbi_open(txn, to_string(type).c_str(), 0, &dbi);
MDB_cursor* cursor;
mdb_cursor_open(txn, dbi, &cursor);
if (mdb_cursor_get(cursor, &key, &data, MDB_FIRST) == 0)
{
minLedgerSeq = *static_cast<LedgerIndex*>(data.mv_data);
}
mdb_cursor_close(cursor);
mdb_txn_commit(txn);
mdb_dbi_close(env, dbi);
return minLedgerSeq;
}
std::optional<LedgerIndex>
getMaxLedgerSeq(MDB_env* env, TableType type)
{
MDB_txn* txn;
MDB_dbi dbi;
MDB_val key, data;
std::optional<LedgerIndex> maxLedgerSeq;
mdb_txn_begin(env, nullptr, MDB_RDONLY, &txn);
mdb_dbi_open(txn, to_string(type).c_str(), 0, &dbi);
MDB_cursor* cursor;
mdb_cursor_open(txn, dbi, &cursor);
if (mdb_cursor_get(cursor, &key, &data, MDB_LAST) == 0)
{
maxLedgerSeq = *static_cast<LedgerIndex*>(data.mv_data);
}
mdb_cursor_close(cursor);
mdb_txn_commit(txn);
mdb_dbi_close(env, dbi);
return maxLedgerSeq;
}
void
deleteByLedgerSeq(MDB_env* env, TableType type, LedgerIndex ledgerSeq)
{
MDB_txn* txn;
MDB_dbi dbi;
MDB_val key, data;
mdb_txn_begin(env, nullptr, 0, &txn);
mdb_dbi_open(txn, to_string(type).c_str(), 0, &dbi);
key.mv_size = sizeof(LedgerIndex);
key.mv_data = &ledgerSeq;
mdb_del(txn, dbi, &key, nullptr);
mdb_txn_commit(txn);
mdb_dbi_close(env, dbi);
}
void
deleteBeforeLedgerSeq(MDB_env* env, TableType type, LedgerIndex ledgerSeq)
{
MDB_txn* txn;
MDB_dbi dbi;
MDB_val key, data;
mdb_txn_begin(env, nullptr, 0, &txn);
mdb_dbi_open(txn, to_string(type).c_str(), 0, &dbi);
MDB_cursor* cursor;
mdb_cursor_open(txn, dbi, &cursor);
while (mdb_cursor_get(cursor, &key, &data, MDB_NEXT) == 0)
{
if (*static_cast<LedgerIndex*>(key.mv_data) < ledgerSeq)
{
mdb_cursor_del(cursor, 0);
}
else
{
break;
}
}
mdb_cursor_close(cursor);
mdb_txn_commit(txn);
mdb_dbi_close(env, dbi);
}
std::optional<LedgerIndex>
getMinLedgerSeq(soci::session& session, TableType type)
{
@@ -156,6 +295,73 @@ deleteBeforeLedgerSeq(
<< ledgerSeq << ";";
}
std::size_t
getRows(MDB_env* env, TableType type)
{
MDB_txn* txn;
MDB_dbi dbi;
MDB_val key;
std::size_t rows = 0;
if (mdb_txn_begin(env, nullptr, MDB_RDONLY, &txn) != 0)
throw std::runtime_error("Failed to begin transaction");
if (mdb_dbi_open(txn, to_string(type).c_str(), 0, &dbi) != 0)
throw std::runtime_error("Failed to open database");
MDB_cursor* cursor;
if (mdb_cursor_open(txn, dbi, &cursor) != 0)
throw std::runtime_error("Failed to open cursor");
while (mdb_cursor_get(cursor, &key, nullptr, MDB_NEXT) == 0)
{
++rows;
}
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
return rows;
}
RelationalDatabase::CountMinMax
getRowsMinMax(MDB_env* env, TableType type)
{
MDB_txn* txn;
MDB_dbi dbi;
MDB_val key, data;
RelationalDatabase::CountMinMax res = {
0,
std::numeric_limits<ripple::LedgerIndex>::max(),
std::numeric_limits<ripple::LedgerIndex>::min()};
if (mdb_txn_begin(env, nullptr, MDB_RDONLY, &txn) != 0)
throw std::runtime_error("Failed to begin transaction");
if (mdb_dbi_open(txn, to_string(type).c_str(), 0, &dbi) != 0)
throw std::runtime_error("Failed to open database");
MDB_cursor* cursor;
if (mdb_cursor_open(txn, dbi, &cursor) != 0)
throw std::runtime_error("Failed to open cursor");
while (mdb_cursor_get(cursor, &key, &data, MDB_NEXT) == 0)
{
++res.numberOfRows;
ripple::LedgerIndex ledgerSeq =
*static_cast<ripple::LedgerIndex*>(data.mv_data);
if (ledgerSeq < res.minLedgerSequence)
res.minLedgerSequence = ledgerSeq;
if (ledgerSeq > res.maxLedgerSequence)
res.maxLedgerSequence = ledgerSeq;
}
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
return res;
}
std::size_t
getRows(soci::session& session, TableType type)
{
@@ -184,7 +390,233 @@ getRowsMinMax(soci::session& session, TableType type)
}
bool
saveValidatedLedger(
saveLMDBValidatedLedger(
DatabaseCon& ldgDB,
DatabaseCon& txnDB,
Application& app,
std::shared_ptr<Ledger const> const& ledger,
bool current)
{
auto j = app.journal("Ledger");
auto seq = ledger->info().seq;
auto hash = ledger->info().hash;
JLOG(j.warn()) << "saveLMDBValidatedLedger "
<< (current ? "" : "fromAcquire ") << seq;
if (!ledger->info().accountHash.isNonZero())
{
JLOG(j.fatal()) << "AH is zero: " << getJson({*ledger, {}});
assert(false);
}
if (ledger->info().accountHash != ledger->stateMap().getHash().as_uint256())
{
JLOG(j.fatal()) << "sAL: " << ledger->info().accountHash
<< " != " << ledger->stateMap().getHash();
JLOG(j.fatal()) << "saveAcceptedLedger: seq=" << seq
<< ", current=" << current;
assert(false);
}
assert(ledger->info().txHash == ledger->txMap().getHash().as_uint256());
// Save the ledger header in the hashed object store
{
Serializer s(128);
s.add32(HashPrefix::ledgerMaster);
addRaw(ledger->info(), s);
app.getNodeStore().store(
hotLEDGER, std::move(s.modData()), ledger->info().hash, seq);
}
std::shared_ptr<AcceptedLedger> aLedger;
try
{
aLedger = app.getAcceptedLedgerCache().fetch(ledger->info().hash);
if (!aLedger)
{
aLedger = std::make_shared<AcceptedLedger>(ledger, app);
app.getAcceptedLedgerCache().canonicalize_replace_client(
ledger->info().hash, aLedger);
}
}
catch (std::exception const&)
{
JLOG(j.warn()) << "An accepted ledger was missing nodes";
app.getLedgerMaster().failedSave(seq, ledger->info().hash);
app.pendingSaves().finishWork(seq);
return false;
}
// Delete ledger and transactions
{
MDB_val key1, key2;
key1.mv_size = sizeof(seq);
key1.mv_data = &seq;
key2.mv_size = sizeof(hash);
key2.mv_data = &hash;
{
MDB_dbi dbi1, dbi2;
MDB_txn* txn;
auto db = ldgDB.checkoutLMDB();
MDB_env* env = db.get();
mdb_txn_begin(env, nullptr, 0, &txn);
try
{
mdb_dbi_open(txn, "LedgerHashBySeq", MDB_CREATE, &dbi1);
mdb_dbi_open(txn, "Ledgers", MDB_CREATE, &dbi2);
mdb_del(txn, dbi1, &key1, nullptr);
mdb_del(txn, dbi2, &key2, nullptr);
mdb_dbi_close(env, dbi1);
mdb_dbi_close(env, dbi2);
if (app.config().useTxTables())
{
MDB_dbi dbi1, dbi2;
mdb_dbi_open(txn, "Transactions", MDB_CREATE, &dbi1);
mdb_dbi_open(txn, "AccountTransactions", MDB_CREATE, &dbi2);
mdb_del(txn, dbi1, &key1, nullptr);
mdb_del(txn, dbi2, &key1, nullptr);
for (auto const& acceptedLedgerTx : *aLedger)
{
uint256 transactionID =
acceptedLedgerTx->getTransactionID();
MDB_val txKey = {
transactionID.size(), (void*)transactionID.data()};
mdb_del(txn, dbi2, &txKey, nullptr);
auto const& accts = acceptedLedgerTx->getAffected();
if (!accts.empty())
{
for (auto const& account : accts)
{
std::string acctKey =
toBase58(account) + std::to_string(seq);
MDB_val acctTxKey = {
acctKey.size(), (void*)acctKey.data()};
mdb_put(txn, dbi2, &acctTxKey, &txKey, 0);
}
}
else if (auto const& sleTxn =
acceptedLedgerTx->getTxn();
!isPseudoTx(*sleTxn))
{
JLOG(j.warn()) << "Transaction in ledger " << seq
<< " affects no accounts";
JLOG(j.warn())
<< sleTxn->getJson(JsonOptions::none);
}
std::string metaKey =
to_string(transactionID) + std::to_string(seq);
MDB_val metaTxKey = {
metaKey.size(), (void*)metaKey.data()};
MDB_val metaData = {
acceptedLedgerTx->getEscMeta().size(),
(void*)acceptedLedgerTx->getEscMeta().data()};
mdb_put(txn, dbi1, &metaTxKey, &metaData, 0);
app.getMasterTransaction().inLedger(
transactionID,
seq,
acceptedLedgerTx->getTxnSeq(),
app.config().NETWORK_ID);
}
mdb_dbi_close(env, dbi1);
mdb_dbi_close(env, dbi2);
}
mdb_txn_commit(txn);
}
catch (std::exception const&)
{
mdb_txn_abort(txn);
mdb_dbi_close(env, dbi1);
mdb_dbi_close(env, dbi2);
JLOG(j.error()) << "Failed to delete ledger and transactions";
throw;
}
}
}
// Insert ledger
{
MDB_dbi dbi1;
MDB_dbi dbi2;
MDB_txn* txn;
auto db = ldgDB.checkoutLMDB();
MDB_env* env = db.get();
if (mdb_txn_begin(env, nullptr, 0, &txn) != 0)
{
std::cout << "Failed to begin transaction" << std::endl;
JLOG(j.error()) << "Failed to begin transaction";
}
try
{
if (mdb_dbi_open(txn, "Ledgers", MDB_CREATE, &dbi1) != 0)
{
// std::cout << "Failed to open database" << std::endl;
JLOG(j.error()) << "Failed to open database";
throw std::runtime_error("Failed to open database");
}
if (mdb_dbi_open(txn, "LedgerHashBySeq", MDB_CREATE, &dbi2) != 0)
{
// std::cout << "Failed to open database" << std::endl;
JLOG(j.error()) << "Failed to open database";
throw std::runtime_error("Failed to open database");
}
MDB_val key1, key2, data1, data2;
std::string ledgerHash = to_string(ledger->info().hash);
key1.mv_size = ledgerHash.size();
key1.mv_data = ledgerHash.data();
std::string ledgerSeq = std::to_string(ledger->info().seq);
key2.mv_size = ledgerSeq.size();
key2.mv_data = ledgerSeq.data();
Serializer s;
addRaw(ledger->info(), s, true);
data1.mv_size = s.getDataLength();
data1.mv_data = s.getDataPtr();
data2.mv_size = ledgerHash.size();
data2.mv_data = ledgerHash.data();
// Slice key2Slice(key2.mv_data, key2.mv_size);
// Slice data2Slice(data2.mv_data, data2.mv_size);
// std::cout << "Inserting DB2 Key: " << key2Slice << std::endl;
// std::cout << "Inserting DB2 Data: " << data2Slice << std::endl;
mdb_put(txn, dbi1, &key1, &data1, 0);
mdb_put(txn, dbi2, &key2, &data2, 0);
mdb_txn_commit(txn);
mdb_dbi_close(env, dbi1);
mdb_dbi_close(env, dbi2);
}
catch (std::exception const&)
{
std::cout << "Failed to insert ledger" << std::endl;
JLOG(j.error()) << "Failed to insert ledger";
mdb_txn_abort(txn);
mdb_dbi_close(env, dbi1);
mdb_dbi_close(env, dbi2);
}
}
return true;
}
bool
saveSQLValidatedLedger(
DatabaseCon& ldgDB,
DatabaseCon& txnDB,
Application& app,
@@ -195,8 +627,8 @@ saveValidatedLedger(
auto seq = ledger->info().seq;
// TODO(tom): Fix this hard-coded SQL!
JLOG(j.trace()) << "saveValidatedLedger " << (current ? "" : "fromAcquire ")
<< seq;
JLOG(j.info()) << "saveSQLValidatedLedger "
<< (current ? "" : "fromAcquire ") << seq;
if (!ledger->info().accountHash.isNonZero())
{
@@ -383,6 +815,118 @@ saveValidatedLedger(
return true;
}
bool
saveValidatedLedger(
DatabaseCon& ldgDB,
DatabaseCon& txnDB,
Application& app,
std::shared_ptr<Ledger const> const& ledger,
bool current)
{
if (app.config().RELATIONAL_DB == 0)
{
return saveSQLValidatedLedger(ldgDB, txnDB, app, ledger, current);
}
return saveLMDBValidatedLedger(ldgDB, txnDB, app, ledger, current);
}
static std::optional<LedgerInfo>
getLedgerInfo(
MDB_env* env,
std::string const& db,
std::string const& key,
beast::Journal j)
{
MDB_txn* txn;
MDB_dbi dbi;
MDB_val mdb_key, mdb_data;
// Start a read-only transaction
if (mdb_txn_begin(env, nullptr, MDB_RDONLY, &txn) != 0)
{
return {};
}
// Open the database
if (mdb_dbi_open(txn, db.c_str(), 0, &dbi) != 0)
{
mdb_txn_abort(txn);
return {};
}
// Set the key to search for
mdb_key.mv_size = key.size();
mdb_key.mv_data = const_cast<char*>(key.data());
// Get the value associated with the key
if (mdb_get(txn, dbi, &mdb_key, &mdb_data) != 0)
{
mdb_txn_abort(txn);
return {};
}
Slice dataSlice(mdb_data.mv_data, mdb_data.mv_size);
LedgerInfo info = deserializeHeader(dataSlice, true);
// Commit the transaction
mdb_txn_commit(txn);
return info;
}
std::optional<LedgerInfo>
getLedgerInfoByIndex(MDB_env* env, LedgerIndex ledgerSeq, beast::Journal j)
{
auto const ledgerHash = getHashByIndex(env, ledgerSeq);
std::ostringstream s;
s << ledgerHash;
return getLedgerInfo(env, "Ledgers", s.str(), j);
}
// Shard (Depricated)
std::optional<LedgerInfo>
getNewestLedgerInfo(MDB_env* env, beast::Journal j)
{
// LMDB does not support SQL-like queries directly
// You need to implement a way to find the newest ledger info
// This might involve iterating over keys or maintaining a separate index
// For simplicity, this example assumes a key "NewestLedger" exists
return getLedgerInfo(env, "Ledgers", "NewestLedger", j);
}
// Shard (Depricated)
std::optional<LedgerInfo>
getLimitedOldestLedgerInfo(
MDB_env* env,
LedgerIndex ledgerFirstIndex,
beast::Journal j)
{
// Similar to getNewestLedgerInfo, you need to implement a way to find the
// oldest ledger info For simplicity, this example assumes a key
// "OldestLedger" exists
return getLedgerInfo(env, "Ledgers", "OldestLedger", j);
}
// Shard (Depricated)
std::optional<LedgerInfo>
getLimitedNewestLedgerInfo(
MDB_env* env,
LedgerIndex ledgerFirstIndex,
beast::Journal j)
{
// Similar to getNewestLedgerInfo, you need to implement a way to find the
// limited newest ledger info For simplicity, this example assumes a key
// "LimitedNewestLedger" exists
return getLedgerInfo(env, "Ledgers", "LimitedNewestLedger", j);
}
std::optional<LedgerInfo>
getLedgerInfoByHash(MDB_env* env, uint256 const& ledgerHash, beast::Journal j)
{
std::ostringstream s;
s << ledgerHash;
return getLedgerInfo(env, "Ledgers", s.str(), j);
}
/**
* @brief getLedgerInfo Returns the info of the ledger retrieved from the
* database by using the provided SQL query suffix.
@@ -515,6 +1059,50 @@ getLedgerInfoByHash(
return getLedgerInfo(session, s.str(), j);
}
uint256
getHashByIndex(MDB_env* env, LedgerIndex ledgerIndex)
{
uint256 ret;
MDB_txn* txn;
MDB_dbi dbi;
MDB_val key, data;
// Convert ledgerIndex to string
std::string keyStr = beast::lexicalCastThrow<std::string>(ledgerIndex);
// Begin a new transaction
if (mdb_txn_begin(env, nullptr, MDB_RDONLY, &txn) != 0)
return ret;
// Open the database
if (mdb_dbi_open(txn, "LedgerHashBySeq", 0, &dbi) != 0)
{
mdb_txn_abort(txn);
return ret;
}
// Set the key
key.mv_size = keyStr.size();
key.mv_data = const_cast<char*>(keyStr.data());
// Get the value
if (mdb_get(txn, dbi, &key, &data) == 0)
{
std::string hash(static_cast<char*>(data.mv_data), data.mv_size);
if (!hash.empty() && ret.parseHex(hash))
{
mdb_txn_commit(txn);
mdb_dbi_close(env, dbi);
return ret;
}
}
// Cleanup
mdb_txn_abort(txn);
mdb_dbi_close(env, dbi);
return ret;
}
uint256
getHashByIndex(soci::session& session, LedgerIndex ledgerIndex)
{
@@ -545,6 +1133,144 @@ getHashByIndex(soci::session& session, LedgerIndex ledgerIndex)
return ret;
}
std::optional<LedgerHashPair>
getHashesByIndex(MDB_env* env, LedgerIndex ledgerIndex, beast::Journal j)
{
MDB_txn* txn;
MDB_dbi dbi;
MDB_val key, data;
int rc;
rc = mdb_txn_begin(env, nullptr, MDB_RDONLY, &txn);
if (rc != 0)
{
JLOG(j.error()) << "Failed to begin transaction: " << mdb_strerror(rc);
return {};
}
rc = mdb_dbi_open(txn, "Ledgers", 0, &dbi);
if (rc != 0)
{
mdb_txn_abort(txn);
JLOG(j.error()) << "Failed to open database: " << mdb_strerror(rc);
return {};
}
key.mv_size = sizeof(ledgerIndex);
key.mv_data = &ledgerIndex;
rc = mdb_get(txn, dbi, &key, &data);
if (rc == MDB_NOTFOUND)
{
mdb_txn_abort(txn);
JLOG(j.trace()) << "Don't have ledger " << ledgerIndex;
return {};
}
else if (rc != 0)
{
mdb_txn_abort(txn);
JLOG(j.error()) << "Failed to get data: " << mdb_strerror(rc);
return {};
}
// Assuming data.mv_data points to a structure containing LedgerHash and
// PrevHash
LedgerHashPair hashes;
std::string lhO(
static_cast<char*>(data.mv_data), 32); // Assuming 32-byte hash
std::string phO(
static_cast<char*>(data.mv_data) + 32, 32); // Assuming 32-byte hash
if (!hashes.ledgerHash.parseHex(lhO) || !hashes.parentHash.parseHex(phO))
{
mdb_txn_abort(txn);
JLOG(j.trace()) << "Error parse hashes for ledger " << ledgerIndex;
return {};
}
mdb_txn_commit(txn);
return hashes;
}
std::map<LedgerIndex, LedgerHashPair>
getHashesByIndex(
MDB_env* env,
LedgerIndex minSeq,
LedgerIndex maxSeq,
beast::Journal j)
{
MDB_txn* txn;
MDB_dbi dbi;
MDB_cursor* cursor;
MDB_val key, data;
int rc;
rc = mdb_txn_begin(env, nullptr, MDB_RDONLY, &txn);
if (rc != 0)
{
JLOG(j.error()) << "Failed to begin transaction: " << mdb_strerror(rc);
return {};
}
rc = mdb_dbi_open(txn, "Ledgers", 0, &dbi);
if (rc != 0)
{
mdb_txn_abort(txn);
JLOG(j.error()) << "Failed to open database: " << mdb_strerror(rc);
return {};
}
rc = mdb_cursor_open(txn, dbi, &cursor);
if (rc != 0)
{
mdb_txn_abort(txn);
JLOG(j.error()) << "Failed to open cursor: " << mdb_strerror(rc);
return {};
}
std::map<LedgerIndex, LedgerHashPair> res;
for (LedgerIndex seq = minSeq; seq <= maxSeq; ++seq)
{
key.mv_size = sizeof(seq);
key.mv_data = &seq;
rc = mdb_cursor_get(cursor, &key, &data, MDB_SET);
if (rc == MDB_NOTFOUND)
{
continue;
}
else if (rc != 0)
{
JLOG(j.error()) << "Failed to get data: " << mdb_strerror(rc);
continue;
}
// Assuming data.mv_data points to a structure containing LedgerHash and
// PrevHash
std::uint64_t ls = *static_cast<std::uint64_t*>(key.mv_data);
std::string lh(
static_cast<char*>(data.mv_data), 32); // Assuming 32-byte hash
std::string ph(
static_cast<char*>(data.mv_data) + 32,
32); // Assuming 32-byte hash
LedgerHashPair& hashes = res[rangeCheckedCast<LedgerIndex>(ls)];
if (!hashes.ledgerHash.parseHex(lh))
{
JLOG(j.warn()) << "Error parsed hash for ledger seq: " << ls;
}
if (!hashes.parentHash.parseHex(ph))
{
JLOG(j.warn()) << "Error parsed prev hash for ledger seq: " << ls;
}
}
mdb_cursor_close(cursor);
mdb_txn_commit(txn);
return res;
}
std::optional<LedgerHashPair>
getHashesByIndex(
soci::session& session,
@@ -1290,6 +2016,129 @@ newestAccountTxPage(
false);
}
std::variant<RelationalDatabase::AccountTx, TxSearched>
getTransaction(
MDB_env* env,
Application& app,
uint256 const& id,
std::optional<ClosedInterval<uint32_t>> const& range,
error_code_i& ec)
{
MDB_txn* txn;
MDB_dbi dbi;
MDB_val key, data;
int rc;
rc = mdb_txn_begin(env, nullptr, MDB_RDONLY, &txn);
if (rc != MDB_SUCCESS)
{
ec = rpcDB_DESERIALIZATION; // Assuming this is the appropriate error
// code
return TxSearched::unknown;
}
rc = mdb_dbi_open(txn, "Transactions", 0, &dbi);
if (rc != MDB_SUCCESS)
{
mdb_txn_abort(txn);
ec = rpcDB_DESERIALIZATION;
return TxSearched::unknown;
}
std::string key_str = to_string(id);
key.mv_size = key_str.size();
key.mv_data = key_str.data();
rc = mdb_get(txn, dbi, &key, &data);
if (rc == MDB_NOTFOUND)
{
if (!range)
{
mdb_txn_abort(txn);
return TxSearched::unknown;
}
// Check the range
MDB_cursor* cursor;
rc = mdb_cursor_open(txn, dbi, &cursor);
if (rc != MDB_SUCCESS)
{
mdb_txn_abort(txn);
ec = rpcDB_DESERIALIZATION;
return TxSearched::unknown;
}
uint64_t count = 0;
MDB_val key_range, data_range;
for (uint32_t i = range->first(); i <= range->last(); ++i)
{
key_range.mv_size = sizeof(i);
key_range.mv_data = &i;
rc = mdb_cursor_get(cursor, &key_range, &data_range, MDB_SET_KEY);
if (rc == MDB_SUCCESS)
{
++count;
}
}
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
return count == (range->last() - range->first() + 1) ? TxSearched::all
: TxSearched::some;
}
else if (rc != MDB_SUCCESS)
{
mdb_txn_abort(txn);
ec = rpcDB_DESERIALIZATION;
return TxSearched::unknown;
}
// Assuming the data is structured as follows: LedgerSeq, Status, RawTxn,
// TxnMeta You will need to parse the data appropriately For simplicity,
// let's assume data.mv_data points to a struct with these fields
struct TransactionData
{
uint64_t ledgerSeq;
std::string status;
std::vector<uint8_t> rawTxn;
std::vector<uint8_t> rawMeta;
};
TransactionData* txnData = static_cast<TransactionData*>(data.mv_data);
try
{
auto _txn = Transaction::transactionFromSQL(
txnData->ledgerSeq, txnData->status, txnData->rawTxn, app);
if (!txnData->ledgerSeq)
{
mdb_txn_commit(txn);
return std::pair{std::move(_txn), nullptr};
}
std::uint32_t inLedger =
rangeCheckedCast<std::uint32_t>(txnData->ledgerSeq);
auto txMeta = std::make_shared<TxMeta>(id, inLedger, txnData->rawMeta);
mdb_txn_commit(txn);
return std::pair{std::move(_txn), std::move(txMeta)};
}
catch (std::exception& e)
{
JLOG(app.journal("Ledger").warn())
<< "Unable to deserialize transaction from raw SQL value. Error: "
<< e.what();
ec = rpcDB_DESERIALIZATION;
}
mdb_txn_abort(txn);
return TxSearched::unknown;
}
std::variant<RelationalDatabase::AccountTx, TxSearched>
getTransaction(
soci::session& session,
@@ -1371,6 +2220,73 @@ getTransaction(
return TxSearched::unknown;
}
bool
dbHasSpace(MDB_env* env, Config const& config, beast::Journal j)
{
boost::filesystem::space_info space =
boost::filesystem::space(config.legacy("database_path"));
if (space.available < megabytes(512))
{
JLOG(j.fatal()) << "Remaining free disk space is less than 512MB";
return false;
}
if (config.useTxTables())
{
DatabaseCon::Setup dbSetup = setup_DatabaseCon(config);
boost::filesystem::path dbPath = dbSetup.dataDir / LMDBTxDBName;
boost::system::error_code ec;
std::optional<std::uint64_t> dbSize =
boost::filesystem::file_size(dbPath, ec);
if (ec)
{
JLOG(j.error())
<< "Error checking transaction db file size: " << ec.message();
dbSize.reset();
}
MDB_stat stat;
if (mdb_env_stat(env, &stat) != MDB_SUCCESS)
{
JLOG(j.error()) << "Error getting LMDB statistics";
return false;
}
std::uint32_t pageSize = stat.ms_psize;
std::uint32_t maxPages =
stat.ms_entries; // This is not exactly the same as max_page_count
// in SQLite
std::uint32_t pageCount =
stat.ms_branch_pages + stat.ms_leaf_pages + stat.ms_overflow_pages;
std::uint32_t freePages = maxPages - pageCount;
std::uint64_t freeSpace =
safe_cast<std::uint64_t>(freePages) * pageSize;
JLOG(j.info())
<< "Transaction DB pathname: " << dbPath.string()
<< "; file size: " << dbSize.value_or(-1) << " bytes"
<< "; LMDB page size: " << pageSize << " bytes"
<< "; Free pages: " << freePages << "; Free space: " << freeSpace
<< " bytes; "
<< "Note that this does not take into account available disk "
"space.";
if (freeSpace < megabytes(512))
{
JLOG(j.fatal())
<< "Free LMDB space for transaction db is less than "
"512MB. To fix this, rippled must be executed with the "
"vacuum parameter before restarting. "
"Note that this activity can take multiple days, "
"depending on database size.";
// return false;
return true;
}
}
return true;
}
bool
dbHasSpace(soci::session& session, Config const& config, beast::Journal j)
{

View File

@@ -0,0 +1,779 @@
//------------------------------------------------------------------------------
/*
This file is part of rippled: https://github.com/ripple/rippled
Copyright (c) 2020 Ripple Labs Inc.
Permission to use, copy, modify, and/or distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#include <ripple/app/ledger/AcceptedLedger.h>
#include <ripple/app/ledger/LedgerMaster.h>
#include <ripple/app/ledger/LedgerToJson.h>
#include <ripple/app/ledger/TransactionMaster.h>
#include <ripple/app/misc/Manifest.h>
#include <ripple/app/misc/impl/AccountTxPaging.h>
#include <ripple/app/rdb/backend/LMDBDatabase.h>
#include <ripple/app/rdb/backend/detail/Node.h>
#include <ripple/app/rdb/backend/detail/Shard.h>
#include <ripple/basics/BasicConfig.h>
#include <ripple/basics/StringUtilities.h>
#include <ripple/core/DatabaseCon.h>
#include <ripple/core/SociDB.h>
#include <ripple/json/to_string.h>
#include <ripple/nodestore/DatabaseShard.h>
// #include <soci/sqlite3/soci-sqlite3.h>
namespace ripple {
class LMDBDatabaseImp final : public LMDBDatabase
{
public:
LMDBDatabaseImp(Application& app, Config const& config, JobQueue& jobQueue)
: app_(app)
, useTxTables_(config.useTxTables())
, j_(app_.journal("LMDBDatabaseImp"))
{
DatabaseCon::Setup const setup = setup_DatabaseCon(config, j_);
if (!makeLedgerDBs(
config,
setup,
DatabaseCon::CheckpointerSetup{&jobQueue, &app_.logs()}))
{
std::string_view constexpr error =
"Failed to create ledger databases";
JLOG(j_.fatal()) << error;
Throw<std::runtime_error>(error.data());
}
if (app.getShardStore() &&
!makeMetaDBs(
config,
setup,
DatabaseCon::CheckpointerSetup{&jobQueue, &app_.logs()}))
{
std::string_view constexpr error =
"Failed to create metadata databases";
JLOG(j_.fatal()) << error;
Throw<std::runtime_error>(error.data());
}
}
std::optional<LedgerIndex>
getMinLedgerSeq() override;
std::optional<LedgerIndex>
getTransactionsMinLedgerSeq() override;
std::optional<LedgerIndex>
getAccountTransactionsMinLedgerSeq() override;
std::optional<LedgerIndex>
getMaxLedgerSeq() override;
void
deleteTransactionByLedgerSeq(LedgerIndex ledgerSeq) override;
void
deleteBeforeLedgerSeq(LedgerIndex ledgerSeq) override;
void
deleteTransactionsBeforeLedgerSeq(LedgerIndex ledgerSeq) override;
void
deleteAccountTransactionsBeforeLedgerSeq(LedgerIndex ledgerSeq) override;
std::size_t
getTransactionCount() override;
std::size_t
getAccountTransactionCount() override;
RelationalDatabase::CountMinMax
getLedgerCountMinMax() override;
bool
saveValidatedLedger(
std::shared_ptr<Ledger const> const& ledger,
bool current) override;
std::optional<LedgerInfo>
getLedgerInfoByIndex(LedgerIndex ledgerSeq) override;
std::optional<LedgerInfo>
getNewestLedgerInfo() override;
std::optional<LedgerInfo>
getLimitedOldestLedgerInfo(LedgerIndex ledgerFirstIndex) override;
std::optional<LedgerInfo>
getLimitedNewestLedgerInfo(LedgerIndex ledgerFirstIndex) override;
std::optional<LedgerInfo>
getLedgerInfoByHash(uint256 const& ledgerHash) override;
uint256
getHashByIndex(LedgerIndex ledgerIndex) override;
std::optional<LedgerHashPair>
getHashesByIndex(LedgerIndex ledgerIndex) override;
std::map<LedgerIndex, LedgerHashPair>
getHashesByIndex(LedgerIndex minSeq, LedgerIndex maxSeq) override;
std::vector<std::shared_ptr<Transaction>>
getTxHistory(LedgerIndex startIndex) override;
AccountTxs
getOldestAccountTxs(AccountTxOptions const& options) override;
AccountTxs
getNewestAccountTxs(AccountTxOptions const& options) override;
MetaTxsList
getOldestAccountTxsB(AccountTxOptions const& options) override;
MetaTxsList
getNewestAccountTxsB(AccountTxOptions const& options) override;
std::pair<AccountTxs, std::optional<AccountTxMarker>>
oldestAccountTxPage(AccountTxPageOptions const& options) override;
std::pair<AccountTxs, std::optional<AccountTxMarker>>
newestAccountTxPage(AccountTxPageOptions const& options) override;
std::pair<MetaTxsList, std::optional<AccountTxMarker>>
oldestAccountTxPageB(AccountTxPageOptions const& options) override;
std::pair<MetaTxsList, std::optional<AccountTxMarker>>
newestAccountTxPageB(AccountTxPageOptions const& options) override;
std::variant<AccountTx, TxSearched>
getTransaction(
uint256 const& id,
std::optional<ClosedInterval<std::uint32_t>> const& range,
error_code_i& ec) override;
bool
ledgerDbHasSpace(Config const& config) override;
bool
transactionDbHasSpace(Config const& config) override;
std::uint32_t
getKBUsedAll() override;
std::uint32_t
getKBUsedLedger() override;
std::uint32_t
getKBUsedTransaction() override;
void
closeLedgerDB() override;
void
closeTransactionDB() override;
private:
Application& app_;
bool const useTxTables_;
beast::Journal j_;
std::unique_ptr<DatabaseCon> lgrdb_, txdb_;
std::unique_ptr<DatabaseCon> lgrMetaDB_, txMetaDB_;
/**
* @brief makeLedgerDBs Opens ledger and transaction databases for the node
* store, and stores their descriptors in private member variables.
* @param config Config object.
* @param setup Path to the databases and other opening parameters.
* @param checkpointerSetup Checkpointer parameters.
* @return True if node databases opened successfully.
*/
bool
makeLedgerDBs(
Config const& config,
DatabaseCon::Setup const& setup,
DatabaseCon::CheckpointerSetup const& checkpointerSetup);
/**
* @brief makeMetaDBs Opens shard index lookup databases, and stores
* their descriptors in private member variables.
* @param config Config object.
* @param setup Path to the databases and other opening parameters.
* @param checkpointerSetup Checkpointer parameters.
* @return True if node databases opened successfully.
*/
bool
makeMetaDBs(
Config const& config,
DatabaseCon::Setup const& setup,
DatabaseCon::CheckpointerSetup const& checkpointerSetup);
/**
* @brief seqToShardIndex Provides the index of the shard that stores the
* ledger with the given sequence.
* @param ledgerSeq Ledger sequence.
* @return Shard index.
*/
std::uint32_t
seqToShardIndex(LedgerIndex ledgerSeq)
{
return app_.getShardStore()->seqToShardIndex(ledgerSeq);
}
/**
* @brief firstLedgerSeq Returns the sequence of the first ledger stored in
* the shard specified by the shard index parameter.
* @param shardIndex Shard Index.
* @return First ledger sequence.
*/
LedgerIndex
firstLedgerSeq(std::uint32_t shardIndex)
{
return app_.getShardStore()->firstLedgerSeq(shardIndex);
}
/**
* @brief lastLedgerSeq Returns the sequence of the last ledger stored in
* the shard specified by the shard index parameter.
* @param shardIndex Shard Index.
* @return Last ledger sequence.
*/
LedgerIndex
lastLedgerSeq(std::uint32_t shardIndex)
{
return app_.getShardStore()->lastLedgerSeq(shardIndex);
}
/**
* @brief existsLedger Checks if the node store ledger database exists.
* @return True if the node store ledger database exists.
*/
bool
existsLedger()
{
return static_cast<bool>(lgrdb_);
}
/**
* @brief existsTransaction Checks if the node store transaction database
* exists.
* @return True if the node store transaction database exists.
*/
bool
existsTransaction()
{
return static_cast<bool>(txdb_);
}
/**
* shardStoreExists Checks whether the shard store exists
* @return True if the shard store exists
*/
bool
shardStoreExists()
{
return app_.getShardStore() != nullptr;
}
/**
* @brief checkoutLedger Checks out and returns node store ledger
* database.
* @return Session to the node store ledger database.
*/
auto
checkoutLedger()
{
return lgrdb_->checkoutLMDB();
}
/**
* @brief checkoutTransaction Checks out and returns the node store
* transaction database.
* @return Session to the node store transaction database.
*/
auto
checkoutTransaction()
{
return txdb_->checkoutLMDB();
}
};
bool
LMDBDatabaseImp::makeLedgerDBs(
Config const& config,
DatabaseCon::Setup const& setup,
DatabaseCon::CheckpointerSetup const& checkpointerSetup)
{
auto [lgr, tx, res] =
detail::makeLedgerDBs(config, setup, checkpointerSetup);
txdb_ = std::move(tx);
lgrdb_ = std::move(lgr);
return res;
}
bool
LMDBDatabaseImp::makeMetaDBs(
Config const& config,
DatabaseCon::Setup const& setup,
DatabaseCon::CheckpointerSetup const& checkpointerSetup)
{
auto [lgrMetaDB, txMetaDB] =
detail::makeMetaDBs(config, setup, checkpointerSetup);
txMetaDB_ = std::move(txMetaDB);
lgrMetaDB_ = std::move(lgrMetaDB);
return true;
}
std::optional<LedgerIndex>
LMDBDatabaseImp::getMinLedgerSeq()
{
/* if databases exists, use it */
if (existsLedger())
{
auto db = checkoutLedger();
return detail::getMinLedgerSeq(db.get(), detail::TableType::Ledgers);
}
/* else return empty value */
return {};
}
std::optional<LedgerIndex>
LMDBDatabaseImp::getTransactionsMinLedgerSeq()
{
if (!useTxTables_)
return {};
if (existsTransaction())
{
auto db = checkoutTransaction();
return detail::getMinLedgerSeq(
db.get(), detail::TableType::Transactions);
}
return {};
}
std::optional<LedgerIndex>
LMDBDatabaseImp::getAccountTransactionsMinLedgerSeq()
{
if (!useTxTables_)
return {};
if (existsTransaction())
{
auto db = checkoutTransaction();
return detail::getMinLedgerSeq(
db.get(), detail::TableType::AccountTransactions);
}
return {};
}
std::optional<LedgerIndex>
LMDBDatabaseImp::getMaxLedgerSeq()
{
if (existsLedger())
{
auto db = checkoutLedger();
return detail::getMaxLedgerSeq(db.get(), detail::TableType::Ledgers);
}
return {};
}
void
LMDBDatabaseImp::deleteTransactionByLedgerSeq(LedgerIndex ledgerSeq)
{
if (!useTxTables_)
return;
if (existsTransaction())
{
auto db = checkoutTransaction();
detail::deleteByLedgerSeq(
db.get(), detail::TableType::Transactions, ledgerSeq);
return;
}
}
void
LMDBDatabaseImp::deleteBeforeLedgerSeq(LedgerIndex ledgerSeq)
{
if (existsLedger())
{
auto db = checkoutLedger();
detail::deleteBeforeLedgerSeq(
db.get(), detail::TableType::Ledgers, ledgerSeq);
return;
}
}
void
LMDBDatabaseImp::deleteTransactionsBeforeLedgerSeq(LedgerIndex ledgerSeq)
{
if (!useTxTables_)
return;
if (existsTransaction())
{
auto db = checkoutTransaction();
detail::deleteBeforeLedgerSeq(
db.get(), detail::TableType::Transactions, ledgerSeq);
return;
}
}
void
LMDBDatabaseImp::deleteAccountTransactionsBeforeLedgerSeq(LedgerIndex ledgerSeq)
{
if (!useTxTables_)
return;
if (existsTransaction())
{
auto db = checkoutTransaction();
detail::deleteBeforeLedgerSeq(
db.get(), detail::TableType::AccountTransactions, ledgerSeq);
return;
}
}
std::size_t
LMDBDatabaseImp::getTransactionCount()
{
if (!useTxTables_)
return 0;
if (existsTransaction())
{
auto db = checkoutTransaction();
return detail::getRows(db.get(), detail::TableType::Transactions);
}
return 0;
}
std::size_t
LMDBDatabaseImp::getAccountTransactionCount()
{
if (!useTxTables_)
return 0;
if (existsTransaction())
{
auto db = checkoutTransaction();
return detail::getRows(
db.get(), detail::TableType::AccountTransactions);
}
return 0;
}
RelationalDatabase::CountMinMax
LMDBDatabaseImp::getLedgerCountMinMax()
{
if (existsLedger())
{
auto db = checkoutLedger();
return detail::getRowsMinMax(db.get(), detail::TableType::Ledgers);
}
return {0, 0, 0};
}
bool
LMDBDatabaseImp::saveValidatedLedger(
std::shared_ptr<Ledger const> const& ledger,
bool current)
{
if (existsLedger())
{
if (!detail::saveValidatedLedger(
*lgrdb_, *txdb_, app_, ledger, current))
return false;
}
return true;
}
std::optional<LedgerInfo>
LMDBDatabaseImp::getLedgerInfoByIndex(LedgerIndex ledgerSeq)
{
if (existsLedger())
{
auto db = checkoutLedger();
auto const res = detail::getLedgerInfoByIndex(db.get(), ledgerSeq, j_);
if (res.has_value())
return res;
}
return {};
}
std::optional<LedgerInfo>
LMDBDatabaseImp::getNewestLedgerInfo()
{
if (existsLedger())
{
auto db = checkoutLedger();
auto const res = detail::getNewestLedgerInfo(db.get(), j_);
if (res.has_value())
return res;
}
return {};
}
std::optional<LedgerInfo>
LMDBDatabaseImp::getLimitedOldestLedgerInfo(LedgerIndex ledgerFirstIndex)
{
return {};
}
std::optional<LedgerInfo>
LMDBDatabaseImp::getLimitedNewestLedgerInfo(LedgerIndex ledgerFirstIndex)
{
return {};
}
std::optional<LedgerInfo>
LMDBDatabaseImp::getLedgerInfoByHash(uint256 const& ledgerHash)
{
if (existsLedger())
{
auto db = checkoutLedger();
auto const res = detail::getLedgerInfoByHash(db.get(), ledgerHash, j_);
if (res.has_value())
return res;
}
return {};
}
uint256
LMDBDatabaseImp::getHashByIndex(LedgerIndex ledgerIndex)
{
if (existsLedger())
{
auto db = checkoutLedger();
auto const res = detail::getHashByIndex(db.get(), ledgerIndex);
if (res.isNonZero())
return res;
}
return uint256();
}
std::optional<LedgerHashPair>
LMDBDatabaseImp::getHashesByIndex(LedgerIndex ledgerIndex)
{
if (existsLedger())
{
auto db = checkoutLedger();
auto const res = detail::getHashesByIndex(db.get(), ledgerIndex, j_);
if (res.has_value())
return res;
}
return {};
}
std::map<LedgerIndex, LedgerHashPair>
LMDBDatabaseImp::getHashesByIndex(LedgerIndex minSeq, LedgerIndex maxSeq)
{
if (existsLedger())
{
auto db = checkoutLedger();
auto const res = detail::getHashesByIndex(db.get(), minSeq, maxSeq, j_);
if (!res.empty())
return res;
}
return {};
}
std::vector<std::shared_ptr<Transaction>>
LMDBDatabaseImp::getTxHistory(LedgerIndex startIndex)
{
return {};
}
RelationalDatabase::AccountTxs
LMDBDatabaseImp::getOldestAccountTxs(AccountTxOptions const& options)
{
return {};
}
RelationalDatabase::AccountTxs
LMDBDatabaseImp::getNewestAccountTxs(AccountTxOptions const& options)
{
return {};
}
RelationalDatabase::MetaTxsList
LMDBDatabaseImp::getOldestAccountTxsB(AccountTxOptions const& options)
{
return {};
}
RelationalDatabase::MetaTxsList
LMDBDatabaseImp::getNewestAccountTxsB(AccountTxOptions const& options)
{
return {};
}
std::pair<
RelationalDatabase::AccountTxs,
std::optional<RelationalDatabase::AccountTxMarker>>
LMDBDatabaseImp::oldestAccountTxPage(AccountTxPageOptions const& options)
{
return {};
}
std::pair<
RelationalDatabase::AccountTxs,
std::optional<RelationalDatabase::AccountTxMarker>>
LMDBDatabaseImp::newestAccountTxPage(AccountTxPageOptions const& options)
{
return {};
}
std::pair<
RelationalDatabase::MetaTxsList,
std::optional<RelationalDatabase::AccountTxMarker>>
LMDBDatabaseImp::oldestAccountTxPageB(AccountTxPageOptions const& options)
{
return {};
}
std::pair<
RelationalDatabase::MetaTxsList,
std::optional<RelationalDatabase::AccountTxMarker>>
LMDBDatabaseImp::newestAccountTxPageB(AccountTxPageOptions const& options)
{
return {};
}
std::variant<RelationalDatabase::AccountTx, TxSearched>
LMDBDatabaseImp::getTransaction(
uint256 const& id,
std::optional<ClosedInterval<std::uint32_t>> const& range,
error_code_i& ec)
{
if (!useTxTables_)
return TxSearched::unknown;
if (existsTransaction())
{
auto db = checkoutTransaction();
return detail::getTransaction(db.get(), app_, id, range, ec);
}
return TxSearched::unknown;
}
bool
LMDBDatabaseImp::ledgerDbHasSpace(Config const& config)
{
if (existsLedger())
{
auto db = checkoutLedger();
return detail::dbHasSpace(db.get(), config, j_);
}
return true;
}
bool
LMDBDatabaseImp::transactionDbHasSpace(Config const& config)
{
if (!useTxTables_)
return true;
if (existsTransaction())
{
auto db = checkoutTransaction();
return detail::dbHasSpace(db.get(), config, j_);
}
return true;
}
std::uint32_t
LMDBDatabaseImp::getKBUsedAll()
{
// if (existsLedger())
// {
// return ripple::getKBUsedAll(lgrdb_->getLMDB());
// }
return 0;
}
std::uint32_t
LMDBDatabaseImp::getKBUsedLedger()
{
// if (existsLedger())
// {
// return ripple::getKBUsedDB(lgrdb_->getLMDB());
// }
return 0;
}
std::uint32_t
LMDBDatabaseImp::getKBUsedTransaction()
{
if (!useTxTables_)
return 0;
// if (existsTransaction())
// {
// return ripple::getKBUsedDB(txdb_->getLMDB());
// }
return 0;
}
void
LMDBDatabaseImp::closeLedgerDB()
{
lgrdb_.reset();
}
void
LMDBDatabaseImp::closeTransactionDB()
{
txdb_.reset();
}
std::unique_ptr<RelationalDatabase>
getLMDBDatabase(Application& app, Config const& config, JobQueue& jobQueue)
{
return std::make_unique<LMDBDatabaseImp>(app, config, jobQueue);
}
} // namespace ripple

View File

@@ -30,6 +30,9 @@ getSQLiteDatabase(Application& app, Config const& config, JobQueue& jobQueue);
extern std::unique_ptr<RelationalDatabase>
getPostgresDatabase(Application& app, Config const& config, JobQueue& jobQueue);
extern std::unique_ptr<RelationalDatabase>
getLMDBDatabase(Application& app, Config const& config, JobQueue& jobQueue);
std::unique_ptr<RelationalDatabase>
RelationalDatabase::init(
Application& app,
@@ -38,32 +41,37 @@ RelationalDatabase::init(
{
bool use_sqlite = false;
bool use_postgres = false;
bool use_lmdb = true;
if (config.reporting())
{
use_postgres = true;
}
else
{
const Section& rdb_section{config.section(SECTION_RELATIONAL_DB)};
if (!rdb_section.empty())
{
if (boost::iequals(get(rdb_section, "backend"), "sqlite"))
{
use_sqlite = true;
}
else
{
Throw<std::runtime_error>(
"Invalid rdb_section backend value: " +
get(rdb_section, "backend"));
}
}
else
{
use_sqlite = true;
}
}
// if (config.reporting())
// {
// use_postgres = true;
// }
// else
// {
// const Section& rdb_section{config.section(SECTION_RELATIONAL_DB)};
// if (!rdb_section.empty())
// {
// if (boost::iequals(get(rdb_section, "backend"), "sqlite"))
// {
// use_sqlite = true;
// }
// else if (boost::iequals(get(rdb_section, "backend"), "lmdb"))
// {
// use_lmdb = true;
// }
// else
// {
// Throw<std::runtime_error>(
// "Invalid rdb_section backend value: " +
// get(rdb_section, "backend"));
// }
// }
// else
// {
// use_sqlite = true;
// }
// }
if (use_sqlite)
{
@@ -73,6 +81,10 @@ RelationalDatabase::init(
{
return getPostgresDatabase(app, config, jobQueue);
}
else if (use_lmdb)
{
return getLMDBDatabase(app, config, jobQueue);
}
return std::unique_ptr<RelationalDatabase>();
}

View File

@@ -26,16 +26,84 @@ std::unique_ptr<DatabaseCon>
makeWalletDB(DatabaseCon::Setup const& setup)
{
// wallet database
return std::make_unique<DatabaseCon>(
setup, WalletDBName, std::array<char const*, 0>(), WalletDBInit);
if (setup.sqlite3)
{
return std::make_unique<DatabaseCon>(
setup, WalletDBName, std::array<char const*, 0>(), WalletDBInit);
}
return std::make_unique<DatabaseCon>(setup, LMDBWalletDBName);
}
std::unique_ptr<DatabaseCon>
makeTestWalletDB(DatabaseCon::Setup const& setup, std::string const& dbname)
{
// wallet database
return std::make_unique<DatabaseCon>(
setup, dbname.data(), std::array<char const*, 0>(), WalletDBInit);
if (setup.sqlite3)
{
return std::make_unique<DatabaseCon>(
setup, dbname.data(), std::array<char const*, 0>(), WalletDBInit);
}
return std::make_unique<DatabaseCon>(setup, dbname);
}
void
getManifests(
MDB_env* env,
std::string const& dbTable,
ManifestCache& mCache,
beast::Journal j)
{
MDB_txn* txn;
MDB_dbi dbi;
MDB_val key, data;
int rc;
rc = mdb_txn_begin(env, nullptr, MDB_RDONLY, &txn);
if (rc != 0)
{
// Handle error
return;
}
rc = mdb_dbi_open(txn, dbTable.c_str(), 0, &dbi);
if (rc != 0)
{
mdb_txn_abort(txn);
// Handle error
return;
}
MDB_cursor* cursor;
rc = mdb_cursor_open(txn, dbi, &cursor);
if (rc != 0)
{
mdb_txn_abort(txn);
// Handle error
return;
}
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0)
{
std::string serialized(static_cast<char*>(data.mv_data), data.mv_size);
if (auto mo = deserializeManifest(serialized))
{
if (!mo->verify())
{
JLOG(j.warn()) << "Unverifiable manifest in db";
continue;
}
mCache.applyManifest(std::move(*mo));
}
else
{
JLOG(j.warn()) << "Malformed manifest in database";
}
}
mdb_cursor_close(cursor);
mdb_txn_commit(txn);
mdb_dbi_close(env, dbi);
}
void
@@ -71,6 +139,18 @@ getManifests(
}
}
static void
saveManifest(MDB_txn* txn, MDB_dbi dbi, std::string const& serialized)
{
MDB_val key, data;
key.mv_size = serialized.size();
key.mv_data = const_cast<char*>(serialized.data());
data.mv_size = serialized.size();
data.mv_data = const_cast<char*>(serialized.data());
mdb_put(txn, dbi, &key, &data, 0);
}
static void
saveManifest(
soci::session& session,
@@ -86,6 +166,56 @@ saveManifest(
soci::use(rawData);
}
void
saveManifests(
MDB_env* env,
std::string const& dbTable,
std::function<bool(PublicKey const&)> const& isTrusted,
hash_map<PublicKey, Manifest> const& map,
beast::Journal j)
{
MDB_txn* txn;
MDB_dbi dbi;
int rc;
rc = mdb_txn_begin(env, nullptr, 0, &txn);
if (rc != 0)
{
// Handle error
return;
}
rc = mdb_dbi_open(txn, dbTable.c_str(), 0, &dbi);
if (rc != 0)
{
mdb_txn_abort(txn);
// Handle error
return;
}
rc = mdb_drop(txn, dbi, 0);
if (rc != 0)
{
mdb_txn_abort(txn);
// Handle error
return;
}
for (auto const& v : map)
{
if (!v.second.revoked() && !isTrusted(v.second.masterKey))
{
JLOG(j.info()) << "Untrusted manifest in cache not saved to db";
continue;
}
saveManifest(txn, dbi, v.second.serialized);
}
mdb_txn_commit(txn);
mdb_dbi_close(env, dbi);
}
void
saveManifests(
soci::session& session,
@@ -111,6 +241,34 @@ saveManifests(
tr.commit();
}
void
addValidatorManifest(MDB_env* env, std::string const& serialized)
{
MDB_txn* txn;
MDB_dbi dbi;
int rc;
rc = mdb_txn_begin(env, nullptr, 0, &txn);
if (rc != 0)
{
// Handle error
return;
}
rc = mdb_dbi_open(txn, "ValidatorManifests", 0, &dbi);
if (rc != 0)
{
mdb_txn_abort(txn);
// Handle error
return;
}
saveManifest(txn, dbi, serialized);
mdb_txn_commit(txn);
mdb_dbi_close(env, dbi);
}
void
addValidatorManifest(soci::session& session, std::string const& serialized)
{
@@ -119,12 +277,91 @@ addValidatorManifest(soci::session& session, std::string const& serialized)
tr.commit();
}
void
clearNodeIdentity(MDB_env* env)
{
MDB_txn* txn;
MDB_dbi dbi;
mdb_txn_begin(env, nullptr, 0, &txn);
mdb_dbi_open(txn, "NodeIdentity", 0, &dbi);
mdb_drop(txn, dbi, 0);
mdb_txn_commit(txn);
mdb_dbi_close(env, dbi);
}
void
clearNodeIdentity(soci::session& session)
{
session << "DELETE FROM NodeIdentity;";
}
std::pair<PublicKey, SecretKey>
getNodeIdentity(MDB_env* env)
{
MDB_txn* txn;
MDB_dbi dbi;
MDB_val key, data;
mdb_txn_begin(env, nullptr, MDB_RDONLY, &txn);
mdb_dbi_open(txn, "NodeIdentity", 0, &dbi);
key.mv_size = sizeof("PublicKey");
key.mv_data = const_cast<char*>("PublicKey");
std::string pubKO, priKO;
if (mdb_get(txn, dbi, &key, &data) == MDB_SUCCESS)
{
pubKO.assign(static_cast<char*>(data.mv_data), data.mv_size);
key.mv_size = sizeof("PrivateKey");
key.mv_data = const_cast<char*>("PrivateKey");
if (mdb_get(txn, dbi, &key, &data) == MDB_SUCCESS)
{
priKO.assign(static_cast<char*>(data.mv_data), data.mv_size);
auto const sk =
parseBase58<SecretKey>(TokenType::NodePrivate, priKO);
auto const pk =
parseBase58<PublicKey>(TokenType::NodePublic, pubKO);
if (sk && pk && (*pk == derivePublicKey(KeyType::secp256k1, *sk)))
{
mdb_txn_commit(txn);
mdb_dbi_close(env, dbi);
return {*pk, *sk};
}
}
}
mdb_txn_abort(txn);
mdb_dbi_close(env, dbi);
auto [newpublicKey, newsecretKey] = randomKeyPair(KeyType::secp256k1);
mdb_txn_begin(env, nullptr, 0, &txn);
mdb_dbi_open(txn, "NodeIdentity", MDB_CREATE, &dbi);
std::string newPubKeyStr = toBase58(TokenType::NodePublic, newpublicKey);
std::string newPriKeyStr = toBase58(TokenType::NodePrivate, newsecretKey);
key.mv_size = sizeof("PublicKey");
key.mv_data = const_cast<char*>("PublicKey");
data.mv_size = newPubKeyStr.size();
data.mv_data = const_cast<char*>(newPubKeyStr.data());
mdb_put(txn, dbi, &key, &data, 0);
key.mv_size = sizeof("PrivateKey");
key.mv_data = const_cast<char*>("PrivateKey");
data.mv_size = newPriKeyStr.size();
data.mv_data = const_cast<char*>(newPriKeyStr.data());
mdb_put(txn, dbi, &key, &data, 0);
mdb_txn_commit(txn);
mdb_dbi_close(env, dbi);
return {newpublicKey, newsecretKey};
}
std::pair<PublicKey, SecretKey>
getNodeIdentity(soci::session& session)
{
@@ -162,6 +399,42 @@ getNodeIdentity(soci::session& session)
return {newpublicKey, newsecretKey};
}
std::unordered_set<PeerReservation, beast::uhash<>, KeyEqual>
getPeerReservationTable(MDB_env* env, beast::Journal j)
{
std::unordered_set<PeerReservation, beast::uhash<>, KeyEqual> table;
MDB_txn* txn;
MDB_dbi dbi;
MDB_val key, data;
MDB_cursor* cursor;
// Start a read-only transaction
mdb_txn_begin(env, nullptr, MDB_RDONLY, &txn);
mdb_dbi_open(txn, "PeerReservations", 0, &dbi);
mdb_cursor_open(txn, dbi, &cursor);
while (mdb_cursor_get(cursor, &key, &data, MDB_NEXT) == 0)
{
std::string valPubKey(static_cast<char*>(key.mv_data), key.mv_size);
std::string valDesc(static_cast<char*>(data.mv_data), data.mv_size);
auto const optNodeId =
parseBase58<PublicKey>(TokenType::NodePublic, valPubKey);
if (!optNodeId)
{
JLOG(j.warn()) << "load: not a public key: " << valPubKey;
continue;
}
table.insert(PeerReservation{*optNodeId, valDesc});
}
mdb_cursor_close(cursor);
mdb_txn_commit(txn);
mdb_dbi_close(env, dbi);
return table;
}
std::unordered_set<PeerReservation, beast::uhash<>, KeyEqual>
getPeerReservationTable(soci::session& session, beast::Journal j)
{
@@ -199,6 +472,30 @@ getPeerReservationTable(soci::session& session, beast::Journal j)
return table;
}
void
insertPeerReservation(
MDB_env* env,
PublicKey const& nodeId,
std::string const& description)
{
MDB_txn* txn;
MDB_dbi dbi;
MDB_val key, data;
auto const sNodeId = toBase58(TokenType::NodePublic, nodeId);
key.mv_size = sNodeId.size();
key.mv_data = const_cast<char*>(sNodeId.data());
data.mv_size = description.size();
data.mv_data = const_cast<char*>(description.data());
// Start a write transaction
mdb_txn_begin(env, nullptr, 0, &txn);
mdb_dbi_open(txn, "PeerReservations", MDB_CREATE, &dbi);
mdb_put(txn, dbi, &key, &data, 0);
mdb_txn_commit(txn);
mdb_dbi_close(env, dbi);
}
void
insertPeerReservation(
soci::session& session,
@@ -213,6 +510,25 @@ insertPeerReservation(
soci::use(sNodeId), soci::use(description);
}
void
deletePeerReservation(MDB_env* env, PublicKey const& nodeId)
{
MDB_txn* txn;
MDB_dbi dbi;
MDB_val key;
auto const sNodeId = toBase58(TokenType::NodePublic, nodeId);
key.mv_size = sNodeId.size();
key.mv_data = const_cast<char*>(sNodeId.data());
// Start a write transaction
mdb_txn_begin(env, nullptr, 0, &txn);
mdb_dbi_open(txn, "PeerReservations", 0, &dbi);
mdb_del(txn, dbi, &key, nullptr);
mdb_txn_commit(txn);
mdb_dbi_close(env, dbi);
}
void
deletePeerReservation(soci::session& session, PublicKey const& nodeId)
{
@@ -221,6 +537,31 @@ deletePeerReservation(soci::session& session, PublicKey const& nodeId)
soci::use(sNodeId);
}
bool
createFeatureVotes(MDB_env* env)
{
// soci::transaction tr(session);
// std::string sql =
// "SELECT count(*) FROM sqlite_master "
// "WHERE type='table' AND name='FeatureVotes'";
// // SOCI requires boost::optional (not std::optional) as the parameter.
// boost::optional<int> featureVotesCount;
// session << sql, soci::into(featureVotesCount);
// bool exists = static_cast<bool>(*featureVotesCount);
// // Create FeatureVotes table in WalletDB if it doesn't exist
// if (!exists)
// {
// session << "CREATE TABLE FeatureVotes ( "
// "AmendmentHash CHARACTER(64) NOT NULL, "
// "AmendmentName TEXT, "
// "Veto INTEGER NOT NULL );";
// tr.commit();
// }
// return exists;
return false;
}
bool
createFeatureVotes(soci::session& session)
{
@@ -245,6 +586,42 @@ createFeatureVotes(soci::session& session)
return exists;
}
void
readAmendments(
MDB_env* env,
std::function<void(
boost::optional<std::string> amendment_hash,
boost::optional<std::string> amendment_name,
boost::optional<AmendmentVote> vote)> const& callback)
{
// lambda that converts the internally stored int to an AmendmentVote.
// auto intToVote = [](boost::optional<int> const& dbVote)
// -> boost::optional<AmendmentVote> {
// return safe_cast<AmendmentVote>(dbVote.value_or(1));
// };
// soci::transaction tr(session);
// std::string sql =
// "SELECT AmendmentHash, AmendmentName, Veto FROM "
// "( SELECT AmendmentHash, AmendmentName, Veto, RANK() OVER "
// "( PARTITION BY AmendmentHash ORDER BY ROWID DESC ) "
// "as rnk FROM FeatureVotes ) WHERE rnk = 1";
// // SOCI requires boost::optional (not std::optional) as parameters.
// boost::optional<std::string> amendment_hash;
// boost::optional<std::string> amendment_name;
// boost::optional<int> vote_to_veto;
// soci::statement st =
// (session.prepare << sql,
// soci::into(amendment_hash),
// soci::into(amendment_name),
// soci::into(vote_to_veto));
// st.execute();
// while (st.fetch())
// {
// callback(amendment_hash, amendment_name, intToVote(vote_to_veto));
// }
}
void
readAmendments(
soci::session& session,
@@ -281,6 +658,25 @@ readAmendments(
}
}
void
voteAmendment(
MDB_env* env,
uint256 const& amendment,
std::string const& name,
AmendmentVote vote)
{
// soci::transaction tr(session);
// std::string sql =
// "INSERT INTO FeatureVotes (AmendmentHash, AmendmentName, Veto) VALUES
// "
// "('";
// sql += to_string(amendment);
// sql += "', '" + name;
// sql += "', '" + std::to_string(safe_cast<int>(vote)) + "');";
// session << sql;
// tr.commit();
}
void
voteAmendment(
soci::session& session,

View File

@@ -194,14 +194,8 @@ ETLSource::onResolve(
{
boost::beast::get_lowest_layer(*ws_).expires_after(
std::chrono::seconds(30));
// Use async_connect with the entire results
boost::beast::get_lowest_layer(*ws_).async_connect(
results,
[this](
boost::beast::error_code ec,
boost::asio::ip::tcp::resolver::results_type::endpoint_type
ep) { onConnect(ec, ep); });
results, [this](auto ec, auto ep) { onConnect(ec, ep); });
}
}

View File

@@ -24,7 +24,6 @@
#include <ripple/beast/utility/Journal.h>
#include <boost/beast/core/string.hpp>
#include <boost/filesystem.hpp>
#include <fstream>
#include <map>
#include <memory>
#include <mutex>

View File

@@ -549,8 +549,7 @@ using uint128 = base_uint<128>;
using uint160 = base_uint<160>;
using uint256 = base_uint<256>;
/*
* template <std::size_t Bits, class Tag>
template <std::size_t Bits, class Tag>
[[nodiscard]] inline constexpr std::strong_ordering
operator<=>(base_uint<Bits, Tag> const& lhs, base_uint<Bits, Tag> const& rhs)
{
@@ -571,19 +570,6 @@ operator<=>(base_uint<Bits, Tag> const& lhs, base_uint<Bits, Tag> const& rhs)
return (*ret.first > *ret.second) ? std::strong_ordering::greater
: std::strong_ordering::less;
}
*/
template <std::size_t Bits, class Tag>
[[nodiscard]] inline constexpr std::strong_ordering
operator<=>(base_uint<Bits, Tag> const& lhs, base_uint<Bits, Tag> const& rhs)
{
return std::lexicographical_compare_three_way(
lhs.cbegin(),
lhs.cend(),
rhs.cbegin(),
rhs.cend(),
std::compare_three_way{});
}
template <std::size_t Bits, typename Tag>
[[nodiscard]] inline constexpr bool

View File

@@ -18,7 +18,6 @@
//==============================================================================
#include <ripple/basics/FileUtilities.h>
#include <fstream>
namespace ripple {
@@ -42,7 +41,7 @@ getFileContents(
return {};
}
std::ifstream fileStream(fullPath.string(), std::ios::in);
ifstream fileStream(fullPath, std::ios::in);
if (!fileStream)
{
@@ -72,8 +71,7 @@ writeFileContents(
using namespace boost::filesystem;
using namespace boost::system::errc;
std::ofstream fileStream(
destPath.string(), std::ios::out | std::ios::trunc);
ofstream fileStream(destPath, std::ios::out | std::ios::trunc);
if (!fileStream)
{

View File

@@ -145,111 +145,78 @@ private:
};
// VFALCO TODO This should only be enabled for maps.
class pair_value_compare
: public beast::detail::empty_base_optimization<Compare>
#ifdef _LIBCPP_VERSION
,
public std::binary_function<value_type, value_type, bool>
#endif
class pair_value_compare : public Compare
{
public:
#ifndef _LIBCPP_VERSION
using first_argument = value_type;
using second_argument = value_type;
using result_type = bool;
#endif
bool
operator()(value_type const& lhs, value_type const& rhs) const
{
return this->member()(lhs.first, rhs.first);
return Compare::operator()(lhs.first, rhs.first);
}
pair_value_compare()
{
}
pair_value_compare(pair_value_compare const& other)
: beast::detail::empty_base_optimization<Compare>(other)
pair_value_compare(pair_value_compare const& other) : Compare(other)
{
}
private:
friend aged_ordered_container;
pair_value_compare(Compare const& compare)
: beast::detail::empty_base_optimization<Compare>(compare)
pair_value_compare(Compare const& compare) : Compare(compare)
{
}
};
// Compares value_type against element, used in insert_check
// VFALCO TODO hoist to remove template argument dependencies
class KeyValueCompare
: public beast::detail::empty_base_optimization<Compare>
#ifdef _LIBCPP_VERSION
,
public std::binary_function<Key, element, bool>
#endif
class KeyValueCompare : public Compare
{
public:
#ifndef _LIBCPP_VERSION
using first_argument = Key;
using second_argument = element;
using result_type = bool;
#endif
KeyValueCompare() = default;
KeyValueCompare(Compare const& compare)
: beast::detail::empty_base_optimization<Compare>(compare)
KeyValueCompare(Compare const& compare) : Compare(compare)
{
}
// VFALCO NOTE WE might want only to enable these overloads
// if Compare has is_transparent
#if 0
template <class K>
bool operator() (K const& k, element const& e) const
{
return this->member() (k, extract (e.value));
}
template <class K>
bool operator() (element const& e, K const& k) const
{
return this->member() (extract (e.value), k);
}
#endif
bool
operator()(Key const& k, element const& e) const
{
return this->member()(k, extract(e.value));
return Compare::operator()(k, extract(e.value));
}
bool
operator()(element const& e, Key const& k) const
{
return this->member()(extract(e.value), k);
return Compare::operator()(extract(e.value), k);
}
bool
operator()(element const& x, element const& y) const
{
return this->member()(extract(x.value), extract(y.value));
return Compare::operator()(extract(x.value), extract(y.value));
}
Compare&
compare()
{
return beast::detail::empty_base_optimization<Compare>::member();
return *this;
}
Compare const&
compare() const
{
return beast::detail::empty_base_optimization<Compare>::member();
return *this;
}
};

View File

@@ -148,115 +148,84 @@ private:
};
// VFALCO TODO hoist to remove template argument dependencies
class ValueHash : private beast::detail::empty_base_optimization<Hash>
#ifdef _LIBCPP_VERSION
,
public std::unary_function<element, std::size_t>
#endif
class ValueHash : public Hash
{
public:
#ifndef _LIBCPP_VERSION
using argument_type = element;
using result_type = size_t;
#endif
ValueHash()
{
}
ValueHash(Hash const& h)
: beast::detail::empty_base_optimization<Hash>(h)
ValueHash(Hash const& h) : Hash(h)
{
}
std::size_t
operator()(element const& e) const
{
return this->member()(extract(e.value));
return Hash::operator()(extract(e.value));
}
Hash&
hash_function()
{
return this->member();
return *this;
}
Hash const&
hash_function() const
{
return this->member();
return *this;
}
};
// Compares value_type against element, used in find/insert_check
// VFALCO TODO hoist to remove template argument dependencies
class KeyValueEqual
: private beast::detail::empty_base_optimization<KeyEqual>
#ifdef _LIBCPP_VERSION
,
public std::binary_function<Key, element, bool>
#endif
class KeyValueEqual : public KeyEqual
{
public:
#ifndef _LIBCPP_VERSION
using first_argument_type = Key;
using second_argument_type = element;
using result_type = bool;
#endif
KeyValueEqual()
{
}
KeyValueEqual(KeyEqual const& keyEqual)
: beast::detail::empty_base_optimization<KeyEqual>(keyEqual)
KeyValueEqual(KeyEqual const& keyEqual) : KeyEqual(keyEqual)
{
}
// VFALCO NOTE WE might want only to enable these overloads
// if KeyEqual has is_transparent
#if 0
template <class K>
bool operator() (K const& k, element const& e) const
{
return this->member() (k, extract (e.value));
}
template <class K>
bool operator() (element const& e, K const& k) const
{
return this->member() (extract (e.value), k);
}
#endif
bool
operator()(Key const& k, element const& e) const
{
return this->member()(k, extract(e.value));
return KeyEqual::operator()(k, extract(e.value));
}
bool
operator()(element const& e, Key const& k) const
{
return this->member()(extract(e.value), k);
return KeyEqual::operator()(extract(e.value), k);
}
bool
operator()(element const& lhs, element const& rhs) const
{
return this->member()(extract(lhs.value), extract(rhs.value));
return KeyEqual::operator()(extract(lhs.value), extract(rhs.value));
}
KeyEqual&
key_eq()
{
return this->member();
return *this;
}
KeyEqual const&
key_eq() const
{
return this->member();
return *this;
}
};

View File

@@ -169,6 +169,8 @@ public:
std::string START_LEDGER;
uint32_t RELATIONAL_DB = 1; // 0 == SQLite, 1 == LMDB
// Network parameters
uint32_t NETWORK_ID = 0;
@@ -240,7 +242,7 @@ public:
bool LEDGER_REPLAY = false;
// Work queue limits
int MAX_TRANSACTIONS = 1000;
int MAX_TRANSACTIONS = 250;
static constexpr int MAX_JOB_QUEUE_TX = 1000;
static constexpr int MIN_JOB_QUEUE_TX = 100;

View File

@@ -24,6 +24,7 @@
#include <ripple/core/Config.h>
#include <ripple/core/SociDB.h>
#include <boost/filesystem/path.hpp>
#include <lmdb.h>
#include <mutex>
#include <optional>
#include <string>
@@ -34,6 +35,51 @@ class session;
namespace ripple {
class LockedLmdbSession
{
public:
using mutex = std::recursive_mutex;
private:
std::shared_ptr<MDB_env> env_;
std::unique_lock<mutex> lmdblock_;
public:
LockedLmdbSession(std::shared_ptr<MDB_env> env, mutex& m)
: env_(std::move(env)), lmdblock_(m)
{
}
LockedLmdbSession(LockedLmdbSession&& rhs) noexcept
: env_(std::move(rhs.env_)), lmdblock_(std::move(rhs.lmdblock_))
{
}
LockedLmdbSession() = delete;
LockedLmdbSession(LockedLmdbSession const& rhs) = delete;
LockedLmdbSession&
operator=(LockedLmdbSession const& rhs) = delete;
MDB_env*
get()
{
return env_.get();
}
MDB_env&
operator*()
{
return *env_;
}
MDB_env*
operator->()
{
return env_.get();
}
explicit
operator bool() const
{
return bool(env_);
}
};
class LockedSociSession
{
public:
@@ -88,6 +134,7 @@ public:
Config::StartUpType startUp = Config::NORMAL;
bool standAlone = false;
bool reporting = false;
bool sqlite3 = true;
boost::filesystem::path dataDir;
// Indicates whether or not to return the `globalPragma`
// from commonPragma()
@@ -166,6 +213,20 @@ public:
setupCheckpointing(checkpointerSetup.jobQueue, *checkpointerSetup.logs);
}
DatabaseCon(
Setup const& setup,
std::string const& dbName,
CheckpointerSetup const& checkpointerSetup)
: DatabaseCon(setup.dataDir / dbName)
{
setupCheckpointing(checkpointerSetup.jobQueue, *checkpointerSetup.logs);
}
DatabaseCon(Setup const& setup, std::string const& dbName)
: DatabaseCon(setup.dataDir / dbName)
{
}
~DatabaseCon();
soci::session&
@@ -180,6 +241,18 @@ public:
return LockedSociSession(session_, lock_);
}
MDB_env&
getLMDB()
{
return *env_;
}
LockedLmdbSession
checkoutLMDB()
{
return LockedLmdbSession(env_, lmdblock_);
}
private:
void
setupCheckpointing(JobQueue*, Logs&);
@@ -214,7 +287,23 @@ private:
}
}
DatabaseCon(boost::filesystem::path const& pPath)
: env_([&pPath]() {
boost::system::error_code ec;
boost::filesystem::create_directories(pPath, ec);
MDB_env *env;
mdb_env_create(&env);
mdb_env_set_mapsize(env, 10737418240);
mdb_env_set_maxdbs(env, 10);
mdb_env_open(env, pPath.c_str(), 0, 0664);
return std::shared_ptr<MDB_env>(
env, [](MDB_env* env) { mdb_env_close(env); });
}())
{
}
LockedSociSession::mutex lock_;
LockedLmdbSession::mutex lmdblock_;
// checkpointer may outlive the DatabaseCon when the checkpointer jobQueue
// callback locks a weak pointer and the DatabaseCon is then destroyed. In
@@ -223,6 +312,7 @@ private:
// the checkpointer can keep a weak_ptr) and the checkpointer is a
// shared_ptr in this class. session_ will never be null.
std::shared_ptr<soci::session> const session_;
std::shared_ptr<MDB_env> const env_;
std::shared_ptr<Checkpointer> checkpointer_;
};

View File

@@ -37,6 +37,7 @@
#include <ripple/core/JobQueue.h>
#define SOCI_USE_BOOST
#include <cstdint>
#include <lmdb.h>
#include <soci/soci.h>
#include <string>
#include <vector>
@@ -97,9 +98,13 @@ open(
std::string const& beName,
std::string const& connectionString);
std::uint32_t
getKBUsedAll(MDB_env* env);
std::uint32_t
getKBUsedAll(soci::session& s);
std::uint32_t
getKBUsedDB(MDB_env* env);
std::uint32_t
getKBUsedDB(soci::session& s);
void

View File

@@ -498,6 +498,19 @@ Config::loadFromString(std::string const& fileContents)
boost::filesystem::path p(dbPath);
legacy("database_path", boost::filesystem::absolute(p).string());
}
const Section& rdb_section{section(SECTION_RELATIONAL_DB)};
if (!rdb_section.empty())
{
if (boost::iequals(get(rdb_section, "backend"), "sqlite"))
{
RELATIONAL_DB = 0;
}
else if (boost::iequals(get(rdb_section, "backend"), "lmdb"))
{
RELATIONAL_DB = 1;
}
}
}
std::string strTemp;

View File

@@ -110,12 +110,26 @@ setup_DatabaseCon(Config const& c, std::optional<beast::Journal> j)
setup.startUp = c.START_UP;
setup.standAlone = c.standalone();
setup.reporting = c.reporting();
setup.sqlite3 = false;
setup.dataDir = c.legacy("database_path");
if (!setup.standAlone && setup.dataDir.empty())
{
Throw<std::runtime_error>("database_path must be set.");
}
const Section& rdb_section{c.section(SECTION_RELATIONAL_DB)};
if (!rdb_section.empty())
{
if (boost::iequals(get(rdb_section, "backend"), "sqlite"))
{
setup.sqlite3 = true;
}
else if (boost::iequals(get(rdb_section, "backend"), "lmdb"))
{
setup.sqlite3 = false;
}
}
if (!setup.globalPragma)
{
setup.globalPragma = [&c, &j]() {
@@ -236,6 +250,7 @@ setup_DatabaseCon(Config const& c, std::optional<beast::Journal> j)
return result;
}();
}
setup.useGlobalPragma = true;
return setup;

View File

@@ -126,6 +126,15 @@ getConnection(soci::session& s)
return result;
}
std::uint32_t
getKBUsedAll(MDB_env* env)
{
// if (!getConnection(s))
// Throw<std::logic_error>("No connection found.");
return static_cast<size_t>(
sqlite_api::sqlite3_memory_used() / kilobytes(1));
}
std::uint32_t
getKBUsedAll(soci::session& s)
{
@@ -135,6 +144,21 @@ getKBUsedAll(soci::session& s)
sqlite_api::sqlite3_memory_used() / kilobytes(1));
}
std::uint32_t
getKBUsedDB(MDB_env* env)
{
// This function will have to be customized when other backends are added
// if (auto conn = getConnection(s))
// {
// int cur = 0, hiw = 0;
// sqlite_api::sqlite3_db_status(
// conn, SQLITE_DBSTATUS_CACHE_USED, &cur, &hiw, 0);
// return cur / kilobytes(1);
// }
// Throw<std::logic_error>("");
return 0; // Silence compiler warning.
}
std::uint32_t
getKBUsedDB(soci::session& s)
{

View File

@@ -18,7 +18,6 @@
//==============================================================================
#include <ripple/app/rdb/Download.h>
#include <fstream>
namespace ripple {

View File

@@ -1894,9 +1894,7 @@ fromNetwork(
constexpr auto RPC_REPLY_MAX_BYTES = megabytes(256);
using namespace std::chrono_literals;
// auto constexpr RPC_NOTIFY = 10min; // Wietse: lolwut 10 minutes for one
// HTTP call?
auto constexpr RPC_NOTIFY = 30s;
auto constexpr RPC_NOTIFY = 10min;
HTTPClient::request(
bSSL,

View File

@@ -78,14 +78,12 @@ public:
{
std::lock_guard sl(mLock);
// Wietse: we're not going to limit this, this is admin-port only, scale
// accordingly Dropping events just like this results in inconsistent
// data on the receiving end if (mDeque.size() >= eventQueueMax)
// {
// // Drop the previous event.
// JLOG(j_.warn()) << "RPCCall::fromNetwork drop";
// mDeque.pop_back();
// }
if (mDeque.size() >= eventQueueMax)
{
// Drop the previous event.
JLOG(j_.warn()) << "RPCCall::fromNetwork drop";
mDeque.pop_back();
}
auto jm = broadcast ? j_.debug() : j_.info();
JLOG(jm) << "RPCCall::fromNetwork push: " << jvObj;
@@ -184,8 +182,7 @@ private:
}
private:
// Wietse: we're not going to limit this, this is admin-port only, scale
// accordingly enum { eventQueueMax = 32 };
enum { eventQueueMax = 32 };
boost::asio::io_service& m_io_service;
JobQueue& m_jobQueue;

View File

@@ -44,7 +44,7 @@ getFeatureValue(
return {};
boost::smatch match;
boost::regex rx(feature + "=([^;\\s]+)");
auto const value = std::string(header->value());
auto const value = header->value().to_string();
if (boost::regex_search(value, match, rx))
return {match[1]};
return {};
@@ -233,7 +233,7 @@ verifyHandshake(
{
if (auto const iter = headers.find("Server-Domain"); iter != headers.end())
{
if (!isProperlyFormedTomlDomain(std::string(iter->value())))
if (!isProperlyFormedTomlDomain(iter->value().to_string()))
throw std::runtime_error("Invalid server domain");
}
@@ -243,8 +243,7 @@ verifyHandshake(
uint32_t peer_nid = 0;
if (auto const iter = headers.find("Network-ID"); iter != headers.end())
{
if (!beast::lexicalCastChecked(
peer_nid, std::string(iter->value())))
if (!beast::lexicalCastChecked(peer_nid, iter->value().to_string()))
throw std::runtime_error("Invalid peer network identifier");
}
@@ -256,7 +255,7 @@ verifyHandshake(
if (auto const iter = headers.find("Network-Time"); iter != headers.end())
{
auto const netTime =
[str = std::string(iter->value())]() -> TimeKeeper::time_point {
[str = iter->value().to_string()]() -> TimeKeeper::time_point {
TimeKeeper::duration::rep val;
if (beast::lexicalCastChecked(val, str))
@@ -292,7 +291,7 @@ verifyHandshake(
if (auto const iter = headers.find("Public-Key"); iter != headers.end())
{
auto pk = parseBase58<PublicKey>(
TokenType::NodePublic, std::string(iter->value()));
TokenType::NodePublic, iter->value().to_string());
if (pk)
{
@@ -318,7 +317,7 @@ verifyHandshake(
if (iter == headers.end())
throw std::runtime_error("No session signature specified");
auto sig = base64_decode(std::string(iter->value()));
auto sig = base64_decode(iter->value().to_string());
if (!verifyDigest(publicKey, sharedValue, makeSlice(sig), false))
throw std::runtime_error("Failed to verify session");
@@ -331,7 +330,7 @@ verifyHandshake(
{
boost::system::error_code ec;
auto const local_ip = boost::asio::ip::address::from_string(
std::string(iter->value()), ec);
iter->value().to_string(), ec);
if (ec)
throw std::runtime_error("Invalid Local-IP");
@@ -346,7 +345,7 @@ verifyHandshake(
{
boost::system::error_code ec;
auto const remote_ip = boost::asio::ip::address::from_string(
std::string(iter->value()), ec);
iter->value().to_string(), ec);
if (ec)
throw std::runtime_error("Invalid Remote-IP");

View File

@@ -633,8 +633,8 @@ OverlayImpl::onManifests(
if (app_.validators().listed(mo->masterKey))
{
auto db = app_.getWalletDB().checkoutDb();
addValidatorManifest(*db, serialized);
auto db = app_.getWalletDB().checkoutLMDB();
addValidatorManifest(db.get(), serialized);
}
}
}

View File

@@ -176,7 +176,7 @@ PeerImp::run()
if (auto const iter = headers_.find("Closed-Ledger");
iter != headers_.end())
{
closed = parseLedgerHash(std::string(iter->value()));
closed = parseLedgerHash(iter->value().to_string());
if (!closed)
fail("Malformed handshake data (1)");
@@ -185,7 +185,7 @@ PeerImp::run()
if (auto const iter = headers_.find("Previous-Ledger");
iter != headers_.end())
{
previous = parseLedgerHash(std::string(iter->value()));
previous = parseLedgerHash(iter->value().to_string());
if (!previous)
fail("Malformed handshake data (2)");
@@ -372,8 +372,8 @@ std::string
PeerImp::getVersion() const
{
if (inbound_)
return std::string(headers_["User-Agent"]);
return std::string(headers_["Server"]);
return headers_["User-Agent"].to_string();
return headers_["Server"].to_string();
}
Json::Value
@@ -399,7 +399,7 @@ PeerImp::json()
if (auto const d = domain(); !d.empty())
ret[jss::server_domain] = domain();
if (auto const nid = std::string(headers_["Network-ID"]); !nid.empty())
if (auto const nid = headers_["Network-ID"].to_string(); !nid.empty())
ret[jss::network_id] = nid;
ret[jss::load] = usage_.balance();
@@ -839,7 +839,7 @@ PeerImp::name() const
std::string
PeerImp::domain() const
{
return std::string(headers_["Server-Domain"]);
return headers_["Server-Domain"].to_string();
}
//------------------------------------------------------------------------------

View File

@@ -73,8 +73,8 @@ PeerReservationTable::load(DatabaseCon& connection)
std::lock_guard lock(mutex_);
connection_ = &connection;
auto db = connection.checkoutDb();
auto table = getPeerReservationTable(*db, journal_);
auto db = connection.checkoutLMDB();
auto table = getPeerReservationTable(db.get(), journal_);
table_.insert(table.begin(), table.end());
return true;
@@ -108,8 +108,9 @@ PeerReservationTable::insert_or_assign(PeerReservation const& reservation)
}
table_.insert(hint, reservation);
auto db = connection_->checkoutDb();
insertPeerReservation(*db, reservation.nodeId, reservation.description);
auto db = connection_->checkoutLMDB();
insertPeerReservation(
db.get(), reservation.nodeId, reservation.description);
return previous;
}
@@ -126,8 +127,8 @@ PeerReservationTable::erase(PublicKey const& nodeId)
{
previous = *it;
table_.erase(it);
auto db = connection_->checkoutDb();
deletePeerReservation(*db, nodeId);
auto db = connection_->checkoutLMDB();
deletePeerReservation(db.get(), nodeId);
}
return previous;

View File

@@ -91,17 +91,10 @@ private:
using value_type = map_type::value_type;
struct Transform
#ifdef _LIBCPP_VERSION
: std::unary_function<
map_type::right_map::const_iterator::value_type const&,
beast::IP::Endpoint const&>
#endif
{
#ifndef _LIBCPP_VERSION
using first_argument_type =
map_type::right_map::const_iterator::value_type const&;
using result_type = beast::IP::Endpoint const&;
#endif
explicit Transform() = default;

View File

@@ -21,6 +21,7 @@
#define RIPPLE_PEERFINDER_CHECKER_H_INCLUDED
#include <ripple/beast/net/IPAddressConversion.h>
#include <boost/asio/detail/handler_invoke_helpers.hpp>
#include <boost/asio/io_service.hpp>
#include <boost/asio/ip/tcp.hpp>
#include <boost/intrusive/list.hpp>

View File

@@ -69,14 +69,9 @@ public:
public:
// Iterator transformation to extract the endpoint from Element
struct Transform
#ifdef _LIBCPP_VERSION
: public std::unary_function<Element, Endpoint>
#endif
{
#ifndef _LIBCPP_VERSION
using first_argument = Element;
using result_type = Endpoint;
#endif
explicit Transform() = default;
@@ -239,15 +234,9 @@ public:
template <bool IsConst>
struct Transform
#ifdef _LIBCPP_VERSION
: public std::
unary_function<typename lists_type::value_type, Hop<IsConst>>
#endif
{
#ifndef _LIBCPP_VERSION
using first_argument = typename lists_type::value_type;
using result_type = Hop<IsConst>;
#endif
explicit Transform() = default;

View File

@@ -226,7 +226,6 @@ doAccountTxHelp(RPC::Context& context, AccountTxArgs const& args)
auto const db =
dynamic_cast<SQLiteDatabase*>(&context.app.getRelationalDatabase());
if (!db)
Throw<std::runtime_error>("Failed to get relational database");

View File

@@ -22,6 +22,7 @@
#include <ripple/app/ledger/LedgerMaster.h>
#include <ripple/app/main/Application.h>
#include <ripple/app/misc/NetworkOPs.h>
#include <ripple/app/rdb/backend/LMDBDatabase.h>
#include <ripple/app/rdb/backend/SQLiteDatabase.h>
#include <ripple/basics/UptimeClock.h>
#include <ripple/json/json_value.h>

View File

@@ -247,11 +247,11 @@ build_map(boost::beast::http::fields const& h)
std::map<std::string, std::string> c;
for (auto const& e : h)
{
auto key(std::string(e.name_string()));
auto key(e.name_string().to_string());
std::transform(key.begin(), key.end(), key.begin(), [](auto kc) {
return std::tolower(static_cast<unsigned char>(kc));
});
c[key] = std::string(e.value());
c[key] = e.value().to_string();
}
return c;
}

View File

@@ -50,7 +50,7 @@ public:
{
auto it = h.find("X-User");
if (it != h.end())
user_ = std::string(it->value());
user_ = it->value().to_string();
fwdfor_ = std::string(forwardedFor(h));
}
}

View File

@@ -398,7 +398,7 @@ SHAMapInnerNode::canonicalizeChild(
void
SHAMapInnerNode::invariants(bool is_root) const
{
unsigned count = 0;
[[maybe_unused]] unsigned count = 0;
auto [numAllocated, hashes, children] =
hashesAndChildren_.getHashesAndChildren();

View File

@@ -154,7 +154,7 @@ class LedgerLoad_test : public beast::unit_test::suite
copy_file(
sd.ledgerFile,
ledgerFileCorrupt,
copy_options::overwrite_existing,
copy_option::overwrite_if_exists,
ec);
if (!BEAST_EXPECTS(!ec, ec.message()))
return;

View File

@@ -29,7 +29,6 @@
#include <boost/asio.hpp>
#include <boost/asio/ip/tcp.hpp>
#include <boost/asio/ssl/stream.hpp>
#include <boost/beast/core/flat_buffer.hpp>
#include <boost/beast/http.hpp>
#include <boost/beast/ssl.hpp>
#include <boost/beast/version.hpp>
@@ -575,7 +574,7 @@ private:
if (ec)
break;
auto path = req.target(); //.to_string();
auto path = req.target().to_string();
res.insert("Server", "TrustedPublisherServer");
res.version(req.version());
res.keep_alive(req.keep_alive());
@@ -678,8 +677,7 @@ private:
// unknown request
res.result(boost::beast::http::status::not_found);
res.insert("Content-Type", "text/html");
res.body() =
"The file '" + std::string(path) + "' was not found";
res.body() = "The file '" + path + "' was not found";
}
if (prepare)

View File

@@ -0,0 +1,158 @@
//------------------------------------------------------------------------------
/*
This file is part of rippled: https://github.com/ripple/rippled
Copyright (c) 2012, 2013 Ripple Labs Inc.
Permission to use, copy, modify, and/or distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#include <ripple/app/misc/Manifest.h>
#include <ripple/app/misc/ValidatorList.h>
#include <ripple/app/rdb/Wallet.h>
#include <ripple/app/rdb/backend/LMDBDatabase.h>
#include <ripple/core/DatabaseCon.h>
#include <ripple/protocol/SecretKey.h>
#include <test/jtx.h>
#include <test/jtx/envconfig.h>
#include <test/nodestore/TestBase.h>
#include <test/unit_test/SuiteJournal.h>
namespace ripple {
namespace test {
struct LMDB_test : public beast::unit_test::suite
{
void
testNodeIdentity(FeatureBitset features)
{
testcase("getNodeIdentity");
using namespace jtx;
Env env = [&]() {
auto p = test::jtx::envconfig();
{
auto& section = p->section("relational_db");
section.set("backend", "lmdb");
}
p->RELATIONAL_DB = 1;
return Env(*this, std::move(p), features);
}();
auto const s = setup_DatabaseCon(env.app().config());
auto db = env.app().getWalletDB().checkoutLMDB();
std::pair<PublicKey, SecretKey> pair = getNodeIdentity(db.get());
BEAST_EXPECT(pair.first.size() == 33);
BEAST_EXPECT(pair.second.size() == 32);
clearNodeIdentity(db.get());
std::pair<PublicKey, SecretKey> pair2 = getNodeIdentity(db.get());
BEAST_EXPECT(pair.first != pair2.first);
BEAST_EXPECT(pair.second != pair2.second);
}
void
testGetHashByIndex(FeatureBitset features)
{
testcase("getHashByIndex");
using namespace jtx;
Env env = [&]() {
auto p = test::jtx::envconfig();
{
auto& section = p->section("relational_db");
section.set("backend", "lmdb");
}
p->RELATIONAL_DB = 1;
return Env(*this, std::move(p), features);
}();
auto const s = setup_DatabaseCon(env.app().config());
auto const db = dynamic_cast<LMDBDatabase*>(&env.app().getRelationalDatabase());
env.close();
auto const resHash = db->getHashByIndex(env.current()->info().seq - 1);
BEAST_EXPECT(resHash == env.current()->info().hash);
}
void
testGetLedgerInfoByHash(FeatureBitset features)
{
testcase("getLedgerInfoByHash");
using namespace jtx;
Env env = [&]() {
auto p = test::jtx::envconfig();
{
auto& section = p->section("relational_db");
section.set("backend", "lmdb");
}
p->RELATIONAL_DB = 1;
return Env(*this, std::move(p), features);
}();
auto const s = setup_DatabaseCon(env.app().config());
auto const db = dynamic_cast<LMDBDatabase*>(&env.app().getRelationalDatabase());
env.close();
auto const info = db->getLedgerInfoByHash(env.current()->info().hash);
BEAST_EXPECT(info->hash == env.current()->info().hash);
BEAST_EXPECT(info->seq == env.current()->info().seq - 1);
}
void
testGetLedgerInfoByIndex(FeatureBitset features)
{
testcase("getLedgerInfoByIndex");
using namespace jtx;
Env env = [&]() {
auto p = test::jtx::envconfig();
{
auto& section = p->section("relational_db");
section.set("backend", "lmdb");
}
p->RELATIONAL_DB = 1;
return Env(*this, std::move(p), features);
}();
auto const s = setup_DatabaseCon(env.app().config());
auto const db = dynamic_cast<LMDBDatabase*>(&env.app().getRelationalDatabase());
env.close();
auto const info = db->getLedgerInfoByIndex(env.current()->info().seq - 1);
BEAST_EXPECT(info->hash == env.current()->info().hash);
BEAST_EXPECT(info->seq == env.current()->info().seq - 1);
}
void
testWithFeats(FeatureBitset features)
{
// testNodeIdentity(features);
testGetHashByIndex(features);
testGetLedgerInfoByHash(features);
testGetLedgerInfoByIndex(features);
}
public:
void
run() override
{
using namespace test::jtx;
auto const sa = supported_amendments();
testWithFeats(sa);
}
};
BEAST_DEFINE_TESTSUITE(LMDB, NodeStore, ripple);
} // namespace test
} // namespace ripple

View File

@@ -674,10 +674,10 @@ class ServerStatus_test : public beast::unit_test::suite,
resp.result() == boost::beast::http::status::switching_protocols);
BEAST_EXPECT(
resp.find("Upgrade") != resp.end() &&
std::string(resp["Upgrade"]) == "websocket");
resp["Upgrade"] == "websocket");
BEAST_EXPECT(
resp.find("Connection") != resp.end() &&
std::string(resp["Connection"]) == "Upgrade");
resp["Connection"] == "upgrade");
}
void