20 #include <ripple/app/ledger/AcceptedLedger.h>
21 #include <ripple/app/ledger/LedgerMaster.h>
22 #include <ripple/app/ledger/LedgerToJson.h>
23 #include <ripple/app/ledger/TransactionMaster.h>
24 #include <ripple/app/misc/Manifest.h>
25 #include <ripple/app/rdb/RelationalDBInterface_global.h>
26 #include <ripple/basics/BasicConfig.h>
27 #include <ripple/basics/StringUtilities.h>
28 #include <ripple/core/DatabaseCon.h>
29 #include <ripple/core/SociDB.h>
30 #include <ripple/json/to_string.h>
31 #include <boost/algorithm/string.hpp>
32 #include <boost/range/adaptor/transformed.hpp>
33 #include <soci/sqlite3/soci-sqlite3.h>
43 return std::make_unique<DatabaseCon>(
51 return std::make_unique<DatabaseCon>(
57 soci::session& session,
63 std::string const sql =
"SELECT RawData FROM " + dbTable +
";";
64 soci::blob sociRawData(session);
65 soci::statement st = (session.prepare << sql, soci::into(sociRawData));
70 convert(sociRawData, serialized);
75 JLOG(j.
warn()) <<
"Unverifiable manifest in db";
83 JLOG(j.
warn()) <<
"Malformed manifest in database";
90 soci::session& session,
97 soci::blob rawData(session);
99 session <<
"INSERT INTO " << dbTable <<
" (RawData) VALUES (:rawData);",
105 soci::session& session,
111 soci::transaction tr(session);
112 session <<
"DELETE FROM " << dbTable;
113 for (
auto const& v : map)
117 if (!v.second.revoked() && !isTrusted(v.second.masterKey))
119 JLOG(j.
info()) <<
"Untrusted manifest in cache not saved to db";
131 soci::transaction tr(session);
132 saveManifest(session,
"ValidatorManifests", serialized);
141 boost::optional<std::string> pubKO, priKO;
144 <<
"SELECT PublicKey, PrivateKey FROM NodeIdentity;",
150 auto const sk = parseBase58<SecretKey>(
152 auto const pk = parseBase58<PublicKey>(
165 boost::format(
"INSERT INTO NodeIdentity (PublicKey,PrivateKey) "
166 "VALUES ('%s','%s');") %
170 return {newpublicKey, newsecretKey};
179 boost::optional<std::string> valPubKey, valDesc;
185 <<
"SELECT PublicKey, Description FROM PeerReservations;",
186 soci::into(valPubKey),
187 soci::into(valDesc));
191 if (!valPubKey || !valDesc)
197 auto const optNodeId =
201 JLOG(j.
warn()) <<
"load: not a public key: " << valPubKey;
212 soci::session& session,
216 session <<
"INSERT INTO PeerReservations (PublicKey, Description) "
217 "VALUES (:nodeId, :desc) "
218 "ON CONFLICT (PublicKey) DO UPDATE SET "
219 "Description=excluded.Description",
221 soci::use(description);
227 session <<
"DELETE FROM PeerReservations WHERE PublicKey = :nodeId",
234 soci::transaction tr(session);
236 "SELECT count(*) FROM sqlite_master "
237 "WHERE type='table' AND name='FeatureVotes'";
239 boost::optional<int> featureVotesCount;
240 session << sql, soci::into(featureVotesCount);
241 bool exists =
static_cast<bool>(*featureVotesCount);
246 session <<
"CREATE TABLE FeatureVotes ( "
247 "AmendmentHash CHARACTER(64) NOT NULL, "
248 "AmendmentName TEXT, "
249 "Veto INTEGER NOT NULL );";
257 soci::session& session,
259 boost::optional<std::string> amendment_hash,
260 boost::optional<std::string> amendment_name,
261 boost::optional<AmendmentVote> vote)>
const& callback)
264 auto intToVote = [](boost::optional<int>
const& dbVote)
265 -> boost::optional<AmendmentVote> {
266 return safe_cast<AmendmentVote>(dbVote.value_or(1));
269 soci::transaction tr(session);
271 "SELECT AmendmentHash, AmendmentName, Veto FROM FeatureVotes";
273 boost::optional<std::string> amendment_hash;
274 boost::optional<std::string> amendment_name;
275 boost::optional<int> vote_to_veto;
277 (session.prepare << sql,
278 soci::into(amendment_hash),
279 soci::into(amendment_name),
280 soci::into(vote_to_veto));
284 callback(amendment_hash, amendment_name, intToVote(vote_to_veto));
290 soci::session& session,
295 soci::transaction tr(session);
297 "INSERT INTO FeatureVotes (AmendmentHash, AmendmentName, Veto) VALUES "
300 sql +=
"', '" + name;
310 soci::session& session,
314 open(session, config, dbName);
316 session <<
"PRAGMA synchronous=FULL;";
318 session <<
"CREATE TABLE IF NOT EXISTS DbState ("
319 " Key INTEGER PRIMARY KEY,"
322 " LastRotatedLedger INTEGER"
325 session <<
"CREATE TABLE IF NOT EXISTS CanDelete ("
326 " Key INTEGER PRIMARY KEY,"
327 " CanDeleteSeq INTEGER"
333 boost::optional<std::int64_t> countO;
334 session <<
"SELECT COUNT(Key) FROM DbState WHERE Key = 1;",
337 Throw<std::runtime_error>(
338 "Failed to fetch Key Count from DbState.");
344 session <<
"INSERT INTO DbState VALUES (1, '', '', 0);";
349 boost::optional<std::int64_t> countO;
350 session <<
"SELECT COUNT(Key) FROM CanDelete WHERE Key = 1;",
353 Throw<std::runtime_error>(
354 "Failed to fetch Key Count from CanDelete.");
360 session <<
"INSERT INTO CanDelete VALUES (1, 0);";
368 session <<
"SELECT CanDeleteSeq FROM CanDelete WHERE Key = 1;",
377 session <<
"UPDATE CanDelete SET CanDeleteSeq = :canDelete WHERE Key = 1;",
378 soci::use(canDelete);
386 session <<
"SELECT WritableDb, ArchiveDb, LastRotatedLedger"
387 " FROM DbState WHERE Key = 1;",
397 session <<
"UPDATE DbState"
398 " SET WritableDb = :writableDb,"
399 " ArchiveDb = :archiveDb,"
400 " LastRotatedLedger = :lastRotated"
409 session <<
"UPDATE DbState SET LastRotatedLedger = :seq"
419 boost::filesystem::path
const& path)
422 boost::optional<std::string> pathFromDb;
423 boost::optional<std::uint64_t> size;
425 auto conn = std::make_unique<DatabaseCon>(
428 auto& session = *conn->checkoutDb();
430 session <<
"SELECT Path FROM Download WHERE Part=0;",
431 soci::into(pathFromDb);
439 if (pathFromDb != path.string())
441 session <<
"DROP TABLE Download;";
447 session <<
"SELECT SUM(LENGTH(Data)) FROM Download;",
457 soci::session& session,
470 dynamic_cast<soci::sqlite3_session_backend*
>(session.get_backend());
476 auto const blobMaxSize =
477 sqlite_api::sqlite3_limit(be->conn_, SQLITE_LIMIT_LENGTH, -1) -
483 session <<
"INSERT INTO Download VALUES (:path, zeroblob(0), 0, :part)",
484 soci::use(newpath), soci::use(part);
486 remainingInRow = blobMaxSize;
490 session <<
"SELECT Path,Size,Part FROM Download ORDER BY Part DESC "
492 soci::into(newpath), soci::into(rowSize), soci::into(part, rti);
494 if (!session.got_data())
500 remainingInRow = blobMaxSize - rowSize;
502 auto insert = [&session, &rowSize, &part, &fs = fileSize](
506 session <<
"UPDATE Download SET Data = CAST(Data || :data AS blob), "
507 "Size = :size WHERE Part = :part;",
508 soci::use(data), soci::use(updatedSize), soci::use(part);
513 size_t currentBase = 0;
515 while (currentBase + remainingInRow < data.size())
519 insert(data.substr(currentBase, remainingInRow));
520 currentBase += remainingInRow;
527 insert(data.substr(currentBase));
535 soci::rowset<std::string> rs =
536 (session.prepare <<
"SELECT Data FROM Download ORDER BY PART ASC;");
539 for (
auto it = rs.begin(); it != rs.end(); ++it)
540 fout.
write(it->data(), it->size());
550 uintmax_t
const dbSize = file_size(dbPath);
551 assert(dbSize !=
static_cast<uintmax_t
>(-1));
553 if (
auto available = space(dbPath.parent_path()).available;
556 std::cerr <<
"The database filesystem must have at least as "
557 "much free space as the size of "
558 << dbPath.string() <<
", which is " << dbSize
559 <<
" bytes. Only " <<
available <<
" bytes are available.\n";
565 auto& session = txnDB->getSession();
572 session <<
"PRAGMA page_size;", soci::into(pageSize);
576 session <<
"VACUUM;";
580 session <<
"PRAGMA page_size;", soci::into(pageSize);
591 soci::session& session,
595 DBConfig m_sociConfig(config,
"peerfinder");
596 m_sociConfig.
open(session);
601 soci::transaction tr(session);
602 session <<
"PRAGMA encoding=\"UTF-8\";";
604 session <<
"CREATE TABLE IF NOT EXISTS SchemaVersion ( "
605 " name TEXT PRIMARY KEY, "
609 session <<
"CREATE TABLE IF NOT EXISTS PeerFinder_BootstrapCache ( "
610 " id INTEGER PRIMARY KEY AUTOINCREMENT, "
611 " address TEXT UNIQUE NOT NULL, "
615 session <<
"CREATE INDEX IF NOT EXISTS "
616 " PeerFinder_BootstrapCache_Index ON "
617 "PeerFinder_BootstrapCache "
627 soci::session& session,
628 int currentSchemaVersion,
631 soci::transaction tr(session);
636 boost::optional<int> vO;
639 "FROM SchemaVersion WHERE "
640 " name = 'PeerFinder';",
643 version = vO.value_or(0);
645 JLOG(j.
info()) <<
"Opened version " << version <<
" database";
649 if (version < currentSchemaVersion)
651 JLOG(j.
info()) <<
"Updating database to version "
652 << currentSchemaVersion;
654 else if (version > currentSchemaVersion)
656 Throw<std::runtime_error>(
657 "The PeerFinder database version is higher than expected");
667 session <<
"CREATE TABLE IF NOT EXISTS "
668 "PeerFinder_BootstrapCache_Next ( "
669 " id INTEGER PRIMARY KEY AUTOINCREMENT, "
670 " address TEXT UNIQUE NOT NULL, "
674 session <<
"CREATE INDEX IF NOT EXISTS "
675 " PeerFinder_BootstrapCache_Next_Index ON "
676 " PeerFinder_BootstrapCache_Next "
680 session <<
"SELECT COUNT(*) FROM PeerFinder_BootstrapCache;",
690 (session.prepare <<
"SELECT "
693 "FROM PeerFinder_BootstrapCache;",
695 soci::into(valence));
702 if (!is_unspecified(entry.
endpoint))
709 JLOG(j.
error()) <<
"Bad address string '" << s
710 <<
"' in Bootcache table";
722 for (
auto iter(list.
cbegin()); iter != list.
cend(); ++iter)
728 session <<
"INSERT INTO PeerFinder_BootstrapCache_Next ( "
734 soci::use(s), soci::use(valence);
737 session <<
"DROP TABLE IF EXISTS PeerFinder_BootstrapCache;";
739 session <<
"DROP INDEX IF EXISTS PeerFinder_BootstrapCache_Index;";
741 session <<
"ALTER TABLE PeerFinder_BootstrapCache_Next "
742 " RENAME TO PeerFinder_BootstrapCache;";
744 session <<
"CREATE INDEX IF NOT EXISTS "
745 " PeerFinder_BootstrapCache_Index ON "
746 "PeerFinder_BootstrapCache "
758 session <<
"DROP TABLE IF EXISTS LegacyEndpoints;";
760 session <<
"DROP TABLE IF EXISTS PeerFinderLegacyEndpoints;";
762 session <<
"DROP TABLE IF EXISTS PeerFinder_LegacyEndpoints;";
764 session <<
"DROP TABLE IF EXISTS PeerFinder_LegacyEndpoints_Index;";
768 int const v(currentSchemaVersion);
769 session <<
"INSERT OR REPLACE INTO SchemaVersion ("
773 " 'PeerFinder', :version "
783 soci::session& session,
789 (session.prepare <<
"SELECT "
792 "FROM PeerFinder_BootstrapCache;",
794 soci::into(valence));
805 soci::session& session,
808 soci::transaction tr(session);
809 session <<
"DELETE FROM PeerFinder_BootstrapCache;";
818 for (
auto const& e : v)
824 session <<
"INSERT INTO PeerFinder_BootstrapCache ( "
830 soci::use(s), soci::use(valence);
std::pair< PublicKey, SecretKey > getNodeIdentity(Application &app)
The cryptographic credentials identifying this server instance.
LedgerIndex setCanDelete(soci::session &session, LedgerIndex canDelete)
setCanDelete Updates ledger sequence which can be deleted.
constexpr auto WalletDBName
static std::unique_ptr< std::vector< std::string > const > globalPragma
std::uint32_t LedgerIndex
A ledger index.
std::string connectionString() const
void initPeerFinderDB(soci::session &session, BasicConfig const &config, beast::Journal j)
initPeerFinderDB Opens session with peer finder database.
void convert(soci::blob &from, std::vector< std::uint8_t > &to)
void setSavedState(soci::session &session, SavedState const &state)
setSavedState Saves given state.
bool createFeatureVotes(soci::session &session)
createFeatureVotes Creates FeatureVote table if it is not exists.
void open(soci::session &s) const
void updatePeerFinderDB(soci::session &session, int currentSchemaVersion, beast::Journal j)
updatePeerFinderDB Update peer finder DB to new version.
std::string toBase58(AccountID const &v)
Convert AccountID to base58 checked string.
void addValidatorManifest(soci::session &session, std::string const &serialized)
addValidatorManifest Saves manifest of validator to database.
DBConfig is used when a client wants to delay opening a soci::session after parsing the config parame...
boost::filesystem::path dataDir
std::unique_ptr< DatabaseCon > makeTestWalletDB(DatabaseCon::Setup const &setup, std::string const &dbname)
makeTestWalletDB Opens test wallet DB with arbitrary name.
static constexpr std::array< char const *, 2 > DownloaderDBPragma
Integers of any length that is a multiple of 32-bits.
PublicKey derivePublicKey(KeyType type, SecretKey const &sk)
Derive the public key from a secret key.
void readPeerFinderDB(soci::session &session, std::function< void(std::string const &, int)> const &func)
readPeerFinderDB Read all entries from peer finder DB and call given callback for each entry.
void getManifests(soci::session &session, std::string const &dbTable, ManifestCache &mCache, beast::Journal j)
getManifests Loads manifest from wallet DB and stores it in the cache.
void insertPeerReservation(soci::session &session, PublicKey const &nodeId, std::string const &description)
insertPeerReservation Adds entry to peer reservation table.
void deletePeerReservation(soci::session &session, PublicKey const &nodeId)
deletePeerReservation Deletes entry from peer reservation table.
void readAmendments(soci::session &session, std::function< void(boost::optional< std::string > amendment_hash, boost::optional< std::string > amendment_name, boost::optional< AmendmentVote > vote)> const &callback)
readAmendments Read all amendments from FeatureVotes table.
void initStateDB(soci::session &session, BasicConfig const &config, std::string const &dbName)
initStateDB Opens DB session with State DB.
A generic endpoint for log messages.
constexpr char const * CommonDBPragmaTemp
SavedState getSavedState(soci::session &session)
getSavedState Returns saved state.
std::uint64_t databaseBodyDoPut(soci::session &session, std::string const &data, std::string const &path, std::uint64_t fileSize, std::uint64_t part, std::uint16_t maxRowSizePad)
databaseBodyDoPut Saves new fragment of downloaded file.
std::pair< PublicKey, SecretKey > randomKeyPair(KeyType type)
Create a key pair using secure random numbers.
std::unordered_set< PeerReservation, beast::uhash<>, KeyEqual > getPeerReservationTable(soci::session &session, beast::Journal j)
getPeerReservationTable Returns peer reservation table.
Remembers manifests with the highest sequence number.
void setLastRotated(soci::session &session, LedgerIndex seq)
setLastRotated Updates last rotated ledger sequence.
T emplace_back(T... args)
Use hash_* containers for keys that do not need a cryptographically secure hashing algorithm.
static Endpoint from_string(std::string const &s)
LedgerIndex getCanDelete(soci::session &session)
getCanDelete Returns ledger sequence which can be deleted.
void voteAmendment(soci::session &session, uint256 const &amendment, std::string const &name, AmendmentVote vote)
voteAmendment Set veto value for particular amendment.
std::unique_ptr< DatabaseCon > makeWalletDB(DatabaseCon::Setup const &setup)
makeWalletDB Opens wallet DB and returns it.
static void saveManifest(soci::session &session, std::string const &dbTable, std::string const &serialized)
void savePeerFinderDB(soci::session &session, std::vector< PeerFinder::Store::Entry > const &v)
savePeerFinderDB Save new entry to peer finder DB.
constexpr std::array TxDBPragma
std::pair< std::unique_ptr< DatabaseCon >, std::optional< std::uint64_t > > openDatabaseBodyDb(DatabaseCon::Setup const &setup, boost::filesystem::path const &path)
openDatabaseBodyDb Opens file download DB and returns its descriptor.
std::string to_string(Manifest const &m)
Format the specified manifest to a string for debugging purposes.
void databaseBodyFinish(soci::session &session, std::ofstream &fout)
databaseBodyFinish Finishes download process and writes file to disk.
static constexpr std::array< char const *, 3 > DatabaseBodyDBInit
bool doVacuumDB(DatabaseCon::Setup const &setup)
doVacuumDB Creates, initialises DB, and performs its cleanup.
constexpr std::array< char const *, 8 > TxDBInit
constexpr std::array< char const *, 6 > WalletDBInit
ManifestDisposition applyManifest(Manifest m)
Add manifest to cache.
std::optional< Manifest > deserializeManifest(Slice s)
Constructs Manifest from serialized string.
void saveManifests(soci::session &session, std::string const &dbTable, std::function< bool(PublicKey const &)> const &isTrusted, hash_map< PublicKey, Manifest > const &map, beast::Journal j)
saveManifests Saves all given manifests to database.
beast::IP::Endpoint endpoint
Holds unparsed configuration information.
void open(soci::session &s, BasicConfig const &config, std::string const &dbName)
Open a soci session.