Merge branch 'develop' into vlntb/mem-leak-ledger-history-3

include/xrpl/core/PeerReservationTable.h

@@ -0,0 +1,98 @@
+#pragma once
+
+#include <xrpl/beast/hash/hash_append.h>
+#include <xrpl/beast/hash/uhash.h>
+#include <xrpl/beast/utility/Journal.h>
+#include <xrpl/protocol/PublicKey.h>
+
+#include <mutex>
+#include <optional>
+#include <string>
+#include <unordered_set>
+#include <vector>
+
+namespace xrpl {
+
+class DatabaseCon;
+
+// Value type for reservations.
+struct PeerReservation final
+{
+public:
+    PublicKey nodeId;
+    std::string description{};
+
+    auto
+    toJson() const -> Json::Value;
+
+    template <typename Hasher>
+    friend void
+    hash_append(Hasher& h, PeerReservation const& x) noexcept
+    {
+        using beast::hash_append;
+        hash_append(h, x.nodeId);
+    }
+
+    friend bool
+    operator<(PeerReservation const& a, PeerReservation const& b)
+    {
+        return a.nodeId < b.nodeId;
+    }
+};
+
+// TODO: When C++20 arrives, take advantage of "equivalence" instead of
+// "equality". Add an overload for `(PublicKey, PeerReservation)`, and just
+// pass a `PublicKey` directly to `unordered_set.find`.
+struct KeyEqual final
+{
+    bool
+    operator()(PeerReservation const& lhs, PeerReservation const& rhs) const
+    {
+        return lhs.nodeId == rhs.nodeId;
+    }
+};
+
+class PeerReservationTable final
+{
+public:
+    explicit PeerReservationTable(beast::Journal journal = beast::Journal(beast::Journal::getNullSink()))
+        : journal_(journal)
+    {
+    }
+
+    std::vector<PeerReservation>
+    list() const;
+
+    bool
+    contains(PublicKey const& nodeId)
+    {
+        std::lock_guard lock(this->mutex_);
+        return table_.find({nodeId}) != table_.end();
+    }
+
+    // Because `ApplicationImp` has two-phase initialization, so must we.
+    // Our dependencies are not prepared until the second phase.
+    bool
+    load(DatabaseCon& connection);
+
+    /**
+     * @return the replaced reservation if it existed
+     * @throw soci::soci_error
+     */
+    std::optional<PeerReservation>
+    insert_or_assign(PeerReservation const& reservation);
+
+    /**
+     * @return the erased reservation if it existed
+     */
+    std::optional<PeerReservation>
+    erase(PublicKey const& nodeId);
+
+private:
+    beast::Journal mutable journal_;
+    std::mutex mutable mutex_;
+    DatabaseCon* connection_;
+    std::unordered_set<PeerReservation, beast::uhash<>, KeyEqual> table_;
+};
+
+}  // namespace xrpl

include/xrpl/core/ServiceRegistry.h

@@ -5,6 +5,8 @@
 #include <xrpl/basics/TaggedCache.h>
 #include <xrpl/ledger/CachedSLEs.h>
 
+#include <boost/asio.hpp>
+
 namespace xrpl {
 
 // Forward declarations
@@ -18,6 +20,10 @@ namespace perf {
 class PerfLog;
 }
 
+// This is temporary until we migrate all code to use ServiceRegistry.
+class Application;
+
+// Forward declarations
 class AcceptedLedger;
 class AmendmentTable;
 class Cluster;
@@ -194,6 +200,24 @@ public:
 
     virtual perf::PerfLog&
     getPerfLog() = 0;
+
+    // Configuration and state
+    virtual bool
+    isStopping() const = 0;
+
+    virtual beast::Journal
+    journal(std::string const& name) = 0;
+
+    virtual boost::asio::io_context&
+    getIOContext() = 0;
+
+    virtual Logs&
+    logs() = 0;
+
+    // Temporary: Get the underlying Application for functions that haven't
+    // been migrated yet. This should be removed once all code is migrated.
+    virtual Application&
+    app() = 0;
 };
 
 }  // namespace xrpl

include/xrpl/core/StartUpType.h

@@ -0,0 +1,16 @@
+#pragma once
+
+#include <iosfwd>
+#include <type_traits>
+
+namespace xrpl {
+
+enum class StartUpType { FRESH, NORMAL, LOAD, LOAD_FILE, REPLAY, NETWORK };
+
+inline std::ostream&
+operator<<(std::ostream& os, StartUpType const& type)
+{
+    return os << static_cast<std::underlying_type_t<StartUpType>>(type);
+}
+
+}  // namespace xrpl

include/xrpl/protocol/detail/features.macro

@@ -15,7 +15,7 @@
 
 // Add new amendments to the top of this list.
 // Keep it sorted in reverse chronological order.
-
+XRPL_FIX   (PermissionedDomainInvariant, Supported::yes, VoteBehavior::DefaultNo)
 XRPL_FIX    (ExpiredNFTokenOfferRemoval, Supported::yes, VoteBehavior::DefaultNo)
 XRPL_FIX    (BatchInnerSigs,             Supported::yes, VoteBehavior::DefaultNo)
 XRPL_FEATURE(LendingProtocol,            Supported::yes, VoteBehavior::DefaultNo)

include/xrpl/rdb/DBInit.h

@@ -0,0 +1,118 @@
+#pragma once
+
+#include <array>
+#include <cstdint>
+
+namespace xrpl {
+
+////////////////////////////////////////////////////////////////////////////////
+
+// These pragmas are built at startup and applied to all database
+// connections, unless otherwise noted.
+inline constexpr char const* CommonDBPragmaJournal{"PRAGMA journal_mode=%s;"};
+inline constexpr char const* CommonDBPragmaSync{"PRAGMA synchronous=%s;"};
+inline constexpr char const* CommonDBPragmaTemp{"PRAGMA temp_store=%s;"};
+// A warning will be logged if any lower-safety sqlite tuning settings
+// are used and at least this much ledger history is configured. This
+// includes full history nodes. This is because such a large amount of
+// data will be more difficult to recover if a rare failure occurs,
+// which are more likely with some of the other available tuning settings.
+inline constexpr std::uint32_t SQLITE_TUNING_CUTOFF = 10'000'000;
+
+// Ledger database holds ledgers and ledger confirmations
+inline constexpr auto LgrDBName{"ledger.db"};
+
+inline constexpr std::array<char const*, 5> LgrDBInit{
+    {"BEGIN TRANSACTION;",
+
+     "CREATE TABLE IF NOT EXISTS Ledgers (           \
+        LedgerHash      CHARACTER(64) PRIMARY KEY,  \
+        LedgerSeq       BIGINT UNSIGNED,            \
+        PrevHash        CHARACTER(64),              \
+        TotalCoins      BIGINT UNSIGNED,            \
+        ClosingTime     BIGINT UNSIGNED,            \
+        PrevClosingTime BIGINT UNSIGNED,            \
+        CloseTimeRes    BIGINT UNSIGNED,            \
+        CloseFlags      BIGINT UNSIGNED,            \
+        AccountSetHash  CHARACTER(64),              \
+        TransSetHash    CHARACTER(64)               \
+    );",
+     "CREATE INDEX IF NOT EXISTS SeqLedger ON Ledgers(LedgerSeq);",
+
+     // Old table and indexes no longer needed
+     "DROP TABLE IF EXISTS Validations;",
+
+     "END TRANSACTION;"}};
+
+////////////////////////////////////////////////////////////////////////////////
+
+// Transaction database holds transactions and public keys
+inline constexpr auto TxDBName{"transaction.db"};
+
+inline constexpr std::array<char const*, 8> TxDBInit{
+    {"BEGIN TRANSACTION;",
+
+     "CREATE TABLE IF NOT EXISTS Transactions (          \
+        TransID     CHARACTER(64) PRIMARY KEY,          \
+        TransType   CHARACTER(24),                      \
+        FromAcct    CHARACTER(35),                      \
+        FromSeq     BIGINT UNSIGNED,                    \
+        LedgerSeq   BIGINT UNSIGNED,                    \
+        Status      CHARACTER(1),                       \
+        RawTxn      BLOB,                               \
+        TxnMeta     BLOB                                \
+    );",
+     "CREATE INDEX IF NOT EXISTS TxLgrIndex ON           \
+        Transactions(LedgerSeq);",
+
+     "CREATE TABLE IF NOT EXISTS AccountTransactions (   \
+        TransID     CHARACTER(64),                      \
+        Account     CHARACTER(64),                      \
+        LedgerSeq   BIGINT UNSIGNED,                    \
+        TxnSeq      INTEGER                             \
+    );",
+     "CREATE INDEX IF NOT EXISTS AcctTxIDIndex ON        \
+        AccountTransactions(TransID);",
+     "CREATE INDEX IF NOT EXISTS AcctTxIndex ON          \
+        AccountTransactions(Account, LedgerSeq, TxnSeq, TransID);",
+     "CREATE INDEX IF NOT EXISTS AcctLgrIndex ON         \
+        AccountTransactions(LedgerSeq, Account, TransID);",
+
+     "END TRANSACTION;"}};
+
+////////////////////////////////////////////////////////////////////////////////
+
+inline constexpr auto WalletDBName{"wallet.db"};
+
+inline constexpr std::array<char const*, 6> WalletDBInit{
+    {"BEGIN TRANSACTION;",
+
+     // A node's identity must be persisted, including
+     // for clustering purposes. This table holds one
+     // entry: the server's unique identity, but the
+     // value can be overriden by specifying a node
+     // identity in the config file using a [node_seed]
+     // entry.
+     "CREATE TABLE IF NOT EXISTS NodeIdentity (			\
+        PublicKey       CHARACTER(53),					\
+        PrivateKey      CHARACTER(52)					\
+    );",
+
+     // Peer reservations
+     "CREATE TABLE IF NOT EXISTS PeerReservations (		\
+        PublicKey       CHARACTER(53) UNIQUE NOT NULL,	\
+        Description     CHARACTER(64) NOT NULL			\
+    );",
+
+     // Validator Manifests
+     "CREATE TABLE IF NOT EXISTS ValidatorManifests (	\
+        RawData          BLOB NOT NULL					\
+    );",
+
+     "CREATE TABLE IF NOT EXISTS PublisherManifests (	\
+        RawData          BLOB NOT NULL					\
+    );",
+
+     "END TRANSACTION;"}};
+
+}  // namespace xrpl

include/xrpl/rdb/DatabaseCon.h

@@ -0,0 +1,231 @@
+#pragma once
+
+#include <xrpl/core/PerfLog.h>
+#include <xrpl/core/StartUpType.h>
+#include <xrpl/rdb/DBInit.h>
+#include <xrpl/rdb/SociDB.h>
+
+#include <boost/filesystem/path.hpp>
+
+#include <mutex>
+#include <optional>
+#include <string>
+
+namespace soci {
+class session;
+}
+
+namespace xrpl {
+
+class LockedSociSession
+{
+public:
+    using mutex = std::recursive_mutex;
+
+private:
+    std::shared_ptr<soci::session> session_;
+    std::unique_lock<mutex> lock_;
+
+public:
+    LockedSociSession(std::shared_ptr<soci::session> it, mutex& m) : session_(std::move(it)), lock_(m)
+    {
+    }
+    LockedSociSession(LockedSociSession&& rhs) noexcept : session_(std::move(rhs.session_)), lock_(std::move(rhs.lock_))
+    {
+    }
+    LockedSociSession() = delete;
+    LockedSociSession(LockedSociSession const& rhs) = delete;
+    LockedSociSession&
+    operator=(LockedSociSession const& rhs) = delete;
+
+    soci::session*
+    get()
+    {
+        return session_.get();
+    }
+    soci::session&
+    operator*()
+    {
+        return *session_;
+    }
+    soci::session*
+    operator->()
+    {
+        return session_.get();
+    }
+    explicit
+    operator bool() const
+    {
+        return bool(session_);
+    }
+};
+
+class DatabaseCon
+{
+public:
+    struct Setup
+    {
+        explicit Setup() = default;
+
+        StartUpType startUp = StartUpType::NORMAL;
+        bool standAlone = false;
+        boost::filesystem::path dataDir;
+        // Indicates whether or not to return the `globalPragma`
+        // from commonPragma()
+        bool useGlobalPragma = false;
+
+        std::vector<std::string> const*
+        commonPragma() const
+        {
+            XRPL_ASSERT(
+                !useGlobalPragma || globalPragma,
+                "xrpl::DatabaseCon::Setup::commonPragma : consistent global "
+                "pragma");
+            return useGlobalPragma && globalPragma ? globalPragma.get() : nullptr;
+        }
+
+        static std::unique_ptr<std::vector<std::string> const> globalPragma;
+        std::array<std::string, 4> txPragma;
+        std::array<std::string, 1> lgrPragma;
+    };
+
+    struct CheckpointerSetup
+    {
+        JobQueue* jobQueue;
+        Logs* logs;
+    };
+
+    template <std::size_t N, std::size_t M>
+    DatabaseCon(
+        Setup const& setup,
+        std::string const& dbName,
+        std::array<std::string, N> const& pragma,
+        std::array<char const*, M> const& initSQL,
+        beast::Journal journal)
+        // Use temporary files or regular DB files?
+        : DatabaseCon(
+              setup.standAlone && setup.startUp != StartUpType::LOAD && setup.startUp != StartUpType::LOAD_FILE &&
+                      setup.startUp != StartUpType::REPLAY
+                  ? ""
+                  : (setup.dataDir / dbName),
+              setup.commonPragma(),
+              pragma,
+              initSQL,
+              journal)
+    {
+    }
+
+    // Use this constructor to setup checkpointing
+    template <std::size_t N, std::size_t M>
+    DatabaseCon(
+        Setup const& setup,
+        std::string const& dbName,
+        std::array<std::string, N> const& pragma,
+        std::array<char const*, M> const& initSQL,
+        CheckpointerSetup const& checkpointerSetup,
+        beast::Journal journal)
+        : DatabaseCon(setup, dbName, pragma, initSQL, journal)
+    {
+        setupCheckpointing(checkpointerSetup.jobQueue, *checkpointerSetup.logs);
+    }
+
+    template <std::size_t N, std::size_t M>
+    DatabaseCon(
+        boost::filesystem::path const& dataDir,
+        std::string const& dbName,
+        std::array<std::string, N> const& pragma,
+        std::array<char const*, M> const& initSQL,
+        beast::Journal journal)
+        : DatabaseCon(dataDir / dbName, nullptr, pragma, initSQL, journal)
+    {
+    }
+
+    // Use this constructor to setup checkpointing
+    template <std::size_t N, std::size_t M>
+    DatabaseCon(
+        boost::filesystem::path const& dataDir,
+        std::string const& dbName,
+        std::array<std::string, N> const& pragma,
+        std::array<char const*, M> const& initSQL,
+        CheckpointerSetup const& checkpointerSetup,
+        beast::Journal journal)
+        : DatabaseCon(dataDir, dbName, pragma, initSQL, journal)
+    {
+        setupCheckpointing(checkpointerSetup.jobQueue, *checkpointerSetup.logs);
+    }
+
+    ~DatabaseCon();
+
+    soci::session&
+    getSession()
+    {
+        return *session_;
+    }
+
+    LockedSociSession
+    checkoutDb()
+    {
+        using namespace std::chrono_literals;
+        LockedSociSession session =
+            perf::measureDurationAndLog([&]() { return LockedSociSession(session_, lock_); }, "checkoutDb", 10ms, j_);
+
+        return session;
+    }
+
+private:
+    void
+    setupCheckpointing(JobQueue*, Logs&);
+
+    template <std::size_t N, std::size_t M>
+    DatabaseCon(
+        boost::filesystem::path const& pPath,
+        std::vector<std::string> const* commonPragma,
+        std::array<std::string, N> const& pragma,
+        std::array<char const*, M> const& initSQL,
+        beast::Journal journal)
+        : session_(std::make_shared<soci::session>()), j_(journal)
+    {
+        open(*session_, "sqlite", pPath.string());
+
+        for (auto const& p : pragma)
+        {
+            soci::statement st = session_->prepare << p;
+            st.execute(true);
+        }
+
+        if (commonPragma)
+        {
+            for (auto const& p : *commonPragma)
+            {
+                soci::statement st = session_->prepare << p;
+                st.execute(true);
+            }
+        }
+
+        for (auto const& sql : initSQL)
+        {
+            soci::statement st = session_->prepare << sql;
+            st.execute(true);
+        }
+    }
+
+    LockedSociSession::mutex lock_;
+
+    // checkpointer may outlive the DatabaseCon when the checkpointer jobQueue
+    // callback locks a weak pointer and the DatabaseCon is then destroyed. In
+    // this case, the checkpointer needs to make sure it doesn't use an already
+    // destroyed session. Thus this class keeps a shared_ptr to the session (so
+    // the checkpointer can keep a weak_ptr) and the checkpointer is a
+    // shared_ptr in this class. session_ will never be null.
+    std::shared_ptr<soci::session> const session_;
+    std::shared_ptr<Checkpointer> checkpointer_;
+
+    beast::Journal const j_;
+};
+
+// Return the checkpointer from its id. If the checkpointer no longer exists, an
+// nullptr is returned
+std::shared_ptr<Checkpointer>
+checkpointerFromId(std::uintptr_t id);
+
+}  // namespace xrpl

include/xrpl/rdb/SociDB.h

@@ -0,0 +1,120 @@
+#pragma once
+
+/** An embedded database wrapper with an intuitive, type-safe interface.
+
+    This collection of classes let's you access embedded SQLite databases
+    using C++ syntax that is very similar to regular SQL.
+
+    This module requires the @ref beast_sqlite external module.
+*/
+
+#if defined(__clang__)
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wdeprecated"
+#endif
+
+#include <xrpl/basics/Log.h>
+#include <xrpl/core/JobQueue.h>
+
+#define SOCI_USE_BOOST
+#include <soci/soci.h>
+
+#include <cstdint>
+#include <string>
+#include <vector>
+
+namespace sqlite_api {
+struct sqlite3;
+}
+
+namespace xrpl {
+
+class BasicConfig;
+
+/**
+   DBConfig is used when a client wants to delay opening a soci::session after
+   parsing the config parameters. If a client want to open a session
+   immediately, use the free function "open" below.
+ */
+class DBConfig
+{
+    std::string connectionString_;
+    explicit DBConfig(std::string const& dbPath);
+
+public:
+    DBConfig(BasicConfig const& config, std::string const& dbName);
+    std::string
+    connectionString() const;
+    void
+    open(soci::session& s) const;
+};
+
+/**
+   Open a soci session.
+
+   @param s Session to open.
+
+   @param config Parameters to pick the soci backend and how to connect to that
+                 backend.
+
+   @param dbName Name of the database. This has different meaning for different
+                 backends. Sometimes it is part of a filename (sqlite3),
+                 other times it is a database name (postgresql).
+*/
+void
+open(soci::session& s, BasicConfig const& config, std::string const& dbName);
+
+/**
+ *  Open a soci session.
+ *
+ *  @param s Session to open.
+ *  @param beName Backend name.
+ *  @param connectionString Connection string to forward to soci::open.
+ *         see the soci::open documentation for how to use this.
+ *
+ */
+void
+open(soci::session& s, std::string const& beName, std::string const& connectionString);
+
+std::uint32_t
+getKBUsedAll(soci::session& s);
+std::uint32_t
+getKBUsedDB(soci::session& s);
+
+void
+convert(soci::blob& from, std::vector<std::uint8_t>& to);
+void
+convert(soci::blob& from, std::string& to);
+void
+convert(std::vector<std::uint8_t> const& from, soci::blob& to);
+void
+convert(std::string const& from, soci::blob& to);
+
+class Checkpointer : public std::enable_shared_from_this<Checkpointer>
+{
+public:
+    virtual std::uintptr_t
+    id() const = 0;
+    virtual ~Checkpointer() = default;
+
+    virtual void
+    schedule() = 0;
+
+    virtual void
+    checkpoint() = 0;
+};
+
+/** Returns a new checkpointer which makes checkpoints of a
+    soci database every checkpointPageCount pages, using a job on the job queue.
+
+    The checkpointer contains references to the session and job queue
+    and so must outlive them both.
+ */
+std::shared_ptr<Checkpointer>
+makeCheckpointer(std::uintptr_t id, std::weak_ptr<soci::session>, JobQueue&, Logs&);
+
+}  // namespace xrpl
+
+#if defined(__clang__)
+#pragma clang diagnostic pop
+#endif

include/xrpl/server/Manifest.h

@@ -0,0 +1,428 @@
+#pragma once
+
+#include <xrpl/basics/UnorderedContainers.h>
+#include <xrpl/beast/utility/Journal.h>
+#include <xrpl/protocol/PublicKey.h>
+#include <xrpl/protocol/SecretKey.h>
+
+#include <optional>
+#include <shared_mutex>
+#include <string>
+
+namespace xrpl {
+
+/*
+    Validator key manifests
+    -----------------------
+
+    Suppose the secret keys installed on a Ripple validator are compromised. Not
+    only do you have to generate and install new key pairs on each validator,
+    EVERY rippled needs to have its config updated with the new public keys, and
+    is vulnerable to forged validation signatures until this is done.  The
+    solution is a new layer of indirection: A master secret key under
+    restrictive access control is used to sign a "manifest": essentially, a
+    certificate including the master public key, an ephemeral public key for
+    verifying validations (which will be signed by its secret counterpart), a
+    sequence number, and a digital signature.
+
+    The manifest has two serialized forms: one which includes the digital
+    signature and one which doesn't.  There is an obvious causal dependency
+    relationship between the (latter) form with no signature, the signature
+    of that form, and the (former) form which includes that signature.  In
+    other words, a message can't contain a signature of itself.  The code
+    below stores a serialized manifest which includes the signature, and
+    dynamically generates the signatureless form when it needs to verify
+    the signature.
+
+    An instance of ManifestCache stores, for each trusted validator, (a) its
+    master public key, and (b) the most senior of all valid manifests it has
+    seen for that validator, if any.  On startup, the [validator_token] config
+    entry (which contains the manifest for this validator) is decoded and
+    added to the manifest cache.  Other manifests are added as "gossip"
+    received from rippled peers.
+
+    When an ephemeral key is compromised, a new signing key pair is created,
+    along with a new manifest vouching for it (with a higher sequence number),
+    signed by the master key.  When a rippled peer receives the new manifest,
+    it verifies it with the master key and (assuming it's valid) discards the
+    old ephemeral key and stores the new one.  If the master key itself gets
+    compromised, a manifest with sequence number 0xFFFFFFFF will supersede a
+    prior manifest and discard any existing ephemeral key without storing a
+    new one.  These revocation manifests are loaded from the
+    [validator_key_revocation] config entry as well as received as gossip from
+    peers.  Since no further manifests for this master key will be accepted
+    (since no higher sequence number is possible), and no signing key is on
+    record, no validations will be accepted from the compromised validator.
+*/
+
+//------------------------------------------------------------------------------
+
+struct Manifest
+{
+    /// The manifest in serialized form.
+    std::string serialized;
+
+    /// The master key associated with this manifest.
+    PublicKey masterKey;
+
+    /// The ephemeral key associated with this manifest.
+    // A revoked manifest does not have a signingKey
+    // This field is specified as "optional" in manifestFormat's
+    // SOTemplate
+    std::optional<PublicKey> signingKey;
+
+    /// The sequence number of this manifest.
+    std::uint32_t sequence = 0;
+
+    /// The domain, if one was specified in the manifest; empty otherwise.
+    std::string domain;
+
+    Manifest() = delete;
+
+    Manifest(
+        std::string const& serialized_,
+        PublicKey const& masterKey_,
+        std::optional<PublicKey> const& signingKey_,
+        std::uint32_t seq,
+        std::string const& domain_)
+        : serialized(serialized_), masterKey(masterKey_), signingKey(signingKey_), sequence(seq), domain(domain_)
+    {
+    }
+
+    Manifest(Manifest const& other) = delete;
+    Manifest&
+    operator=(Manifest const& other) = delete;
+    Manifest(Manifest&& other) = default;
+    Manifest&
+    operator=(Manifest&& other) = default;
+
+    /// Returns `true` if manifest signature is valid
+    bool
+    verify() const;
+
+    /// Returns hash of serialized manifest data
+    uint256
+    hash() const;
+
+    /// Returns `true` if manifest revokes master key
+    // The maximum possible sequence number means that the master key has
+    // been revoked
+    static bool
+    revoked(std::uint32_t sequence);
+
+    /// Returns `true` if manifest revokes master key
+    bool
+    revoked() const;
+
+    /// Returns manifest signature
+    std::optional<Blob>
+    getSignature() const;
+
+    /// Returns manifest master key signature
+    Blob
+    getMasterSignature() const;
+};
+
+/** Format the specified manifest to a string for debugging purposes. */
+std::string
+to_string(Manifest const& m);
+
+/** Constructs Manifest from serialized string
+
+    @param s Serialized manifest string
+
+    @return `std::nullopt` if string is invalid
+
+    @note This does not verify manifest signatures.
+          `Manifest::verify` should be called after constructing manifest.
+*/
+/** @{ */
+std::optional<Manifest>
+deserializeManifest(Slice s, beast::Journal journal);
+
+inline std::optional<Manifest>
+deserializeManifest(std::string const& s, beast::Journal journal = beast::Journal(beast::Journal::getNullSink()))
+{
+    return deserializeManifest(makeSlice(s), journal);
+}
+
+template <class T, class = std::enable_if_t<std::is_same<T, char>::value || std::is_same<T, unsigned char>::value>>
+std::optional<Manifest>
+deserializeManifest(std::vector<T> const& v, beast::Journal journal = beast::Journal(beast::Journal::getNullSink()))
+{
+    return deserializeManifest(makeSlice(v), journal);
+}
+/** @} */
+
+inline bool
+operator==(Manifest const& lhs, Manifest const& rhs)
+{
+    // In theory, comparing the two serialized strings should be
+    // sufficient.
+    return lhs.sequence == rhs.sequence && lhs.masterKey == rhs.masterKey && lhs.signingKey == rhs.signingKey &&
+        lhs.domain == rhs.domain && lhs.serialized == rhs.serialized;
+}
+
+inline bool
+operator!=(Manifest const& lhs, Manifest const& rhs)
+{
+    return !(lhs == rhs);
+}
+
+struct ValidatorToken
+{
+    std::string manifest;
+    SecretKey validationSecret;
+};
+
+std::optional<ValidatorToken>
+loadValidatorToken(
+    std::vector<std::string> const& blob,
+    beast::Journal journal = beast::Journal(beast::Journal::getNullSink()));
+
+enum class ManifestDisposition {
+    /// Manifest is valid
+    accepted = 0,
+
+    /// Sequence is too old
+    stale,
+
+    /// The master key is not acceptable to us
+    badMasterKey,
+
+    /// The ephemeral key is not acceptable to us
+    badEphemeralKey,
+
+    /// Timely, but invalid signature
+    invalid
+};
+
+inline std::string
+to_string(ManifestDisposition m)
+{
+    switch (m)
+    {
+        case ManifestDisposition::accepted:
+            return "accepted";
+        case ManifestDisposition::stale:
+            return "stale";
+        case ManifestDisposition::badMasterKey:
+            return "badMasterKey";
+        case ManifestDisposition::badEphemeralKey:
+            return "badEphemeralKey";
+        case ManifestDisposition::invalid:
+            return "invalid";
+        default:
+            return "unknown";
+    }
+}
+
+class DatabaseCon;
+
+/** Remembers manifests with the highest sequence number. */
+class ManifestCache
+{
+private:
+    beast::Journal j_;
+    std::shared_mutex mutable mutex_;
+
+    /** Active manifests stored by master public key. */
+    hash_map<PublicKey, Manifest> map_;
+
+    /** Master public keys stored by current ephemeral public key. */
+    hash_map<PublicKey, PublicKey> signingToMasterKeys_;
+
+    std::atomic<std::uint32_t> seq_{0};
+
+public:
+    explicit ManifestCache(beast::Journal j = beast::Journal(beast::Journal::getNullSink())) : j_(j)
+    {
+    }
+
+    /** A monotonically increasing number used to detect new manifests. */
+    std::uint32_t
+    sequence() const
+    {
+        return seq_.load();
+    }
+
+    /** Returns master key's current signing key.
+
+        @param pk Master public key
+
+        @return pk if no known signing key from a manifest
+
+        @par Thread Safety
+
+        May be called concurrently
+    */
+    std::optional<PublicKey>
+    getSigningKey(PublicKey const& pk) const;
+
+    /** Returns ephemeral signing key's master public key.
+
+        @param pk Ephemeral signing public key
+
+        @return pk if signing key is not in a valid manifest
+
+        @par Thread Safety
+
+        May be called concurrently
+    */
+    PublicKey
+    getMasterKey(PublicKey const& pk) const;
+
+    /** Returns master key's current manifest sequence.
+
+        @return sequence corresponding to Master public key
+          if configured or std::nullopt otherwise
+    */
+    std::optional<std::uint32_t>
+    getSequence(PublicKey const& pk) const;
+
+    /** Returns domain claimed by a given public key
+
+        @return domain corresponding to Master public key
+          if present, otherwise std::nullopt
+    */
+    std::optional<std::string>
+    getDomain(PublicKey const& pk) const;
+
+    /** Returns manifest corresponding to a given public key
+
+        @return manifest corresponding to Master public key
+          if present, otherwise std::nullopt
+    */
+    std::optional<std::string>
+    getManifest(PublicKey const& pk) const;
+
+    /** Returns `true` if master key has been revoked in a manifest.
+
+        @param pk Master public key
+
+        @par Thread Safety
+
+        May be called concurrently
+    */
+    bool
+    revoked(PublicKey const& pk) const;
+
+    /** Add manifest to cache.
+
+        @param m Manifest to add
+
+        @return `ManifestDisposition::accepted` if successful, or
+                `stale` or `invalid` otherwise
+
+        @par Thread Safety
+
+        May be called concurrently
+    */
+    ManifestDisposition
+    applyManifest(Manifest m);
+
+    /** Populate manifest cache with manifests in database and config.
+
+        @param dbCon Database connection with dbTable
+
+        @param dbTable Database table
+
+        @param configManifest Base64 encoded manifest for local node's
+            validator keys
+
+        @param configRevocation Base64 encoded validator key revocation
+            from the config
+
+        @par Thread Safety
+
+        May be called concurrently
+    */
+    bool
+    load(
+        DatabaseCon& dbCon,
+        std::string const& dbTable,
+        std::string const& configManifest,
+        std::vector<std::string> const& configRevocation);
+
+    /** Populate manifest cache with manifests in database.
+
+        @param dbCon Database connection with dbTable
+
+        @param dbTable Database table
+
+        @par Thread Safety
+
+        May be called concurrently
+    */
+    void
+    load(DatabaseCon& dbCon, std::string const& dbTable);
+
+    /** Save cached manifests to database.
+
+        @param dbCon Database connection with `ValidatorManifests` table
+
+        @param isTrusted Function that returns true if manifest is trusted
+
+        @par Thread Safety
+
+        May be called concurrently
+    */
+    void
+    save(DatabaseCon& dbCon, std::string const& dbTable, std::function<bool(PublicKey const&)> const& isTrusted);
+
+    /** Invokes the callback once for every populated manifest.
+
+        @note Do not call ManifestCache member functions from within the
+        callback. This can re-lock the mutex from the same thread, which is UB.
+        @note Do not write ManifestCache member variables from within the
+        callback. This can lead to data races.
+
+        @param f Function called for each manifest
+
+        @par Thread Safety
+
+        May be called concurrently
+    */
+    template <class Function>
+    void
+    for_each_manifest(Function&& f) const
+    {
+        std::shared_lock lock{mutex_};
+        for (auto const& [_, manifest] : map_)
+        {
+            (void)_;
+            f(manifest);
+        }
+    }
+
+    /** Invokes the callback once for every populated manifest.
+
+        @note Do not call ManifestCache member functions from within the
+        callback. This can re-lock the mutex from the same thread, which is UB.
+        @note Do not write ManifestCache member variables from
+        within the callback. This can lead to data races.
+
+        @param pf Pre-function called with the maximum number of times f will be
+            called (useful for memory allocations)
+
+        @param f Function called for each manifest
+
+        @par Thread Safety
+
+        May be called concurrently
+    */
+    template <class PreFun, class EachFun>
+    void
+    for_each_manifest(PreFun&& pf, EachFun&& f) const
+    {
+        std::shared_lock lock{mutex_};
+        pf(map_.size());
+        for (auto const& [_, manifest] : map_)
+        {
+            (void)_;
+            f(manifest);
+        }
+    }
+};
+
+}  // namespace xrpl

include/xrpl/server/State.h

@@ -0,0 +1,71 @@
+#pragma once
+
+#include <xrpl/protocol/Protocol.h>
+#include <xrpl/rdb/DatabaseCon.h>
+#include <xrpl/server/Manifest.h>
+
+#include <boost/filesystem.hpp>
+
+namespace xrpl {
+
+struct SavedState
+{
+    std::string writableDb;
+    std::string archiveDb;
+    LedgerIndex lastRotated;
+};
+
+/**
+ * @brief initStateDB Opens a session with the State database.
+ * @param session Provides a session with the database.
+ * @param config Path to the database and other opening parameters.
+ * @param dbName Name of the database.
+ */
+void
+initStateDB(soci::session& session, BasicConfig const& config, std::string const& dbName);
+
+/**
+ * @brief getCanDelete Returns the ledger sequence which can be deleted.
+ * @param session Session with the database.
+ * @return Ledger sequence.
+ */
+LedgerIndex
+getCanDelete(soci::session& session);
+
+/**
+ * @brief setCanDelete Updates the ledger sequence which can be deleted.
+ * @param session Session with the database.
+ * @param canDelete Ledger sequence to save.
+ * @return Previous value of the ledger sequence which can be deleted.
+ */
+LedgerIndex
+setCanDelete(soci::session& session, LedgerIndex canDelete);
+
+/**
+ * @brief getSavedState Returns the saved state.
+ * @param session Session with the database.
+ * @return The SavedState structure which contains the names of the writable
+ *         database, the archive database and the last rotated ledger sequence.
+ */
+SavedState
+getSavedState(soci::session& session);
+
+/**
+ * @brief setSavedState Saves the given state.
+ * @param session Session with the database.
+ * @param state The SavedState structure which contains the names of the
+ *        writable database, the archive database and the last rotated ledger
+ *        sequence.
+ */
+void
+setSavedState(soci::session& session, SavedState const& state);
+
+/**
+ * @brief setLastRotated Updates the last rotated ledger sequence.
+ * @param session Session with the database.
+ * @param seq New value of the last rotated ledger sequence.
+ */
+void
+setLastRotated(soci::session& session, LedgerIndex seq);
+
+}  // namespace xrpl

include/xrpl/server/Vacuum.h

@@ -0,0 +1,16 @@
+#pragma once
+
+#include <xrpl/rdb/DatabaseCon.h>
+
+namespace xrpl {
+
+/**
+ * @brief doVacuumDB Creates, initialises, and performs cleanup on a database.
+ * @param setup Path to the database and other opening parameters.
+ * @param j Journal.
+ * @return True if the vacuum process completed successfully.
+ */
+bool
+doVacuumDB(DatabaseCon::Setup const& setup, beast::Journal j);
+
+}  // namespace xrpl

include/xrpl/server/Wallet.h

@@ -0,0 +1,145 @@
+#pragma once
+
+#include <xrpl/core/PeerReservationTable.h>
+#include <xrpl/rdb/DatabaseCon.h>
+#include <xrpl/server/Manifest.h>
+
+namespace xrpl {
+
+/**
+ * @brief makeWalletDB Opens the wallet database and returns it.
+ * @param setup Path to the database and other opening parameters.
+ * @param j Journal.
+ * @return Unique pointer to the database descriptor.
+ */
+std::unique_ptr<DatabaseCon>
+makeWalletDB(DatabaseCon::Setup const& setup, beast::Journal j);
+
+/**
+ * @brief makeTestWalletDB Opens a test wallet database with an arbitrary name.
+ * @param setup Path to the database and other opening parameters.
+ * @param dbname Name of the database.
+ * @param j Journal.
+ * @return Unique pointer to the database descriptor.
+ */
+std::unique_ptr<DatabaseCon>
+makeTestWalletDB(DatabaseCon::Setup const& setup, std::string const& dbname, beast::Journal j);
+
+/**
+ * @brief getManifests Loads a manifest from the wallet database and stores it
+ *        in the cache.
+ * @param session Session with the database.
+ * @param dbTable Name of the database table from which the manifest will be
+ *        extracted.
+ * @param mCache Cache for storing the manifest.
+ * @param j Journal.
+ */
+void
+getManifests(soci::session& session, std::string const& dbTable, ManifestCache& mCache, beast::Journal j);
+
+/**
+ * @brief saveManifests Saves all given manifests to the database.
+ * @param session Session with the database.
+ * @param dbTable Name of the database table that will store the manifest.
+ * @param isTrusted Callback that returns true if the key is trusted.
+ * @param map Maps public keys to manifests.
+ * @param j Journal.
+ */
+void
+saveManifests(
+    soci::session& session,
+    std::string const& dbTable,
+    std::function<bool(PublicKey const&)> const& isTrusted,
+    hash_map<PublicKey, Manifest> const& map,
+    beast::Journal j);
+
+/**
+ * @brief addValidatorManifest Saves the manifest of a validator to the
+ *        database.
+ * @param session Session with the database.
+ * @param serialized Manifest of the validator in raw format.
+ */
+void
+addValidatorManifest(soci::session& session, std::string const& serialized);
+
+/** Delete any saved public/private key associated with this node. */
+void
+clearNodeIdentity(soci::session& session);
+
+/** Returns a stable public and private key for this node.
+
+    The node's public identity is defined by a secp256k1 keypair
+    that is (normally) randomly generated. This function will
+    return such a keypair, securely generating one if needed.
+
+    @param session Session with the database.
+
+    @return Pair of public and private secp256k1 keys.
+ */
+std::pair<PublicKey, SecretKey>
+getNodeIdentity(soci::session& session);
+
+/**
+ * @brief getPeerReservationTable Returns the peer reservation table.
+ * @param session Session with the database.
+ * @param j Journal.
+ * @return Peer reservation hash table.
+ */
+std::unordered_set<PeerReservation, beast::uhash<>, KeyEqual>
+getPeerReservationTable(soci::session& session, beast::Journal j);
+
+/**
+ * @brief insertPeerReservation Adds an entry to the peer reservation table.
+ * @param session Session with the database.
+ * @param nodeId Public key of the node.
+ * @param description Description of the node.
+ */
+void
+insertPeerReservation(soci::session& session, PublicKey const& nodeId, std::string const& description);
+
+/**
+ * @brief deletePeerReservation Deletes an entry from the peer reservation
+ *        table.
+ * @param session Session with the database.
+ * @param nodeId Public key of the node to remove.
+ */
+void
+deletePeerReservation(soci::session& session, PublicKey const& nodeId);
+
+/**
+ * @brief createFeatureVotes Creates the FeatureVote table if it does not exist.
+ * @param session Session with the wallet database.
+ * @return true if the table already exists
+ */
+bool
+createFeatureVotes(soci::session& session);
+
+// For historical reasons the up-vote and down-vote integer representations
+// are unintuitive.
+enum class AmendmentVote : int { obsolete = -1, up = 0, down = 1 };
+
+/**
+ * @brief readAmendments Reads all amendments from the FeatureVotes table.
+ * @param session Session with the wallet database.
+ * @param callback Callback called for each amendment with its hash, name and
+ *        optionally a flag denoting whether the amendment should be vetoed.
+ */
+void
+readAmendments(
+    soci::session& session,
+    std::function<void(
+        boost::optional<std::string> amendment_hash,
+        boost::optional<std::string> amendment_name,
+        boost::optional<AmendmentVote> vote)> const& callback);
+
+/**
+ * @brief voteAmendment Set the veto value for a particular amendment.
+ * @param session Session with the wallet database.
+ * @param amendment Hash of the amendment.
+ * @param name Name of the amendment.
+ * @param vote Whether to vote in favor of this amendment.
+ */
+void
+voteAmendment(soci::session& session, uint256 const& amendment, std::string const& name, AmendmentVote vote);
+
+}  // namespace xrpl