ARCHIVE/clio
1
0
Fork 0
mirror of https://github.com/XRPLF/clio.git synced 2025-11-26 14:45:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add state data cache and successor table. Remove keys table

Browse Source 
* Adds a cache for the most recently validated ledger state
* Replaces the keys table with successor table
* Adds support for ledger diffs in the database
* Removes BackendIndexer
This commit is contained in:
CJ Cobb
2021-11-19 16:10:50 -05:00
parent e930ff04df
commit c7e31aff56
35 changed files with 2801 additions and 1241 deletions
Download Patch File Download Diff File Expand all files Collapse all files

173
src/backend/BackendInterface.h
View File

@@ -2,67 +2,11 @@
#define RIPPLE_APP_REPORTING_BACKENDINTERFACE_H_INCLUDED
#include <ripple/ledger/ReadView.h>
#include <boost/asio.hpp>
#include <backend/BackendIndexer.h>
#include <backend/DBHelpers.h>
class ReportingETL;
class AsyncCallData;
class BackendTest_Basic_Test;
#include <backend/SimpleCache.h>
#include <backend/Types.h>
namespace Backend {
// *** return types
using Blob = std::vector<unsigned char>;
struct LedgerObject
{
ripple::uint256 key;
Blob blob;
};
struct LedgerPage
{
std::vector<LedgerObject> objects;
std::optional<ripple::uint256> cursor;
std::optional<std::string> warning;
};
struct BookOffersPage
{
std::vector<LedgerObject> offers;
std::optional<ripple::uint256> cursor;
std::optional<std::string> warning;
};
struct TransactionAndMetadata
{
Blob transaction;
Blob metadata;
uint32_t ledgerSequence;
uint32_t date;
bool
operator==(const TransactionAndMetadata& other) const
{
return transaction == other.transaction && metadata == other.metadata &&
ledgerSequence == other.ledgerSequence && date == other.date;
}
};
struct AccountTransactionsCursor
{
uint32_t ledgerSequence;
uint32_t transactionIndex;
};
struct AccountTransactions
{
std::vector<TransactionAndMetadata> txns;
std::optional<AccountTransactionsCursor> cursor;
};
struct LedgerRange
{
uint32_t minSequence;
uint32_t maxSequence;
};
class DatabaseTimeout : public std::exception
{
const char*
@@ -75,30 +19,37 @@ class DatabaseTimeout : public std::exception
class BackendInterface
{
protected:
mutable BackendIndexer indexer_;
mutable bool isFirst_ = true;
mutable std::optional<LedgerRange> range;
bool isFirst_ = true;
std::optional<LedgerRange> range;
SimpleCache cache_;
public:
BackendInterface(boost::json::object const& config) : indexer_(config)
BackendInterface(boost::json::object const& config)
{
}
virtual ~BackendInterface()
{
}
BackendIndexer&
getIndexer() const
{
return indexer_;
}
// *** public read methods ***
// All of these reads methods can throw DatabaseTimeout. When writing code
// in an RPC handler, this exception does not need to be caught: when an RPC
// results in a timeout, an error is returned to the client
public:
// *** ledger methods
//
SimpleCache const&
cache() const
{
return cache_;
}
SimpleCache&
cache()
{
return cache_;
}
virtual std::optional<ripple::LedgerInfo>
fetchLedgerBySequence(uint32_t sequence) const = 0;
@@ -139,15 +90,25 @@ public:
fetchAllTransactionHashesInLedger(uint32_t ledgerSequence) const = 0;
// *** state data methods
std::optional<Blob>
fetchLedgerObject(ripple::uint256 const& key, uint32_t sequence) const;
virtual std::optional<Blob>
fetchLedgerObject(ripple::uint256 const& key, uint32_t sequence) const = 0;
virtual std::vector<Blob>
std::vector<Blob>
fetchLedgerObjects(
std::vector<ripple::uint256> const& keys,
uint32_t sequence) const;
virtual std::optional<Blob>
doFetchLedgerObject(ripple::uint256 const& key, uint32_t sequence)
const = 0;
virtual std::vector<Blob>
doFetchLedgerObjects(
std::vector<ripple::uint256> const& keys,
uint32_t sequence) const = 0;
virtual std::vector<LedgerObject>
fetchLedgerDiff(uint32_t ledgerSequence) const = 0;
// Fetches a page of ledger objects, ordered by key/index.
// Used by ledger_data
LedgerPage
@@ -157,10 +118,15 @@ public:
std::uint32_t limit,
std::uint32_t limitHint = 0) const;
// Fetches the successor to key/index. key need not actually be a valid
// key/index.
// Fetches the successor to key/index
std::optional<LedgerObject>
fetchSuccessor(ripple::uint256 key, uint32_t ledgerSequence) const;
fetchSuccessorObject(ripple::uint256 key, uint32_t ledgerSequence) const;
std::optional<ripple::uint256>
fetchSuccessorKey(ripple::uint256 key, uint32_t ledgerSequence) const;
// Fetches the successor to key/index
virtual std::optional<ripple::uint256>
doFetchSuccessorKey(ripple::uint256 key, uint32_t ledgerSequence) const = 0;
BookOffersPage
fetchBookOffers(
@@ -169,21 +135,6 @@ public:
std::uint32_t limit,
std::optional<ripple::uint256> const& cursor = {}) const;
// Methods related to the indexer
bool
isLedgerIndexed(std::uint32_t ledgerSequence) const;
std::optional<KeyIndex>
getKeyIndexOfSeq(uint32_t seq) const;
// *** protected write methods
protected:
friend class ::ReportingETL;
friend class BackendIndexer;
friend class ::AsyncCallData;
friend std::shared_ptr<BackendInterface>
make_Backend(boost::json::object const& config);
friend class ::BackendTest_Basic_Test;
virtual std::optional<LedgerRange>
hardFetchLedgerRange() const = 0;
// Doesn't throw DatabaseTimeout. Should be used with care.
@@ -203,11 +154,10 @@ protected:
writeLedger(
ripple::LedgerInfo const& ledgerInfo,
std::string&& ledgerHeader,
bool isFirst = false) const = 0;
bool isFirst = false) = 0;
void
writeLedgerObject(std::string&& key, uint32_t seq, std::string&& blob)
const;
writeLedgerObject(std::string&& key, uint32_t seq, std::string&& blob);
virtual void
writeTransaction(
@@ -215,24 +165,20 @@ protected:
uint32_t seq,
uint32_t date,
std::string&& transaction,
std::string&& metadata) const = 0;
std::string&& metadata) = 0;
virtual void
writeAccountTransactions(
std::vector<AccountTransactionsData>&& data) const = 0;
// TODO: this function, or something similar, could be called internally by
// writeLedgerObject
virtual bool
writeKeys(
std::unordered_set<ripple::uint256> const& keys,
KeyIndex const& index,
bool isAsync = false) const = 0;
writeAccountTransactions(std::vector<AccountTransactionsData>&& data) = 0;
virtual void
writeSuccessor(
std::string&& key,
uint32_t seq,
std::string&& successor) = 0;
// Tell the database we are about to begin writing data for a particular
// ledger.
virtual void
startWrites() const = 0;
startWrites() = 0;
// Tell the database we have finished writing all data for a particular
// ledger
@@ -254,21 +200,14 @@ protected:
// *** private helper methods
private:
virtual LedgerPage
doFetchLedgerPage(
std::optional<ripple::uint256> const& cursor,
std::uint32_t ledgerSequence,
std::uint32_t limit) const = 0;
virtual void
doWriteLedgerObject(std::string&& key, uint32_t seq, std::string&& blob)
const = 0;
doWriteLedgerObject(
std::string&& key,
uint32_t seq,
std::string&& blob) = 0;
virtual bool
doFinishWrites() const = 0;
void
checkFlagLedgers() const;
doFinishWrites() = 0;
};
} // namespace Backend