ARCHIVE/hpcore
1
0
Fork 0
mirror of https://github.com/EvernodeXRPL/hpcore.git synced 2026-04-29 15:37:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

Save raw inputs alongside the ledger (#167)

Browse Source 
* Saving raw inputs in the ledger.
* Store full history in separate folder.

* Moved full history fb structs to a seperate fbs file

* Removed raw_inputs from the proposal

* Fixed code comments

* Saving raw user input string

* Code refactoring

* Resolved PR comments
This commit is contained in:
Chalith Desaman
2020-11-30 15:42:20 +05:30
committed by GitHub
parent e90e9bb5dd
commit 2f5d95ae7a
17 changed files with 410 additions and 144 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
src/conf.cpp
View File

@@ -79,6 +79,7 @@ namespace conf
// Recursivly create contract directories.
util::create_dir_tree_recursive(ctx.config_dir);
util::create_dir_tree_recursive(ctx.hist_dir);
util::create_dir_tree_recursive(ctx.full_hist_dir);
util::create_dir_tree_recursive(ctx.log_dir);
util::create_dir_tree_recursive(ctx.state_dir);
@@ -100,6 +101,7 @@ namespace conf
cfg.msgforwarding = false;
cfg.dynamicpeerdiscovery = false;
cfg.fullhistory = false;
#ifndef NDEBUG
cfg.loglevel_type = conf::LOG_SEVERITY::DEBUG;
@@ -158,6 +160,7 @@ namespace conf
ctx.tls_key_file = ctx.config_dir + "/tlskey.pem";
ctx.tls_cert_file = ctx.config_dir + "/tlscert.pem";
ctx.hist_dir = basedir + "/hist";
ctx.full_hist_dir = basedir + "/fullhist";
ctx.state_dir = basedir + "/state";
ctx.state_rw_dir = ctx.state_dir + "/rw";
ctx.state_serve_dir = ctx.state_dir + "/ss";
@@ -314,6 +317,7 @@ namespace conf
cfg.msgforwarding = d["msgforwarding"].as<bool>();
cfg.dynamicpeerdiscovery = d["dynamicpeerdiscovery"].as<bool>();
cfg.fullhistory = d["fullhistory"].as<bool>();
cfg.loglevel = d["loglevel"].as<std::string>();
cfg.loglevel_type = get_loglevel_type(cfg.loglevel);
@@ -388,6 +392,7 @@ namespace conf
d.insert_or_assign("msgforwarding", cfg.msgforwarding);
d.insert_or_assign("dynamicpeerdiscovery", cfg.dynamicpeerdiscovery);
d.insert_or_assign("fullhistory", cfg.fullhistory);
d.insert_or_assign("loglevel", cfg.loglevel);
@@ -558,10 +563,11 @@ namespace conf
*/
int validate_contract_dir_paths()
{
const std::string paths[6] = {
const std::string paths[7] = {
ctx.contract_dir,
ctx.config_file,
ctx.hist_dir,
ctx.full_hist_dir,
ctx.state_dir,
ctx.tls_key_file,
ctx.tls_cert_file};

2
src/conf.hpp
View File

@@ -61,6 +61,7 @@ namespace conf
std::string hpfs_exe_path; // hpfs executable file path.
std::string contract_dir; // Contract base directory full path
std::string full_hist_dir; // Contract full history dir full path
std::string hist_dir; // Contract ledger history dir full path
std::string state_dir; // Contract state maintenence path (hpfs path)
std::string state_rw_dir; // Contract executation read/write state path.
@@ -114,6 +115,7 @@ namespace conf
bool msgforwarding = false; // Whether peer message forwarding is on/off.
bool dynamicpeerdiscovery = false; // Whether dynamic peer discovery is on/off.
bool fullhistory = false; // Whether full history mode is on/off.
std::string loglevel; // Log severity level (debug, info, warn, error)
LOG_SEVERITY loglevel_type; // Log severity level enum (debug, info, warn, error)

26
src/consensus.cpp
View File

@@ -728,7 +728,31 @@ namespace consensus
*/
int update_ledger_and_execute_contract(const p2p::proposal &cons_prop, std::string &new_lcl, hpfs::h32 &new_state)
{
if (ledger::save_ledger(cons_prop) == -1)
// Map to temporarily store the raw inputs along with the hash.
std::unordered_map<std::string, usr::raw_user_input> raw_inputs;
// Add raw_inputs to the proposal if full history mode is on.
if (conf::cfg.fullhistory)
{
for (const auto &hash : cons_prop.hash_inputs)
{
const auto itr = ctx.candidate_user_inputs.find(hash);
if (itr != ctx.candidate_user_inputs.end())
{
// Add raw_input to the map along with the input hash.
candidate_user_input &cand_input = itr->second;
// Taking the raw input string from the buffer_view.
std::string input;
if (usr::input_store.read_buf(cand_input.input, input) != -1)
{
usr::raw_user_input raw_input(cand_input.userpubkey, std::move(input));
raw_inputs.emplace(hash, std::move(raw_input));
}
}
}
}
if (ledger::save_ledger(cons_prop, std::move(raw_inputs)) == -1)
return -1;
new_lcl = ledger::ctx.get_lcl();

54
src/ledger.cpp
View File

@@ -290,8 +290,9 @@ namespace ledger
/**
* Create and save ledger from the given proposal message. Called by consensus.
* @param proposal Consensus-reached Stage 3 proposal.
* @param raw_inputs Raw inputs that are going to store.
*/
int save_ledger(const p2p::proposal &proposal)
int save_ledger(const p2p::proposal &proposal, const std::unordered_map<std::string, usr::raw_user_input> raw_inputs)
{
uint64_t seq_no = 0;
std::string hash;
@@ -331,6 +332,15 @@ namespace ledger
ctx.cache.emplace(seq_no, std::move(file_name));
// Write full history to the full history directory if full history mode is on.
if (conf::cfg.fullhistory)
{
builder.Clear();
msg::fbuf::ledger::create_full_history_block_from_raw_input_map(builder, raw_inputs);
if (write_full_history(file_name, builder.GetBufferPointer(), builder.GetSize()) == -1)
return -1;
}
//Remove old ledgers that exceeds max sequence range.
if (seq_no > MAX_LEDGER_SEQUENCE)
remove_old_ledgers(seq_no - MAX_LEDGER_SEQUENCE);
@@ -437,6 +447,38 @@ namespace ledger
return 0;
}
/**
* Write full history to file system.
* @param file_name current ledger sequence number.
* @param full_history_raw raw full history data.
* @param full_history_size size of the raw full history data.
*/
int write_full_history(const std::string &file_name, const uint8_t *full_history_raw, const size_t full_history_size)
{
// Create file path to save full history.
// file name -> [ledger sequnce numer]-[lcl hex]
const std::string file_path = conf::ctx.full_hist_dir + "/" + file_name + ".flcl";
// Write full history to file system
const int fd = open(file_path.data(), O_CREAT | O_RDWR, FILE_PERMS);
if (fd == -1)
{
LOG_ERROR << errno << ": Error creating full history file. " << file_path;
return -1;
}
if (write(fd, full_history_raw, full_history_size) == -1)
{
LOG_ERROR << errno << ": Error writing to new full history file. " << file_path;
close(fd);
return -1;
}
close(fd);
return 0;
}
/**
* Delete ledger from file system.
* @param file_name name of ledger to be deleted.
@@ -450,6 +492,16 @@ namespace ledger
.append(file_name)
.append(".lcl");
util::remove_file(file_path);
// Removing full history if exists.
file_path.clear();
file_path.reserve(conf::ctx.full_hist_dir.size() + file_name.size() + 6);
file_path.append(conf::ctx.full_hist_dir)
.append("/")
.append(file_name)
.append(".flcl");
if (util::is_file_exists(file_path))
util::remove_file(file_path);
}
/**

4
src/ledger.hpp
View File

@@ -76,7 +76,7 @@ namespace ledger
const std::pair<uint64_t, std::string> get_ledger_cache_top();
int save_ledger(const p2p::proposal &proposal);
int save_ledger(const p2p::proposal &proposal, const std::unordered_map<std::string, usr::raw_user_input> raw_inputs);
void remove_old_ledgers(const uint64_t led_seq_no);
@@ -86,6 +86,8 @@ namespace ledger
int write_ledger(const std::string &file_name, const uint8_t *ledger_raw, const size_t ledger_size);
int write_full_history(const std::string &file_name, const uint8_t *full_history_raw, const size_t full_history_size);
void remove_ledger(const std::string &file_name);
void send_ledger_history_request(std::string_view current_lcl, std::string_view required_lcl);

1
src/msg/fbuf/common_helpers.cpp
View File

@@ -122,5 +122,4 @@ namespace msg::fbuf
}
return builder.CreateVector(fbvec);
}
} // namespace msg::fbuf

42
src/msg/fbuf/common_helpers.hpp
View File

@@ -7,42 +7,42 @@
namespace msg::fbuf
{
/**
/**
* This section contains common Flatbuffer message reading/writing helpers.
*/
//---Conversion helpers from flatbuffers data types to std data types---//
//---Conversion helpers from flatbuffers data types to std data types---//
std::string_view flatbuff_bytes_to_sv(const uint8_t *data, const flatbuffers::uoffset_t length);
std::string_view flatbuff_bytes_to_sv(const uint8_t *data, const flatbuffers::uoffset_t length);
std::string_view flatbuff_bytes_to_sv(const flatbuffers::Vector<uint8_t> *buffer);
std::string_view flatbuff_bytes_to_sv(const flatbuffers::Vector<uint8_t> *buffer);
std::string_view flatbuff_str_to_sv(const flatbuffers::String *buffer);
std::string_view flatbuff_str_to_sv(const flatbuffers::String *buffer);
hpfs::h32 flatbuff_bytes_to_hash(const flatbuffers::Vector<uint8_t> *buffer);
hpfs::h32 flatbuff_bytes_to_hash(const flatbuffers::Vector<uint8_t> *buffer);
const std::set<std::string>
flatbuf_bytearrayvector_to_stringlist(const flatbuffers::Vector<flatbuffers::Offset<ByteArray>> *fbvec);
const std::set<std::string>
flatbuf_bytearrayvector_to_stringlist(const flatbuffers::Vector<flatbuffers::Offset<ByteArray>> *fbvec);
const std::unordered_map<std::string, const std::string>
flatbuf_pairvector_to_stringmap(const flatbuffers::Vector<flatbuffers::Offset<BytesKeyValuePair>> *fbvec);
const std::unordered_map<std::string, const std::string>
flatbuf_pairvector_to_stringmap(const flatbuffers::Vector<flatbuffers::Offset<BytesKeyValuePair>> *fbvec);
//---Conversion helpers from std data types to flatbuffers data types---//
//---Conversion helpers from std data types to flatbuffers data types---//
const flatbuffers::Offset<flatbuffers::Vector<uint8_t>>
sv_to_flatbuff_bytes(flatbuffers::FlatBufferBuilder &builder, std::string_view sv);
const flatbuffers::Offset<flatbuffers::Vector<uint8_t>>
sv_to_flatbuff_bytes(flatbuffers::FlatBufferBuilder &builder, std::string_view sv);
const flatbuffers::Offset<flatbuffers::String>
sv_to_flatbuff_str(flatbuffers::FlatBufferBuilder &builder, std::string_view sv);
const flatbuffers::Offset<flatbuffers::String>
sv_to_flatbuff_str(flatbuffers::FlatBufferBuilder &builder, std::string_view sv);
const flatbuffers::Offset<flatbuffers::Vector<uint8_t>>
hash_to_flatbuff_bytes(flatbuffers::FlatBufferBuilder &builder, hpfs::h32 hash);
const flatbuffers::Offset<flatbuffers::Vector<uint8_t>>
hash_to_flatbuff_bytes(flatbuffers::FlatBufferBuilder &builder, hpfs::h32 hash);
const flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<ByteArray>>>
stringlist_to_flatbuf_bytearrayvector(flatbuffers::FlatBufferBuilder &builder, const std::set<std::string> &set);
const flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<ByteArray>>>
stringlist_to_flatbuf_bytearrayvector(flatbuffers::FlatBufferBuilder &builder, const std::set<std::string> &set);
const flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<BytesKeyValuePair>>>
stringmap_to_flatbuf_bytepairvector(flatbuffers::FlatBufferBuilder &builder, const std::unordered_map<std::string, const std::string> &map);
const flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<BytesKeyValuePair>>>
stringmap_to_flatbuf_bytepairvector(flatbuffers::FlatBufferBuilder &builder, const std::unordered_map<std::string, const std::string> &map);
} // namespace msg::fbuf

13
src/msg/fbuf/fullhistory_schema.fbs Normal file
View File

@@ -0,0 +1,13 @@
namespace msg.fbuf.ledger;
table FullHistoryBlock {
raw_inputs: [RawInput];
}
table RawInput {
hash:[ubyte];
pubkey:[ubyte];
input:[ubyte];
}
root_type FullHistoryBlock;

200
src/msg/fbuf/fullhistory_schema_generated.h Normal file
View File

@@ -0,0 +1,200 @@
// automatically generated by the FlatBuffers compiler, do not modify
#ifndef FLATBUFFERS_GENERATED_FULLHISTORYSCHEMA_MSG_FBUF_LEDGER_H_
#define FLATBUFFERS_GENERATED_FULLHISTORYSCHEMA_MSG_FBUF_LEDGER_H_
#include "flatbuffers/flatbuffers.h"
namespace msg {
namespace fbuf {
namespace ledger {
struct FullHistoryBlock;
struct FullHistoryBlockBuilder;
struct RawInput;
struct RawInputBuilder;
struct FullHistoryBlock FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
typedef FullHistoryBlockBuilder Builder;
enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
VT_RAW_INPUTS = 4
};
const flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ledger::RawInput>> *raw_inputs() const {
return GetPointer<const flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ledger::RawInput>> *>(VT_RAW_INPUTS);
}
flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ledger::RawInput>> *mutable_raw_inputs() {
return GetPointer<flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ledger::RawInput>> *>(VT_RAW_INPUTS);
}
bool Verify(flatbuffers::Verifier &verifier) const {
return VerifyTableStart(verifier) &&
VerifyOffset(verifier, VT_RAW_INPUTS) &&
verifier.VerifyVector(raw_inputs()) &&
verifier.VerifyVectorOfTables(raw_inputs()) &&
verifier.EndTable();
}
};
struct FullHistoryBlockBuilder {
typedef FullHistoryBlock Table;
flatbuffers::FlatBufferBuilder &fbb_;
flatbuffers::uoffset_t start_;
void add_raw_inputs(flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ledger::RawInput>>> raw_inputs) {
fbb_.AddOffset(FullHistoryBlock::VT_RAW_INPUTS, raw_inputs);
}
explicit FullHistoryBlockBuilder(flatbuffers::FlatBufferBuilder &_fbb)
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
flatbuffers::Offset<FullHistoryBlock> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<FullHistoryBlock>(end);
return o;
}
};
inline flatbuffers::Offset<FullHistoryBlock> CreateFullHistoryBlock(
flatbuffers::FlatBufferBuilder &_fbb,
flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ledger::RawInput>>> raw_inputs = 0) {
FullHistoryBlockBuilder builder_(_fbb);
builder_.add_raw_inputs(raw_inputs);
return builder_.Finish();
}
inline flatbuffers::Offset<FullHistoryBlock> CreateFullHistoryBlockDirect(
flatbuffers::FlatBufferBuilder &_fbb,
const std::vector<flatbuffers::Offset<msg::fbuf::ledger::RawInput>> *raw_inputs = nullptr) {
auto raw_inputs__ = raw_inputs ? _fbb.CreateVector<flatbuffers::Offset<msg::fbuf::ledger::RawInput>>(*raw_inputs) : 0;
return msg::fbuf::ledger::CreateFullHistoryBlock(
_fbb,
raw_inputs__);
}
struct RawInput FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
typedef RawInputBuilder Builder;
enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
VT_HASH = 4,
VT_PUBKEY = 6,
VT_INPUT = 8
};
const flatbuffers::Vector<uint8_t> *hash() const {
return GetPointer<const flatbuffers::Vector<uint8_t> *>(VT_HASH);
}
flatbuffers::Vector<uint8_t> *mutable_hash() {
return GetPointer<flatbuffers::Vector<uint8_t> *>(VT_HASH);
}
const flatbuffers::Vector<uint8_t> *pubkey() const {
return GetPointer<const flatbuffers::Vector<uint8_t> *>(VT_PUBKEY);
}
flatbuffers::Vector<uint8_t> *mutable_pubkey() {
return GetPointer<flatbuffers::Vector<uint8_t> *>(VT_PUBKEY);
}
const flatbuffers::Vector<uint8_t> *input() const {
return GetPointer<const flatbuffers::Vector<uint8_t> *>(VT_INPUT);
}
flatbuffers::Vector<uint8_t> *mutable_input() {
return GetPointer<flatbuffers::Vector<uint8_t> *>(VT_INPUT);
}
bool Verify(flatbuffers::Verifier &verifier) const {
return VerifyTableStart(verifier) &&
VerifyOffset(verifier, VT_HASH) &&
verifier.VerifyVector(hash()) &&
VerifyOffset(verifier, VT_PUBKEY) &&
verifier.VerifyVector(pubkey()) &&
VerifyOffset(verifier, VT_INPUT) &&
verifier.VerifyVector(input()) &&
verifier.EndTable();
}
};
struct RawInputBuilder {
typedef RawInput Table;
flatbuffers::FlatBufferBuilder &fbb_;
flatbuffers::uoffset_t start_;
void add_hash(flatbuffers::Offset<flatbuffers::Vector<uint8_t>> hash) {
fbb_.AddOffset(RawInput::VT_HASH, hash);
}
void add_pubkey(flatbuffers::Offset<flatbuffers::Vector<uint8_t>> pubkey) {
fbb_.AddOffset(RawInput::VT_PUBKEY, pubkey);
}
void add_input(flatbuffers::Offset<flatbuffers::Vector<uint8_t>> input) {
fbb_.AddOffset(RawInput::VT_INPUT, input);
}
explicit RawInputBuilder(flatbuffers::FlatBufferBuilder &_fbb)
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
flatbuffers::Offset<RawInput> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<RawInput>(end);
return o;
}
};
inline flatbuffers::Offset<RawInput> CreateRawInput(
flatbuffers::FlatBufferBuilder &_fbb,
flatbuffers::Offset<flatbuffers::Vector<uint8_t>> hash = 0,
flatbuffers::Offset<flatbuffers::Vector<uint8_t>> pubkey = 0,
flatbuffers::Offset<flatbuffers::Vector<uint8_t>> input = 0) {
RawInputBuilder builder_(_fbb);
builder_.add_input(input);
builder_.add_pubkey(pubkey);
builder_.add_hash(hash);
return builder_.Finish();
}
inline flatbuffers::Offset<RawInput> CreateRawInputDirect(
flatbuffers::FlatBufferBuilder &_fbb,
const std::vector<uint8_t> *hash = nullptr,
const std::vector<uint8_t> *pubkey = nullptr,
const std::vector<uint8_t> *input = nullptr) {
auto hash__ = hash ? _fbb.CreateVector<uint8_t>(*hash) : 0;
auto pubkey__ = pubkey ? _fbb.CreateVector<uint8_t>(*pubkey) : 0;
auto input__ = input ? _fbb.CreateVector<uint8_t>(*input) : 0;
return msg::fbuf::ledger::CreateRawInput(
_fbb,
hash__,
pubkey__,
input__);
}
inline const msg::fbuf::ledger::FullHistoryBlock *GetFullHistoryBlock(const void *buf) {
return flatbuffers::GetRoot<msg::fbuf::ledger::FullHistoryBlock>(buf);
}
inline const msg::fbuf::ledger::FullHistoryBlock *GetSizePrefixedFullHistoryBlock(const void *buf) {
return flatbuffers::GetSizePrefixedRoot<msg::fbuf::ledger::FullHistoryBlock>(buf);
}
inline FullHistoryBlock *GetMutableFullHistoryBlock(void *buf) {
return flatbuffers::GetMutableRoot<FullHistoryBlock>(buf);
}
inline bool VerifyFullHistoryBlockBuffer(
flatbuffers::Verifier &verifier) {
return verifier.VerifyBuffer<msg::fbuf::ledger::FullHistoryBlock>(nullptr);
}
inline bool VerifySizePrefixedFullHistoryBlockBuffer(
flatbuffers::Verifier &verifier) {
return verifier.VerifySizePrefixedBuffer<msg::fbuf::ledger::FullHistoryBlock>(nullptr);
}
inline void FinishFullHistoryBlockBuffer(
flatbuffers::FlatBufferBuilder &fbb,
flatbuffers::Offset<msg::fbuf::ledger::FullHistoryBlock> root) {
fbb.Finish(root);
}
inline void FinishSizePrefixedFullHistoryBlockBuffer(
flatbuffers::FlatBufferBuilder &fbb,
flatbuffers::Offset<msg::fbuf::ledger::FullHistoryBlock> root) {
fbb.FinishSizePrefixed(root);
}
} // namespace ledger
} // namespace fbuf
} // namespace msg
#endif // FLATBUFFERS_GENERATED_FULLHISTORYSCHEMA_MSG_FBUF_LEDGER_H_

41
src/msg/fbuf/ledger_helpers.cpp
View File

@@ -1,12 +1,12 @@
#include "../../pchheader.hpp"
#include "../../p2p/p2p.hpp"
#include "ledger_schema_generated.h"
#include "fullhistory_schema_generated.h"
#include "common_helpers.hpp"
#include "ledger_helpers.hpp"
namespace msg::fbuf::ledger
{
/**
* Create ledger block from the given proposal struct.
* @param p The proposal struct to be placed in ledger.
@@ -43,4 +43,43 @@ namespace msg::fbuf::ledger
return VerifyLedgerBlockBuffer(ledger_verifier);
}
/**
* Create full history block from the given raw input map.
* @param map The raw input map to be placed in full history.
*/
void create_full_history_block_from_raw_input_map(flatbuffers::FlatBufferBuilder &builder, const std::unordered_map<std::string, usr::raw_user_input> &map)
{
std::vector<flatbuffers::Offset<msg::fbuf::ledger::RawInput>> fbvec;
fbvec.reserve(map.size());
for (auto const &[key, value] : map)
{
fbvec.push_back(ledger::CreateRawInput(
builder,
sv_to_flatbuff_bytes(builder, key),
sv_to_flatbuff_bytes(builder, value.pubkey),
sv_to_flatbuff_bytes(builder, value.input)));
}
flatbuffers::Offset<ledger::FullHistoryBlock> fullhistory =
ledger::CreateFullHistoryBlock(
builder,
builder.CreateVector(fbvec));
builder.Finish(fullhistory); // Finished building message content to get serialised content.
}
const std::unordered_map<std::string, usr::raw_user_input> create_raw_input_map_from_full_history_block(const std::vector<uint8_t> &fullhist_buf)
{
const auto fullhistory = msg::fbuf::ledger::GetFullHistoryBlock(fullhist_buf.data());
const auto fbvec = fullhistory->raw_inputs();
std::unordered_map<std::string, usr::raw_user_input> map;
map.reserve(fbvec->size());
for (auto el : *fbvec)
{
usr::raw_user_input raw_user_input(flatbuff_bytes_to_sv(el->pubkey()), flatbuff_bytes_to_sv(el->input()));
map.emplace(flatbuff_bytes_to_sv(el->hash()), raw_user_input);
}
return map;
}
} // namespace msg::fbuf::ledger

3
src/msg/fbuf/ledger_helpers.hpp
View File

@@ -14,6 +14,9 @@ namespace msg::fbuf::ledger
bool verify_ledger_block_buffer(const uint8_t *ledger_buf_ptr, const size_t buf_len);
void create_full_history_block_from_raw_input_map(flatbuffers::FlatBufferBuilder &builder, const std::unordered_map<std::string, usr::raw_user_input> &map);
const std::unordered_map<std::string, usr::raw_user_input> create_raw_input_map_from_full_history_block(const std::vector<uint8_t> &fullhist_buf);
} // namespace msg::fbuf::ledger
#endif

5
src/msg/fbuf/ledger_schema.fbs
View File

@@ -12,9 +12,4 @@ table LedgerBlock {
outputs: [ByteArray];
}
table RawInputList {
hash:[ubyte];
inputs:[ByteArray];
}
root_type LedgerBlock;

75
src/msg/fbuf/ledger_schema_generated.h
View File

@@ -15,9 +15,6 @@ namespace ledger {
struct LedgerBlock;
struct LedgerBlockBuilder;
struct RawInputList;
struct RawInputListBuilder;
struct LedgerBlock FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
typedef LedgerBlockBuilder Builder;
enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
@@ -173,78 +170,6 @@ inline flatbuffers::Offset<LedgerBlock> CreateLedgerBlockDirect(
outputs__);
}
struct RawInputList FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
typedef RawInputListBuilder Builder;
enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
VT_HASH = 4,
VT_INPUTS = 6
};
const flatbuffers::Vector<uint8_t> *hash() const {
return GetPointer<const flatbuffers::Vector<uint8_t> *>(VT_HASH);
}
flatbuffers::Vector<uint8_t> *mutable_hash() {
return GetPointer<flatbuffers::Vector<uint8_t> *>(VT_HASH);
}
const flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ByteArray>> *inputs() const {
return GetPointer<const flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ByteArray>> *>(VT_INPUTS);
}
flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ByteArray>> *mutable_inputs() {
return GetPointer<flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ByteArray>> *>(VT_INPUTS);
}
bool Verify(flatbuffers::Verifier &verifier) const {
return VerifyTableStart(verifier) &&
VerifyOffset(verifier, VT_HASH) &&
verifier.VerifyVector(hash()) &&
VerifyOffset(verifier, VT_INPUTS) &&
verifier.VerifyVector(inputs()) &&
verifier.VerifyVectorOfTables(inputs()) &&
verifier.EndTable();
}
};
struct RawInputListBuilder {
typedef RawInputList Table;
flatbuffers::FlatBufferBuilder &fbb_;
flatbuffers::uoffset_t start_;
void add_hash(flatbuffers::Offset<flatbuffers::Vector<uint8_t>> hash) {
fbb_.AddOffset(RawInputList::VT_HASH, hash);
}
void add_inputs(flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ByteArray>>> inputs) {
fbb_.AddOffset(RawInputList::VT_INPUTS, inputs);
}
explicit RawInputListBuilder(flatbuffers::FlatBufferBuilder &_fbb)
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
flatbuffers::Offset<RawInputList> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<RawInputList>(end);
return o;
}
};
inline flatbuffers::Offset<RawInputList> CreateRawInputList(
flatbuffers::FlatBufferBuilder &_fbb,
flatbuffers::Offset<flatbuffers::Vector<uint8_t>> hash = 0,
flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ByteArray>>> inputs = 0) {
RawInputListBuilder builder_(_fbb);
builder_.add_inputs(inputs);
builder_.add_hash(hash);
return builder_.Finish();
}
inline flatbuffers::Offset<RawInputList> CreateRawInputListDirect(
flatbuffers::FlatBufferBuilder &_fbb,
const std::vector<uint8_t> *hash = nullptr,
const std::vector<flatbuffers::Offset<msg::fbuf::ByteArray>> *inputs = nullptr) {
auto hash__ = hash ? _fbb.CreateVector<uint8_t>(*hash) : 0;
auto inputs__ = inputs ? _fbb.CreateVector<flatbuffers::Offset<msg::fbuf::ByteArray>>(*inputs) : 0;
return msg::fbuf::ledger::CreateRawInputList(
_fbb,
hash__,
inputs__);
}
inline const msg::fbuf::ledger::LedgerBlock *GetLedgerBlock(const void *buf) {
return flatbuffers::GetRoot<msg::fbuf::ledger::LedgerBlock>(buf);
}

23
src/msg/fbuf/p2pmsg_content_generated.h
View File

@@ -324,7 +324,6 @@ struct Peer_Challenge_MessageBuilder {
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
Peer_Challenge_MessageBuilder &operator=(const Peer_Challenge_MessageBuilder &);
flatbuffers::Offset<Peer_Challenge_Message> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<Peer_Challenge_Message>(end);
@@ -391,7 +390,6 @@ struct Peer_Challenge_Response_MessageBuilder {
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
Peer_Challenge_Response_MessageBuilder &operator=(const Peer_Challenge_Response_MessageBuilder &);
flatbuffers::Offset<Peer_Challenge_Response_Message> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<Peer_Challenge_Response_Message>(end);
@@ -474,7 +472,6 @@ struct UserInputBuilder {
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
UserInputBuilder &operator=(const UserInputBuilder &);
flatbuffers::Offset<UserInput> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<UserInput>(end);
@@ -551,7 +548,6 @@ struct UserInputGroupBuilder {
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
UserInputGroupBuilder &operator=(const UserInputGroupBuilder &);
flatbuffers::Offset<UserInputGroup> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<UserInputGroup>(end);
@@ -711,7 +707,6 @@ struct ContentBuilder {
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
ContentBuilder &operator=(const ContentBuilder &);
flatbuffers::Offset<Content> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<Content>(end);
@@ -760,7 +755,6 @@ struct NonUnl_Proposal_MessageBuilder {
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
NonUnl_Proposal_MessageBuilder &operator=(const NonUnl_Proposal_MessageBuilder &);
flatbuffers::Offset<NonUnl_Proposal_Message> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<NonUnl_Proposal_Message>(end);
@@ -888,7 +882,6 @@ struct Proposal_MessageBuilder {
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
Proposal_MessageBuilder &operator=(const Proposal_MessageBuilder &);
flatbuffers::Offset<Proposal_Message> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<Proposal_Message>(end);
@@ -971,7 +964,6 @@ struct Npl_MessageBuilder {
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
Npl_MessageBuilder &operator=(const Npl_MessageBuilder &);
flatbuffers::Offset<Npl_Message> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<Npl_Message>(end);
@@ -1026,7 +1018,6 @@ struct History_Request_MessageBuilder {
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
History_Request_MessageBuilder &operator=(const History_Request_MessageBuilder &);
flatbuffers::Offset<History_Request_Message> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<History_Request_Message>(end);
@@ -1105,7 +1096,6 @@ struct History_Response_MessageBuilder {
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
History_Response_MessageBuilder &operator=(const History_Response_MessageBuilder &);
flatbuffers::Offset<History_Response_Message> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<History_Response_Message>(end);
@@ -1180,7 +1170,6 @@ struct HistoryLedgerBlockPairBuilder {
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
HistoryLedgerBlockPairBuilder &operator=(const HistoryLedgerBlockPairBuilder &);
flatbuffers::Offset<HistoryLedgerBlockPair> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<HistoryLedgerBlockPair>(end);
@@ -1240,7 +1229,6 @@ struct HistoryLedgerBlockBuilder {
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
HistoryLedgerBlockBuilder &operator=(const HistoryLedgerBlockBuilder &);
flatbuffers::Offset<HistoryLedgerBlock> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<HistoryLedgerBlock>(end);
@@ -1334,7 +1322,6 @@ struct State_Request_MessageBuilder {
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
State_Request_MessageBuilder &operator=(const State_Request_MessageBuilder &);
flatbuffers::Offset<State_Request_Message> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<State_Request_Message>(end);
@@ -1456,7 +1443,6 @@ struct State_Response_MessageBuilder {
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
State_Response_MessageBuilder &operator=(const State_Response_MessageBuilder &);
flatbuffers::Offset<State_Response_Message> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<State_Response_Message>(end);
@@ -1525,7 +1511,6 @@ struct Fs_Entry_ResponseBuilder {
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
Fs_Entry_ResponseBuilder &operator=(const Fs_Entry_ResponseBuilder &);
flatbuffers::Offset<Fs_Entry_Response> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<Fs_Entry_Response>(end);
@@ -1591,7 +1576,6 @@ struct File_HashMap_ResponseBuilder {
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
File_HashMap_ResponseBuilder &operator=(const File_HashMap_ResponseBuilder &);
flatbuffers::Offset<File_HashMap_Response> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<File_HashMap_Response>(end);
@@ -1661,7 +1645,6 @@ struct Block_ResponseBuilder {
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
Block_ResponseBuilder &operator=(const Block_ResponseBuilder &);
flatbuffers::Offset<Block_Response> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<Block_Response>(end);
@@ -1743,7 +1726,6 @@ struct State_FS_Hash_EntryBuilder {
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
State_FS_Hash_EntryBuilder &operator=(const State_FS_Hash_EntryBuilder &);
flatbuffers::Offset<State_FS_Hash_Entry> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<State_FS_Hash_Entry>(end);
@@ -1806,7 +1788,6 @@ struct Peer_Requirement_Announcement_MessageBuilder {
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
Peer_Requirement_Announcement_MessageBuilder &operator=(const Peer_Requirement_Announcement_MessageBuilder &);
flatbuffers::Offset<Peer_Requirement_Announcement_Message> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<Peer_Requirement_Announcement_Message>(end);
@@ -1862,7 +1843,6 @@ struct Available_Capacity_Announcement_MessageBuilder {
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
Available_Capacity_Announcement_MessageBuilder &operator=(const Available_Capacity_Announcement_MessageBuilder &);
flatbuffers::Offset<Available_Capacity_Announcement_Message> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<Available_Capacity_Announcement_Message>(end);
@@ -1896,7 +1876,6 @@ struct Peer_List_Request_MessageBuilder {
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
Peer_List_Request_MessageBuilder &operator=(const Peer_List_Request_MessageBuilder &);
flatbuffers::Offset<Peer_List_Request_Message> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<Peer_List_Request_Message>(end);
@@ -1941,7 +1920,6 @@ struct Peer_List_Response_MessageBuilder {
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
Peer_List_Response_MessageBuilder &operator=(const Peer_List_Response_MessageBuilder &);
flatbuffers::Offset<Peer_List_Response_Message> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<Peer_List_Response_Message>(end);
@@ -2029,7 +2007,6 @@ struct Peer_PropertiesBuilder {
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
Peer_PropertiesBuilder &operator=(const Peer_PropertiesBuilder &);
flatbuffers::Offset<Peer_Properties> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<Peer_Properties>(end);

39
src/usr/user_input.hpp
View File

@@ -7,25 +7,36 @@
namespace usr
{
/**
/**
* Represents a signed contract input message a network user has submitted.
*/
struct user_input
{
const std::string input_container;
const std::string sig;
const util::PROTOCOL protocol; // The encoding protocol used for the input container.
user_input(const std::string input_container, const std::string sig, const util::PROTOCOL protocol)
: input_container(std::move(input_container)), sig(std::move(sig)), protocol(protocol)
struct user_input
{
}
const std::string input_container;
const std::string sig;
const util::PROTOCOL protocol; // The encoding protocol used for the input container.
user_input(std::string_view input_container, std::string_view sig, const util::PROTOCOL protocol)
: input_container(input_container), sig(sig), protocol(protocol)
user_input(const std::string input_container, const std::string sig, const util::PROTOCOL protocol)
: input_container(std::move(input_container)), sig(std::move(sig)), protocol(protocol)
{
}
user_input(std::string_view input_container, std::string_view sig, const util::PROTOCOL protocol)
: input_container(input_container), sig(sig), protocol(protocol)
{
}
};
struct raw_user_input
{
}
};
const std::string pubkey;
const std::string input;
raw_user_input(std::string_view pubkey, std::string_view input)
: pubkey(pubkey), input(input)
{
}
};
} // namespace usr

17
src/util/buffer_store.cpp
View File

@@ -42,6 +42,23 @@ namespace util
}
}
/**
* Reads the string content from the given buffer_view.
* @param view The buffer_view that should be read.
* @param buf output string buffer.
* @return Returns number of bytes read. -1 on error.
*/
int buffer_store::read_buf(const buffer_view &view, std::string &buf)
{
buf.resize(view.size);
const int res = pread(fd, buf.data(), view.size, view.offset);
if (res < view.size)
{
LOG_ERROR << errno << ": Error reading from buffer store fd " << fd;
}
return res;
}
int buffer_store::purge(const buffer_view &buf)
{
const size_t purge_size = BLOCK_ALIGN(buf.size);

1
src/util/buffer_store.hpp
View File

@@ -26,6 +26,7 @@ namespace util
int fd;
int init();
const buffer_view write_buf(const void *buf, const uint32_t size);
int read_buf(const buffer_view &view, std::string &buf);
int purge(const buffer_view &buf);
void deinit();
};