ARCHIVE/hpcore
1
0
Fork 0
mirror of https://github.com/EvernodeXRPL/hpcore.git synced 2026-04-29 15:37:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

Introducing ledger shards and new ledger syncing logic. (#247)

Browse Source 
- The ledgers are stored in a sqlite database in ledger file system.
- Ledgers are organized in shards inside primary folder.
- Raw inputs are saved as shards inside blob folder. No input files are created if neither inputs nor outputs are available.
- Both last primary shard hash and last blob shard hashes are subjected to consensus and necessary sync operations are performed if out of sync.
- Hpfs sync support setting single sync targets from both ends of the list. (Targets set from front are prioritized).
- Contract and ledger syncs and serving are performed independently.
This commit is contained in:
Savinda Senevirathne
2021-02-18 11:24:05 +05:30
committed by GitHub
parent e394497698
commit 16c5b3fae2
53 changed files with 1917 additions and 2317 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
CMakeLists.txt
View File

@@ -61,12 +61,11 @@ add_executable(hpcore
src/usr/input_nonce_map.cpp
src/usr/usr.cpp
src/usr/read_req.cpp
src/ledger.cpp
src/ledger/sqlite.cpp
src/ledger/ledger_mount.cpp
src/ledger/ledger_sync.cpp
src/ledger/ledger_serve.cpp
src/ledger/ledger_sample.cpp
src/ledger/ledger.cpp
src/consensus.cpp
src/main.cpp
)

26
src/conf.cpp
View File

@@ -3,6 +3,8 @@
#include "crypto.hpp"
#include "sc/sc.hpp"
#include "util/util.hpp"
#include "ledger/ledger_mount.hpp"
#include "sc/contract_mount.hpp"
namespace conf
{
@@ -107,13 +109,11 @@ namespace conf
// Recursivly create contract directories. Return an error if unable to create
if (util::create_dir_tree_recursive(ctx.config_dir) == -1 ||
util::create_dir_tree_recursive(ctx.hist_dir) == -1 ||
util::create_dir_tree_recursive(ctx.full_hist_dir) == -1 ||
util::create_dir_tree_recursive(ctx.log_dir) == -1 ||
util::create_dir_tree_recursive(ctx.contract_hpfs_dir + "/seed" + hpfs::STATE_DIR_PATH) == -1 ||
util::create_dir_tree_recursive(ctx.contract_hpfs_dir + "/seed" + sc::STATE_DIR_PATH) == -1 ||
util::create_dir_tree_recursive(ctx.contract_hpfs_mount_dir) == -1 ||
util::create_dir_tree_recursive(ctx.ledger_hpfs_dir + "/seed" + hpfs::LEDGER_PRIMARY_DIR) == -1 ||
util::create_dir_tree_recursive(ctx.ledger_hpfs_dir + "/seed" + hpfs::LEDGER_BLOB_DIR) == -1 ||
util::create_dir_tree_recursive(ctx.ledger_hpfs_dir + "/seed" + ledger::PRIMARY_DIR) == -1 ||
util::create_dir_tree_recursive(ctx.ledger_hpfs_dir + "/seed" + ledger::BLOB_DIR) == -1 ||
util::create_dir_tree_recursive(ctx.ledger_hpfs_mount_dir) == -1 ||
util::create_dir_tree_recursive(ctx.contract_log_dir) == -1)
{
@@ -135,6 +135,7 @@ namespace conf
cfg.node.role = startup_role = ROLE::VALIDATOR;
cfg.node.full_history = false;
cfg.node.max_shards = 4;
cfg.contract.id = crypto::generate_uuid();
cfg.contract.execute = true;
@@ -218,8 +219,6 @@ namespace conf
ctx.config_file = ctx.config_dir + "/hp.cfg";
ctx.tls_key_file = ctx.config_dir + "/tlskey.pem";
ctx.tls_cert_file = ctx.config_dir + "/tlscert.pem";
ctx.hist_dir = basedir + "/hist";
ctx.full_hist_dir = basedir + "/fullhist";
ctx.contract_hpfs_dir = basedir + "/contract_fs";
ctx.contract_hpfs_mount_dir = ctx.contract_hpfs_dir + "/mnt";
ctx.contract_hpfs_rw_dir = ctx.contract_hpfs_mount_dir + "/rw";
@@ -324,6 +323,8 @@ namespace conf
return -1;
}
startup_role = cfg.node.role;
cfg.node.max_shards = node["max_shards"].as<uint64_t>();
}
catch (const std::exception &e)
{
@@ -482,7 +483,8 @@ namespace conf
node_config.insert_or_assign("public_key", cfg.node.public_key_hex);
node_config.insert_or_assign("private_key", cfg.node.private_key_hex);
// We always save the startup role to config. Not the current role which might get changed dynamically during syncing.
node_config.insert_or_assign("role", startup_role == ROLE::OBSERVER ? ROLE_OBSERVER : ROLE_VALIDATOR);
node_config.insert_or_assign("role", cfg.node.role == ROLE::OBSERVER ? ROLE_OBSERVER : ROLE_VALIDATOR);
node_config.insert_or_assign("max_shards", cfg.node.max_shards);
d.insert_or_assign("node", node_config);
}
@@ -642,11 +644,9 @@ namespace conf
*/
int validate_contract_dir_paths()
{
const std::string paths[10] = {
const std::string paths[8] = {
ctx.contract_dir,
ctx.config_file,
ctx.hist_dir,
ctx.full_hist_dir,
ctx.contract_hpfs_dir,
ctx.ledger_hpfs_dir,
ctx.tls_key_file,
@@ -732,7 +732,7 @@ namespace conf
jsoncons::ojson jdoc;
populate_contract_section_json(jdoc, cfg.contract, true);
const std::string patch_file_path = sc::contract_fs.physical_path(hpfs::RW_SESSION_NAME, hpfs::PATCH_FILE_PATH);
const std::string patch_file_path = sc::contract_fs.physical_path(hpfs::RW_SESSION_NAME, sc::PATCH_FILE_PATH);
return write_json_file(patch_file_path, jdoc);
}
@@ -744,7 +744,7 @@ namespace conf
*/
int apply_patch_config(std::string_view hpfs_session_name)
{
const std::string path = sc::contract_fs.physical_path(hpfs_session_name, hpfs::PATCH_FILE_PATH);
const std::string path = sc::contract_fs.physical_path(hpfs_session_name, sc::PATCH_FILE_PATH);
if (!util::is_file_exists(path))
return 0;

3
src/conf.hpp
View File

@@ -74,6 +74,7 @@ namespace conf
std::string public_key_hex; // Contract hex public key
std::string private_key_hex; // Contract hex private key
bool full_history = false; // Whether full history mode is on/off.
uint64_t max_shards = 0; // Maximum number of shards to store.
};
struct appbill_config
@@ -165,8 +166,6 @@ namespace conf
std::string hpfs_exe_path; // hpfs executable file path.
std::string contract_dir; // Contract base directory full path.
std::string full_hist_dir; // Contract full history dir full path.
std::string hist_dir; // Contract ledger history dir full path.
std::string contract_hpfs_dir; // Contract hpfs metdata dir (The location of hpfs log file).
std::string contract_hpfs_mount_dir; // Contract hpfs fuse file system mount path.
std::string contract_hpfs_rw_dir; // Contract hpfs read/write fs session path.

182
src/consensus.cpp
View File

@@ -12,7 +12,7 @@
#include "crypto.hpp"
#include "util/h32.hpp"
#include "unl.hpp"
#include "ledger.hpp"
#include "ledger/ledger.hpp"
#include "consensus.hpp"
namespace p2pmsg = msg::fbuf::p2pmsg;
@@ -111,8 +111,10 @@ namespace consensus
// Get current lcl and state.
std::string lcl = ledger::ctx.get_lcl();
const uint64_t lcl_seq_no = ledger::ctx.get_seq_no();
util::h32 state_hash = sc::contract_fs.get_parent_hash(hpfs::STATE_DIR_PATH);
util::h32 patch_hash = sc::contract_fs.get_parent_hash(hpfs::PATCH_FILE_PATH);
util::h32 state_hash = sc::contract_fs.get_parent_hash(sc::STATE_DIR_PATH);
const util::h32 patch_hash = sc::contract_fs.get_parent_hash(sc::PATCH_FILE_PATH);
const p2p::sequence_hash last_primary_shard_id = ledger::ctx.get_last_primary_shard_id();
const p2p::sequence_hash last_blob_shard_id = ledger::ctx.get_last_blob_shard_id();
if (ctx.stage == 0)
{
@@ -121,7 +123,7 @@ namespace consensus
if (verify_and_populate_candidate_user_inputs(lcl_seq_no) == -1)
return -1;
const p2p::proposal p = create_stage0_proposal(lcl, state_hash, patch_hash);
const p2p::proposal p = create_stage0_proposal(lcl, state_hash, patch_hash, last_primary_shard_id, last_blob_shard_id);
broadcast_proposal(p);
ctx.stage = 1; // Transition to next stage.
@@ -132,7 +134,7 @@ namespace consensus
const size_t unl_count = unl::count();
vote_counter votes;
const int sync_status = check_sync_status(lcl, unl_count, votes);
const int sync_status = check_sync_status(unl_count, votes);
if (sync_status == -2) // Unreliable votes.
{
@@ -151,7 +153,7 @@ namespace consensus
if (sync_status == 0)
{
// If we are in sync, vote and broadcast the winning votes to next stage.
const p2p::proposal p = create_stage123_proposal(votes, lcl, unl_count, state_hash, patch_hash);
const p2p::proposal p = create_stage123_proposal(votes, lcl, unl_count, state_hash, patch_hash, last_primary_shard_id, last_blob_shard_id);
broadcast_proposal(p);
// Upon successful consensus at stage 3, update the ledger and execute the contract using the consensus proposal.
@@ -180,24 +182,32 @@ namespace consensus
/**
* Checks whether we are in sync with the received votes.
* @return 0 if we are in sync. -1 on lcl or hpfs desync. -2 if majority lcl unreliable.
* @return 0 if we are in sync. -1 on ledger or hpfs desync. -2 if majority last ledger primary shard hash unreliable.
*/
int check_sync_status(std::string_view lcl, const size_t unl_count, vote_counter &votes)
int check_sync_status(const size_t unl_count, vote_counter &votes)
{
// Check if we're ahead/behind of consensus lcl.
bool is_lcl_desync = false;
std::string majority_lcl;
if (check_lcl_votes(is_lcl_desync, majority_lcl, votes, lcl, unl_count))
bool is_last_primary_shard_desync = false;
p2p::sequence_hash majority_primary_shard_id;
if (check_last_primary_shard_hash_votes(is_last_primary_shard_desync, majority_primary_shard_id, votes, unl_count))
{
// We proceed further only if lcl check was success (meaning lcl check could be reliably performed).
// State lcl sync if we are out-of-sync with majority lcl.
if (is_lcl_desync)
// We proceed further only if last primary shard hash check was success (meaning last primary shard hash check could be reliably performed).
// Last primary shard hash sync is commenced if we are out-of-sync with majority last primary shard hash.
if (is_last_primary_shard_desync)
{
conf::change_role(conf::ROLE::OBSERVER);
ledger::set_sync_target(majority_lcl);
// We first request the latest shard.
const std::string majority_shard_seq_no_str = std::to_string(majority_primary_shard_id.seq_no);
const std::string sync_name = "primary shard " + majority_shard_seq_no_str;
const std::string shard_path = std::string(ledger::PRIMARY_DIR).append("/").append(majority_shard_seq_no_str);
ledger::ledger_sync_worker.set_target_push_front(hpfs::sync_target{sync_name, majority_primary_shard_id.hash, shard_path, hpfs::BACKLOG_ITEM_TYPE::DIR});
}
// Check out blob shard hash with majority blob shard hash.
bool is_last_blob_shard_desync = false;
p2p::sequence_hash majority_blob_shard_id;
check_last_blob_shard_hash_votes(is_last_blob_shard_desync, majority_blob_shard_id, votes);
// Check our state with majority state.
bool is_state_desync = false;
bool is_patch_desync = false;
@@ -212,34 +222,39 @@ namespace consensus
is_patch_update_pending = false;
// Start hpfs sync if we are out-of-sync with majority hpfs patch hash or state hash.
if (is_state_desync || is_patch_desync)
if (is_state_desync || is_patch_desync || is_last_blob_shard_desync)
{
conf::change_role(conf::ROLE::OBSERVER);
// This queue holds all the sync targets which needs to get synced in contract fs.
std::queue<hpfs::sync_target> sync_target_list;
// Patch file sync is prioritized, Therefore it is set in the front of the sync target list.
if (is_patch_desync)
sync_target_list.push(hpfs::sync_target{"patch", majority_patch_hash, hpfs::PATCH_FILE_PATH, hpfs::BACKLOG_ITEM_TYPE::FILE});
sc::contract_sync_worker.set_target_push_front(hpfs::sync_target{"patch", majority_patch_hash, sc::PATCH_FILE_PATH, hpfs::BACKLOG_ITEM_TYPE::FILE});
if (is_state_desync)
sync_target_list.push(hpfs::sync_target{"state", majority_state_hash, hpfs::STATE_DIR_PATH, hpfs::BACKLOG_ITEM_TYPE::DIR});
sc::contract_sync_worker.set_target_push_back(hpfs::sync_target{"state", majority_state_hash, sc::STATE_DIR_PATH, hpfs::BACKLOG_ITEM_TYPE::DIR});
// Set sync targets for contract fs.
sc::contract_sync_worker.set_target(std::move(sync_target_list));
// If ledger blob shard is desync, We first request the latest blob shard.
if (is_last_blob_shard_desync)
{
const std::string majority_shard_seq_no_str = std::to_string(majority_blob_shard_id.seq_no);
const std::string sync_name = "blob shard " + majority_shard_seq_no_str;
const std::string shard_path = std::string(ledger::BLOB_DIR).append("/").append(majority_shard_seq_no_str);
ledger::ledger_sync_worker.set_target_push_back(hpfs::sync_target{sync_name, majority_blob_shard_id.hash, shard_path, hpfs::BACKLOG_ITEM_TYPE::DIR});
}
}
// Proceed further only if both lcl and state are in sync with majority.
if (!is_lcl_desync && !is_state_desync && !is_patch_desync)
// Proceed further only if last primary shard, last blob shard, state and patch hashes are in sync with majority.
if (!is_last_primary_shard_desync && !is_last_blob_shard_desync && !is_state_desync && !is_patch_desync)
{
conf::change_role(conf::ROLE::VALIDATOR);
return 0;
}
// lcl or hpfs desync.
// Last primary shard hash, last blob shard hash, patch or state desync.
return -1;
}
// Majority lcl couldn't be detected reliably.
// Majority last primary shard hash couldn't be detected reliably.
return -2;
}
@@ -249,7 +264,7 @@ namespace consensus
*/
void check_sync_completion()
{
if (conf::cfg.node.role == conf::ROLE::OBSERVER && !sc::contract_sync_worker.is_syncing && !ledger::sync_ctx.is_syncing)
if (conf::cfg.node.role == conf::ROLE::OBSERVER && !sc::contract_sync_worker.is_syncing && !ledger::ledger_sync_worker.is_syncing)
conf::change_role(conf::ROLE::VALIDATOR);
}
@@ -414,7 +429,9 @@ namespace consensus
<< " ts:" << std::to_string(p.time)
<< " lcl:" << p.lcl.substr(0, 15)
<< " state:" << p.state_hash
<< " patch:" << p.patch_hash;
<< " patch:" << p.patch_hash
<< " last_primary_shard_id:" << p.last_primary_shard_id
<< " last_blob_shard_id:" << p.last_blob_shard_id;
}
/**
@@ -528,7 +545,8 @@ namespace consensus
return 0;
}
p2p::proposal create_stage0_proposal(std::string_view lcl, util::h32 state_hash, util::h32 patch_hash)
p2p::proposal create_stage0_proposal(std::string_view lcl, const util::h32 &state_hash, const util::h32 &patch_hash,
const p2p::sequence_hash &last_primary_shard_id, const p2p::sequence_hash &last_blob_shard_id)
{
// This is the proposal that stage 0 votes on.
// We report our own values in stage 0.
@@ -538,6 +556,8 @@ namespace consensus
p.lcl = lcl;
p.state_hash = state_hash;
p.patch_hash = patch_hash;
p.last_primary_shard_id = last_primary_shard_id;
p.last_blob_shard_id = last_blob_shard_id;
crypto::random_bytes(p.nonce, ROUND_NONCE_SIZE);
// Populate the proposal with set of candidate user pubkeys.
@@ -554,13 +574,16 @@ namespace consensus
return p;
}
p2p::proposal create_stage123_proposal(vote_counter &votes, std::string_view lcl, const size_t unl_count, const util::h32 state_hash, const util::h32 patch_hash)
p2p::proposal create_stage123_proposal(vote_counter &votes, std::string_view lcl, const size_t unl_count, const util::h32 &state_hash, const util::h32 &patch_hash,
const p2p::sequence_hash &last_primary_shard_id, const p2p::sequence_hash &last_blob_shard_id)
{
// The proposal to be emited at the end of this stage.
p2p::proposal p;
p.stage = ctx.stage;
p.state_hash = state_hash;
p.patch_hash = patch_hash;
p.last_primary_shard_id = last_primary_shard_id;
p.last_blob_shard_id = last_blob_shard_id;
// We always vote for our current lcl and state regardless of what other peers are saying.
// If there's a fork condition we will either request history and hpfs state from
@@ -673,44 +696,44 @@ namespace consensus
}
/**
* Check whether our lcl is consistent with the proposals being made by our UNL peers lcl votes.
* @param is_desync Indicates whether our lcl is out-of-sync with majority lcl. Only valid if this method returns True.
* @param majority_lcl The majority lcl based on the votes received. Only valid if this method returns True.
* Check whether our last primary shard hash is consistent with the proposals being made by our UNL peers last primary shard hash votes.
* @param is_desync Indicates whether our ledger primary hash is out-of-sync with majority ledger primary hash. Only valid if this method returns True.
* @param majority_primary_shard_id Majority primary shard id.
* @param votes Vote counter for this stage.
* @param lcl Our lcl.
* @return True if majority lcl could be calculated reliably. False if lcl check failed due to unreliable votes.
* @param unl_count Number of unl peers.
* @return True if majority ledger primary hash could be calculated reliably. False if shard index hash check failed due to unreliable votes.
*/
bool check_lcl_votes(bool &is_desync, std::string &majority_lcl, vote_counter &votes, std::string_view lcl, const size_t unl_count)
bool check_last_primary_shard_hash_votes(bool &is_desync, p2p::sequence_hash &majority_primary_shard_id, vote_counter &votes, const size_t unl_count)
{
uint32_t total_lcl_votes = 0;
uint32_t total_ledger_primary_hash_votes = 0;
for (const auto &[pubkey, cp] : ctx.candidate_proposals)
{
increment(votes.lcl, cp.lcl);
total_lcl_votes++;
increment(votes.last_ledger_primary_shard, cp.last_primary_shard_id);
total_ledger_primary_hash_votes++;
}
// Check whether we have received enough votes in total.
const uint32_t min_required = ceil(MAJORITY_THRESHOLD * unl_count);
if (total_lcl_votes < min_required)
if (total_ledger_primary_hash_votes < min_required)
{
LOG_INFO << "Not enough peers proposing to perform consensus. votes:" << total_lcl_votes << " needed:" << min_required;
LOG_INFO << "Not enough peers proposing to perform consensus. votes:" << total_ledger_primary_hash_votes << " needed:" << min_required;
return false;
}
uint32_t winning_votes = 0;
for (const auto [lcl, votes] : votes.lcl)
for (const auto [shard_id, votes] : votes.last_ledger_primary_shard)
{
if (votes > winning_votes)
{
winning_votes = votes;
majority_lcl = lcl;
majority_primary_shard_id = shard_id;
}
}
// If winning lcl is not matched with our lcl, that means we are not on the consensus ledger.
// If that's the case we should request history straight away.
if (lcl != majority_lcl)
// If winning last primary shard hash is not matched with our last primary shard hash, that means we are not on the consensus ledger.
// If that's the case we should request shards straight away.
if (ledger::ctx.get_last_primary_shard_id() != majority_primary_shard_id)
{
LOG_DEBUG << "We are not on the consensus ledger, we must request history from a peer.";
is_desync = true;
@@ -718,22 +741,48 @@ namespace consensus
}
else
{
// Check wheher there are enough winning votes for the lcl to be reliable.
// Check wheher there are enough winning votes for the last shard to be reliable.
const uint32_t min_wins_required = ceil(MAJORITY_THRESHOLD * ctx.candidate_proposals.size());
if (winning_votes < min_wins_required)
{
LOG_INFO << "No consensus on lcl. Possible fork condition. won:" << winning_votes << " needed:" << min_wins_required;
LOG_INFO << "No consensus on last shard hash. Possible fork condition. won:" << winning_votes << " needed:" << min_wins_required;
return false;
}
else
{
// Reaching here means we have reliable amount of winning lcl votes and our lcl matches with majority lcl.
// Reaching here means we have reliable amount of winning last shard hash votes and our last shard hash matches with majority last shard hash.
is_desync = false;
return true;
}
}
}
/**
* Check whether our last blob shard hash is consistent with the proposals being made by our UNL peers last blob shard hash votes.
* @param is_ledger_blob_desync Indicates whether our ledger blob hash is out-of-sync with majority ledger blob hash.
* @param majority_primary_shard_id Majority primary shard id.
* @param votes Vote counter for this stage.
*/
void check_last_blob_shard_hash_votes(bool &is_ledger_blob_desync, p2p::sequence_hash &majority_blob_shard_id, vote_counter &votes)
{
for (const auto &[pubkey, cp] : ctx.candidate_proposals)
{
increment(votes.last_ledger_blob_shard, cp.last_blob_shard_id);
}
uint32_t winning_votes = 0;
for (const auto [shard_id, votes] : votes.last_ledger_blob_shard)
{
if (votes > winning_votes)
{
winning_votes = votes;
majority_blob_shard_id = shard_id;
}
}
is_ledger_blob_desync = (ledger::ctx.get_last_blob_shard_id() != majority_blob_shard_id);
}
/**
* Check state hash against the winning and canonical state hash.
* @param is_state_desync Flag to determine whether contract state is out of sync.
@@ -757,7 +806,7 @@ namespace consensus
}
}
is_state_desync = (sc::contract_fs.get_parent_hash(hpfs::STATE_DIR_PATH) != majority_state_hash);
is_state_desync = (sc::contract_fs.get_parent_hash(sc::STATE_DIR_PATH) != majority_state_hash);
}
/**
@@ -783,7 +832,7 @@ namespace consensus
}
}
is_patch_desync = (sc::contract_fs.get_parent_hash(hpfs::PATCH_FILE_PATH) != majority_patch_hash);
is_patch_desync = (sc::contract_fs.get_parent_hash(sc::PATCH_FILE_PATH) != majority_patch_hash);
}
/**
@@ -794,34 +843,13 @@ namespace consensus
*/
int update_ledger_and_execute_contract(const p2p::proposal &cons_prop, std::string &new_lcl, util::h32 &new_state_hash, const util::h32 &patch_hash)
{
// Map to temporarily store the raw inputs along with the hash.
std::unordered_map<std::string, usr::raw_user_input> raw_inputs;
// Add raw_inputs to the proposal if full history mode is on.
if (conf::cfg.node.full_history)
{
for (const auto &hash : cons_prop.input_hashes)
{
const auto itr = ctx.candidate_user_inputs.find(hash);
if (itr != ctx.candidate_user_inputs.end())
{
// Add raw_input to the map along with the input hash.
candidate_user_input &cand_input = itr->second;
// Taking the raw input string from the buffer_view.
std::string input;
if (usr::input_store.read_buf(cand_input.input, input) != -1)
raw_inputs.emplace(hash, usr::raw_user_input{cand_input.userpubkey, std::move(input)});
}
}
}
if (ledger::save_ledger(cons_prop, std::move(raw_inputs)) == -1)
if (ledger::save_ledger(cons_prop, ctx.candidate_user_inputs, ctx.generated_user_outputs) == -1)
return -1;
new_lcl = ledger::ctx.get_lcl();
const uint64_t new_lcl_seq_no = ledger::ctx.get_seq_no();
LOG_INFO << "****Ledger created**** (lcl:" << new_lcl.substr(0, 15) << " state:" << cons_prop.state_hash << " patch:" << cons_prop.patch_hash << ")";
LOG_INFO << "****Ledger created**** (lcl:" << new_lcl.substr(0, 15) << " lps:" << cons_prop.last_primary_shard_id << " lbs:" << cons_prop.last_blob_shard_id << " state:" << cons_prop.state_hash << " patch:" << cons_prop.patch_hash << ")";
// Apply consensed patch file changes to the hpcore runtime and hp.cfg.
if (apply_consensed_patch_file_changes(cons_prop.patch_hash, patch_hash) == -1)
@@ -867,7 +895,7 @@ namespace consensus
}
// Update state hash in contract fs global hash tracker.
sc::contract_fs.set_parent_hash(hpfs::STATE_DIR_PATH, args.post_execution_state_hash);
sc::contract_fs.set_parent_hash(sc::STATE_DIR_PATH, args.post_execution_state_hash);
new_state_hash = args.post_execution_state_hash;
extract_user_outputs_from_contract_bufmap(args.userbufs);

14
src/consensus.hpp
View File

@@ -95,6 +95,8 @@ namespace consensus
std::map<std::string, uint32_t> output_hash;
std::map<util::h32, uint32_t> state_hash;
std::map<util::h32, uint32_t> patch_hash;
std::map<p2p::sequence_hash, uint32_t> last_ledger_primary_shard;
std::map<p2p::sequence_hash, uint32_t> last_ledger_blob_shard;
};
extern std::atomic<bool> is_patch_update_pending; // Keep track whether the patch file is changed by the SC and is not yet applied to runtime.
@@ -109,7 +111,7 @@ namespace consensus
int consensus();
int check_sync_status(std::string_view lcl, const size_t unl_count, vote_counter &votes);
int check_sync_status(const size_t unl_count, vote_counter &votes);
void check_sync_completion();
@@ -123,13 +125,17 @@ namespace consensus
int verify_and_populate_candidate_user_inputs(const uint64_t lcl_seq_no);
p2p::proposal create_stage0_proposal(std::string_view lcl, util::h32 state_hash, util::h32 patch_hash);
p2p::proposal create_stage0_proposal(std::string_view lcl, const util::h32 &state_hash, const util::h32 &patch_hash,
const p2p::sequence_hash &last_primary_shard_id, const p2p::sequence_hash &last_blob_shard_id);
p2p::proposal create_stage123_proposal(vote_counter &votes, std::string_view lcl, const size_t unl_count, const util::h32 state_hash, const util::h32 patch_hash);
p2p::proposal create_stage123_proposal(vote_counter &votes, std::string_view lcl, const size_t unl_count, const util::h32 &state_hash, const util::h32 &patch_hash,
const p2p::sequence_hash &last_primary_shard_id, const p2p::sequence_hash &last_blob_shard_id);
void broadcast_proposal(const p2p::proposal &p);
bool check_lcl_votes(bool &is_desync, std::string &majority_lcl, vote_counter &votes, std::string_view lcl, const size_t unl_count);
bool check_last_primary_shard_hash_votes(bool &is_desync, p2p::sequence_hash &majority_primary_shard_id, vote_counter &votes, const size_t unl_count);
void check_last_blob_shard_hash_votes(bool &is_ledger_blob_desync, p2p::sequence_hash &majority_blob_shard_id, vote_counter &votes);
void check_state_votes(bool &is_state_desync, util::h32 &majority_state_hash, vote_counter &votes);

8
src/hpfs/hpfs_mount.hpp
View File

@@ -7,12 +7,8 @@
namespace hpfs
{
constexpr size_t BLOCK_SIZE = 4 * 1024 * 1024; // 4MB;
constexpr const char *RW_SESSION_NAME = "rw"; // The built-in session name used by hpfs for RW sessions.
constexpr const char *STATE_DIR_PATH = "/state"; // State directory name.
constexpr const char *PATCH_FILE_PATH = "/patch.cfg"; // Config patch filename.
constexpr const char *LEDGER_PRIMARY_DIR = "/primary"; // Ledger primary directory name.
constexpr const char *LEDGER_BLOB_DIR = "/blob"; // Ledger blob directory name.
constexpr size_t BLOCK_SIZE = 4 * 1024 * 1024; // 4MB;
constexpr const char *RW_SESSION_NAME = "rw"; // The built-in session name used by hpfs for RW sessions.
struct child_hash_node
{

27
src/hpfs/hpfs_serve.cpp
View File

@@ -5,7 +5,7 @@
#include "../msg/fbuf/p2pmsg_content_generated.h"
#include "../msg/fbuf/p2pmsg_helpers.hpp"
#include "../msg/fbuf/common_helpers.hpp"
#include "../ledger.hpp"
#include "../ledger/ledger.hpp"
#include "../hplog.hpp"
#include "hpfs_serve.hpp"
#include "hpfs_sync.hpp"
@@ -21,17 +21,17 @@ namespace hpfs
constexpr const char *HPFS_SESSION_NAME = "rw";
/**
* @param name The name of the serving instance. (For identification purpose)
* @param fs_mount The pointer to the relavent hpfs mount instance this server is serving.
* @param server_name The name of the serving instance. (For identification purpose)
* @param fs_mount_ptr The pointer to the relavent hpfs mount instance this server is serving.
* @return This returns -1 on error and 0 on success.
*/
int hpfs_serve::init(std::string_view name, hpfs::hpfs_mount *fs_mount)
int hpfs_serve::init(std::string_view server_name, hpfs::hpfs_mount *fs_mount_ptr)
{
if (fs_mount == NULL)
if (fs_mount_ptr == NULL)
return -1;
this->name = name;
this->fs_mount = fs_mount;
name = server_name;
fs_mount = fs_mount_ptr;
hpfs_serve_thread = std::thread(&hpfs_serve::hpfs_serve_loop, this);
init_success = true;
@@ -70,6 +70,7 @@ namespace hpfs
prev_requests_processed = !hpfs_requests.empty();
const uint64_t time_start = util::get_epoch_milliseconds();
const std::string lcl = ledger::ctx.get_lcl();
const p2p::sequence_hash last_primary_shard_id = ledger::ctx.get_last_primary_shard_id();
const uint32_t request_batch_timeout = hpfs::get_request_resubmit_timeout() * 0.9;
if (hpfs_requests.empty())
@@ -95,7 +96,7 @@ namespace hpfs
LOG_DEBUG << "Serving hpfs request from [" << util::to_hex(session_id).substr(2, 10) << "]";
flatbuffers::FlatBufferBuilder fbuf(1024);
if (hpfs_serve::create_hpfs_response(fbuf, hr, lcl) == 1)
if (hpfs_serve::create_hpfs_response(fbuf, hr, lcl, last_primary_shard_id) == 1)
{
// Find the peer that we should send the hpfs response to.
std::scoped_lock<std::mutex> lock(p2p::ctx.peer_connections_mutex);
@@ -124,10 +125,12 @@ namespace hpfs
* Creates the reply message for a given hpfs request.
* @param fbuf The flatbuffer builder to construct the reply message.
* @param hr The hpfs request which should be replied to.
* @param lcl Our lcl.
* @param last_primary_shard_id Last primary shard id.
* @return 1 if successful hpfs response was generated. 0 if request is invalid
* and no response was generated. -1 on error.
*/
int hpfs_serve::create_hpfs_response(flatbuffers::FlatBufferBuilder &fbuf, const p2p::hpfs_request &hr, std::string_view lcl)
int hpfs_serve::create_hpfs_response(flatbuffers::FlatBufferBuilder &fbuf, const p2p::hpfs_request &hr, std::string_view lcl, const p2p::sequence_hash &last_primary_shard_id)
{
LOG_DEBUG << "Serving hpfs req. path:" << hr.parent_path << " block_id:" << hr.block_id;
@@ -152,7 +155,7 @@ namespace hpfs
resp.hash = hr.expected_hash;
resp.data = std::string_view(reinterpret_cast<const char *>(block.data()), block.size());
msg::fbuf::p2pmsg::create_msg_from_block_response(fbuf, resp, fs_mount->mount_id, lcl);
msg::fbuf::p2pmsg::create_msg_from_block_response(fbuf, resp, fs_mount->mount_id, lcl, last_primary_shard_id);
return 1; // Success.
}
}
@@ -174,7 +177,7 @@ namespace hpfs
{
msg::fbuf::p2pmsg::create_msg_from_filehashmap_response(
fbuf, hr.parent_path, fs_mount->mount_id, block_hashes,
file_length, hr.expected_hash, lcl);
file_length, hr.expected_hash, lcl, last_primary_shard_id);
return 1; // Success.
}
}
@@ -193,7 +196,7 @@ namespace hpfs
else if (result == 1)
{
msg::fbuf::p2pmsg::create_msg_from_fsentry_response(
fbuf, hr.parent_path, fs_mount->mount_id, child_hash_nodes, hr.expected_hash, lcl);
fbuf, hr.parent_path, fs_mount->mount_id, child_hash_nodes, hr.expected_hash, lcl, last_primary_shard_id);
return 1; // Success.
}
}

4
src/hpfs/hpfs_serve.hpp
View File

@@ -24,11 +24,11 @@ namespace hpfs
virtual void swap_collected_requests() = 0; // Must override in child classes.
public:
int init(std::string_view name, hpfs::hpfs_mount *fs_mount);
int init(std::string_view server_name, hpfs::hpfs_mount *fs_mount_ptr);
void deinit();
int create_hpfs_response(flatbuffers::FlatBufferBuilder &fbuf, const p2p::hpfs_request &sr, std::string_view lcl);
int create_hpfs_response(flatbuffers::FlatBufferBuilder &fbuf, const p2p::hpfs_request &hr, std::string_view lcl, const p2p::sequence_hash &last_primary_shard_id);
int get_data_block(std::vector<uint8_t> &block, const std::string_view vpath,
const uint32_t block_id, const util::h32 expected_hash);

136
src/hpfs/hpfs_sync.cpp
View File

@@ -3,7 +3,7 @@
#include "../msg/fbuf/common_helpers.hpp"
#include "../p2p/p2p.hpp"
#include "../pchheader.hpp"
#include "../ledger.hpp"
#include "../ledger/ledger.hpp"
#include "../hplog.hpp"
#include "../util/util.hpp"
#include "../util/h32.hpp"
@@ -30,13 +30,13 @@ namespace hpfs
/**
* This should be called to activate the hpfs sync.
*/
int hpfs_sync::init(std::string_view name, hpfs::hpfs_mount *fs_mount)
int hpfs_sync::init(std::string_view worker_name, hpfs::hpfs_mount *fs_mount_ptr)
{
if (fs_mount == NULL)
if (fs_mount_ptr == NULL)
return -1;
this->name = name;
this->fs_mount = fs_mount;
name = worker_name;
fs_mount = fs_mount_ptr;
hpfs_sync_thread = std::thread(&hpfs_sync::hpfs_syncer_loop, this);
init_success = true;
return 0;
@@ -58,25 +58,71 @@ namespace hpfs
/**
* Sets a list of sync targets. Sync finishes when all the targets are synced.
* Syncing happens sequentially.
* @param target_list List of sync targets to sync towards.
* @param sync_target_list List of sync targets to sync towards.
*/
void hpfs_sync::set_target(const std::queue<sync_target> &target_list)
void hpfs_sync::set_target(const std::list<sync_target> &sync_target_list)
{
if (target_list.empty())
if (sync_target_list.empty())
return;
// Do not do anything if we are already syncing towards the specified target states.
if (is_shutting_down || (is_syncing && original_target_list == target_list))
if (is_shutting_down || (is_syncing && original_target_list == sync_target_list))
return;
this->original_target_list = target_list;
this->target_list = std::move(target_list);
original_target_list = sync_target_list;
target_list = std::move(sync_target_list);
std::unique_lock lock(current_target_mutex);
current_target = target_list.front(); // Make the first element of the list the first target to sync.
is_syncing = true;
}
/**
* This sets a prioritized sync target. This target will replace current sync target.
* This target will immediately starting to sync and the interupted sync will resume
* once this sync target is acheived.
*/
void hpfs_sync::set_target_push_front(const sync_target &target)
{
{
std::shared_lock lock(current_target_mutex);
if (is_shutting_down || (is_syncing && current_target == target))
return;
}
target_list.push_front(target);
is_syncing = true;
std::unique_lock lock(current_target_mutex);
// Make the first element of the list the first target to sync.
current_target = target_list.front();
}
/**
* Adds a new target to the syncing list. If the list was previously empty, current target
* will be updated and syncing will start.
*/
void hpfs_sync::set_target_push_back(const sync_target &target)
{
// Current_target_mutex is not required since this function is currently used in a unique_lock
// scope.
if (is_shutting_down || (is_syncing && current_target == target))
return;
// Check whether this target is already in the sync target list.
const auto itr = std::find(target_list.begin(), target_list.end(), target);
if (itr != target_list.end())
return;
target_list.push_back(target);
if (!is_syncing)
{
std::unique_lock lock(current_target_mutex);
// Make the first element of the list the first target to sync.
current_target = target_list.front();
is_syncing = true;
}
}
/**
* Runs the hpfs sync worker loop.
*/
@@ -95,6 +141,7 @@ namespace hpfs
if (!is_syncing)
continue;
bool is_sync_complete = false;
if (fs_mount->acquire_rw_session() != -1)
{
while (!is_shutting_down)
@@ -114,12 +161,12 @@ namespace hpfs
break;
{
std::shared_lock lock(current_target_mutex);
std::unique_lock lock(current_target_mutex);
if (new_state == current_target.hash)
{
LOG_INFO << "Hpfs " << name << " sync: Target " << current_target.name << " hash achieved: " << new_state;
on_current_sync_state_acheived(new_state);
on_current_sync_state_acheived(current_target);
// Start syncing to next target.
const int result = start_syncing_next_target();
@@ -130,14 +177,13 @@ namespace hpfs
}
else
{
LOG_INFO << "Hpfs " << name << " sync: Continuing sync for new target: " << current_target.hash;
LOG_INFO << "Hpfs " << name << " sync: Continuing sync for new " << current_target.name << " hash: " << current_target.hash;
continue;
}
}
}
LOG_INFO << "Hpfs " << name << " sync: All parents synced.";
fs_mount->release_rw_session();
is_sync_complete = true;
}
else
{
@@ -147,6 +193,10 @@ namespace hpfs
target_list = {};
original_target_list = {};
is_syncing = false;
const sync_target last_sync_target = current_target;
current_target = {};
if (is_sync_complete)
on_sync_complete(last_sync_target);
}
LOG_INFO << "Hpfs " << name << " sync: Worker stopped.";
@@ -160,9 +210,10 @@ namespace hpfs
int hpfs_sync::request_loop(const util::h32 current_target_hash, util::h32 &updated_state)
{
std::string lcl = ledger::ctx.get_lcl();
p2p::sequence_hash last_primary_shard_id = ledger::ctx.get_last_primary_shard_id();
// Send the initial root hpfs request of the current target.
submit_request(backlog_item{this->current_target.item_type, this->current_target.vpath, -1, current_target_hash}, lcl);
submit_request(backlog_item{current_target.item_type, current_target.vpath, -1, current_target_hash}, lcl, last_primary_shard_id);
// Indicates whether any responses were processed in the previous loop iteration.
bool prev_responses_processed = false;
@@ -178,6 +229,8 @@ namespace hpfs
// Get current lcl.
std::string lcl = ledger::ctx.get_lcl();
// Get the current last shard information.
last_primary_shard_id = ledger::ctx.get_last_primary_shard_id();
// Move the received hpfs responses to the local response list.
swap_collected_responses();
@@ -270,14 +323,14 @@ namespace hpfs
// After handling each response, check whether we have reached target hpfs state.
// get_hash returns 0 incase target parent is not existing in our side.
if (fs_mount->get_hash(updated_state, hpfs::RW_SESSION_NAME, this->current_target.vpath) == -1)
if (fs_mount->get_hash(updated_state, hpfs::RW_SESSION_NAME, current_target.vpath) == -1)
{
LOG_ERROR << "Hpfs " << name << " sync: exiting due to hash check error.";
return -1;
}
// Update the central hpfs state tracker.
fs_mount->set_parent_hash(this->current_target.vpath, updated_state);
fs_mount->set_parent_hash(current_target.vpath, updated_state);
LOG_DEBUG << "Hpfs " << name << " sync: current:" << updated_state << " | target:" << current_target_hash;
if (updated_state == current_target_hash)
@@ -306,17 +359,21 @@ namespace hpfs
{
LOG_INFO << "Hpfs " << name << " sync: Resubmission threshold exceeded. Abandoning sync.";
std::shared_lock lock(current_target_mutex);
std::unique_lock lock(current_target_mutex);
const int result = start_syncing_next_target();
if (result == 0)
{
current_target = {};
on_sync_abandoned();
return 1; // To stop syncing since we have sync all the targets.
}
return 0;
}
// Reset the counter and re-submit request.
request.waiting_time = 0;
LOG_DEBUG << "Hpfs " << name << " sync: Resubmitting request...";
submit_request(request, lcl);
submit_request(request, lcl, last_primary_shard_id);
}
}
@@ -330,7 +387,7 @@ namespace hpfs
return 0;
const backlog_item &request = pending_requests.front();
submit_request(request, lcl);
submit_request(request, lcl, last_primary_shard_id);
pending_requests.pop_front();
}
}
@@ -414,7 +471,7 @@ namespace hpfs
// Stop request loop if the target has changed.
std::shared_lock lock(current_target_mutex);
return current_target_hash != this->current_target.hash;
return current_target_hash != current_target.hash;
}
/**
@@ -423,10 +480,11 @@ namespace hpfs
* @param is_file Whether the requested path if a file or dir.
* @param block_id The requested block id. Only relevant if requesting a file block. Otherwise -1.
* @param expected_hash The expected hash of the requested data. The peer will ignore the request if their hash is different.
* @param last_primary_shard_id The last primary shard id.
* @param target_pubkey The peer pubkey the request was submitted to.
*/
void hpfs_sync::request_state_from_peer(const std::string &path, const bool is_file, const int32_t block_id,
const util::h32 expected_hash, std::string_view lcl, std::string &target_pubkey)
const util::h32 expected_hash, std::string_view lcl, const p2p::sequence_hash &last_primary_shard_id, std::string &target_pubkey)
{
p2p::hpfs_request hr;
hr.parent_path = path;
@@ -436,14 +494,14 @@ namespace hpfs
hr.mount_id = fs_mount->mount_id;
flatbuffers::FlatBufferBuilder fbuf(1024);
msg::fbuf::p2pmsg::create_msg_from_hpfs_request(fbuf, hr, lcl);
msg::fbuf::p2pmsg::create_msg_from_hpfs_request(fbuf, hr, lcl, last_primary_shard_id);
p2p::send_message_to_random_peer(fbuf, target_pubkey); //todo: send to a node that hold the majority hpfs state to improve reliability of retrieving hpfs state.
}
/**
* Submits a pending hpfs request to the peer.
*/
void hpfs_sync::submit_request(const backlog_item &request, std::string_view lcl)
void hpfs_sync::submit_request(const backlog_item &request, std::string_view lcl, const p2p::sequence_hash &last_primary_shard_id)
{
const std::string key = std::string(request.path)
.append(reinterpret_cast<const char *>(&request.expected_hash), sizeof(util::h32));
@@ -451,7 +509,7 @@ namespace hpfs
const bool is_file = request.type != BACKLOG_ITEM_TYPE::DIR;
std::string target_pubkey;
request_state_from_peer(request.path, is_file, request.block_id, request.expected_hash, lcl, target_pubkey);
request_state_from_peer(request.path, is_file, request.block_id, request.expected_hash, lcl, last_primary_shard_id, target_pubkey);
if (!target_pubkey.empty())
LOG_DEBUG << "Hpfs " << name << " sync: Requesting from [" << target_pubkey.substr(2, 10) << "]. type:" << request.type
@@ -608,23 +666,39 @@ namespace hpfs
}
/**
* This method can be used to invoke mount specific custom logic (after extending this super class) to be executed after
* This method can be used to invoke mount specific custom logic (after overriding this method) to be executed after
* a sync target is acheived.
*/
void hpfs_sync::on_current_sync_state_acheived(const util::h32 &acheived_hash)
void hpfs_sync::on_current_sync_state_acheived(const sync_target &synced_target)
{
}
/**
* This method can be used to invoke mount specific custom logic (after overriding this method) to be executed after
* a sync is abondened.
*/
void hpfs_sync::on_sync_abandoned()
{
}
/**
* This method can be used to invoke mount specific custom logic (after overriding this method) to be executed after
* a full sync is complete.
*/
void hpfs_sync::on_sync_complete(const sync_target &last_sync_target)
{
LOG_INFO << "Hpfs " << name << " sync: All parents synced.";
}
/**
* Starts syncing next target if available after current target finishes.
* @return returns 0 when the full sync is complete and 1 when more sync targets are available.
*/
int hpfs_sync::start_syncing_next_target()
{
target_list.pop(); // Remove the synced parent from the target list.
target_list.pop_front(); // Remove the synced parent from the target list.
if (target_list.empty())
{
current_target = {};
return 0;
}
else

27
src/hpfs/hpfs_sync.hpp
View File

@@ -40,7 +40,7 @@ namespace hpfs
bool operator==(const sync_target &target) const
{
return this->hash == target.hash;
return this->vpath == target.vpath && this->hash == target.hash;
}
};
@@ -50,11 +50,12 @@ namespace hpfs
bool init_success = false;
std::string name; // Name used for logging.
std::queue<sync_target> target_list; // The current target hashes we are syncing towards.
sync_target current_target = {};
std::list<sync_target> target_list; // The current target hashes we are syncing towards.
// Store the originally submitted sync target list. This list is used to avoid submitting same list multiple times
// because target list is updated when the sync targets are acheived.
std::queue<sync_target> original_target_list;
std::list<sync_target> original_target_list;
std::list<backlog_item> pending_requests; // List of pending sync requests to be sent out.
@@ -72,14 +73,16 @@ namespace hpfs
int start_syncing_next_target();
protected:
sync_target current_target = {};
// List of sender pubkeys and hpfs responses(flatbuffer messages) to be processed.
std::list<std::pair<std::string, std::string>> candidate_hpfs_responses;
hpfs::hpfs_mount *fs_mount = NULL;
virtual void on_current_sync_state_acheived(const util::h32 &acheived_hash);
virtual void on_current_sync_state_acheived(const sync_target &synced_target);
virtual void on_sync_abandoned();
virtual void on_sync_complete(const sync_target &last_sync_target);
// Move the collected responses from hpfs responses to a local response list.
virtual void swap_collected_responses() = 0; // Must override in child classes.
@@ -87,11 +90,15 @@ namespace hpfs
public:
std::atomic<bool> is_syncing = false;
int init(std::string_view name, hpfs::hpfs_mount *fs_mount);
int init(std::string_view worker_name, hpfs::hpfs_mount *fs_mount_ptr);
void deinit();
void set_target(const std::queue<sync_target> &target_list);
void set_target(const std::list<sync_target> &sync_target_list);
void set_target_push_front(const sync_target &target);
void set_target_push_back(const sync_target &target);
bool validate_fs_entry_hash(std::string_view vpath, std::string_view hash, const std::unordered_map<std::string, p2p::hpfs_fs_hash_entry> &fs_entry_map);
@@ -102,9 +109,9 @@ namespace hpfs
bool should_stop_request_loop(const util::h32 &current_target_hash);
void request_state_from_peer(const std::string &path, const bool is_file, const int32_t block_id,
const util::h32 expected_hash, std::string_view lcl, std::string &target_pubkey);
const util::h32 expected_hash, std::string_view lcl, const p2p::sequence_hash &last_primary_shard_id, std::string &target_pubkey);
void submit_request(const backlog_item &request, std::string_view lcl);
void submit_request(const backlog_item &request, std::string_view lcl, const p2p::sequence_hash &last_primary_shard_id);
int handle_fs_entry_response(std::string_view vpath, std::unordered_map<std::string, p2p::hpfs_fs_hash_entry> &fs_entry_map);

2
src/hplog.cpp
View File

@@ -72,7 +72,7 @@ namespace hplog
const std::string trace_file = conf::ctx.log_dir + "/hp.log";
static plog::RollingFileAppender<plog_formatter> fileAppender(trace_file.c_str(), conf::cfg.log.max_mbytes_per_file * 1024 * 1024, conf::cfg.log.max_file_count);
static plog::ConsoleAppender<plog_formatter> consoleAppender;
static plog::ColorConsoleAppender<plog_formatter> consoleAppender;
plog::Logger<0> &logger = plog::init(level);

903
src/ledger.cpp
View File

@@ -1,903 +0,0 @@
#include "pchheader.hpp"
#include "conf.hpp"
#include "crypto.hpp"
#include "p2p/p2p.hpp"
#include "msg/fbuf/common_helpers.hpp"
#include "msg/fbuf/ledger_helpers.hpp"
#include "msg/fbuf/p2pmsg_helpers.hpp"
#include "hplog.hpp"
#include "ledger.hpp"
#include "ledger/ledger_sample.hpp"
namespace p2pmsg = msg::fbuf::p2pmsg;
namespace ledger
{
constexpr int FILE_PERMS = 0644;
constexpr uint64_t MAX_LEDGER_SEQUENCE = 256; // Max ledger block count to keep.
constexpr uint16_t SYNCER_IDLE_WAIT = 20; // lcl syncer loop sleep time (milliseconds).
// Max no. of repetitive reqeust resubmissions before abandoning the sync.
constexpr uint16_t ABANDON_THRESHOLD = 10;
// No. of milliseconds to wait before resubmitting a request.
uint16_t REQUEST_RESUBMIT_TIMEOUT;
ledger_context ctx;
sync_context sync_ctx;
bool init_success = false;
/**
* Retrieve ledger history information from persisted ledgers.
*/
int init()
{
REQUEST_RESUBMIT_TIMEOUT = conf::cfg.contract.roundtime;
// Filename list of the history folder.
std::list<std::string> sorted_folder_entries = util::fetch_dir_entries(conf::ctx.hist_dir);
// Sorting to make filenames in seq_no order.
if (sort_lcl_filenames_and_validate(sorted_folder_entries) == -1)
{
return -1;
}
std::string previous_block_lcl;
uint64_t previous_block_seq_no;
// Get all records at lcl history direcory and find the last closed ledger.
for (const auto &entry : sorted_folder_entries)
{
const std::string file_path = conf::ctx.hist_dir + "/" + entry;
if (util::is_dir_exists(file_path))
{
LOG_ERROR << "Found directory " << entry << " in " << conf::ctx.hist_dir << ". There should be no folders in this directory.";
return -1;
}
else
{
const std::string file_name(util::remove_file_extension(entry));
uint64_t seq_no;
std::string hash;
if (extract_lcl(file_name, seq_no, hash) != -1)
{
std::vector<uint8_t> buffer;
if (read_ledger(file_path, buffer) == -1)
return -1;
if (!msg::fbuf::ledger::verify_ledger_block_buffer(buffer.data(), buffer.size()))
{
LOG_ERROR << "Ledger data verification failed. " << file_name;
return -1;
}
if (!check_block_integrity(hash, buffer))
{
LOG_ERROR << "Ledger block integrity check failed. " << file_name;
return -1;
}
// Ledger integrity check.
if (!previous_block_lcl.empty())
{
const p2p::proposal proposal = msg::fbuf::ledger::create_proposal_from_ledger_block(buffer);
if ((seq_no - previous_block_seq_no != 1) && (previous_block_lcl != proposal.lcl))
{
LOG_ERROR << "Ledger block chain-link verification failed. " << file_name;
return -1;
}
}
previous_block_lcl = file_name;
previous_block_seq_no = seq_no;
ctx.cache.emplace(seq_no, std::move(file_name)); // cache -> [seq_no - hash]
}
else
{
// lcl records should follow [ledger sequnce numer]-lcl[lcl hex] format.
LOG_ERROR << "Invalid lcl file name: " << file_name;
return -1;
}
}
}
// Check if there is a saved lcl file -> if no send genesis lcl.
if (ctx.cache.empty())
{
ctx.set_lcl(0, GENESIS_LEDGER);
}
else
{
const auto last_ledger = ctx.cache.rbegin();
ctx.set_lcl(last_ledger->first, last_ledger->second);
const uint64_t seq_no = ctx.get_seq_no();
// Remove old ledgers that exceeds max sequence range.
if (seq_no > MAX_LEDGER_SEQUENCE)
remove_old_ledgers(seq_no - MAX_LEDGER_SEQUENCE);
}
sync_ctx.lcl_sync_thread = std::thread(lcl_syncer_loop);
init_success = true;
return 0;
}
void deinit()
{
if (init_success)
{
sync_ctx.is_shutting_down = true;
sync_ctx.lcl_sync_thread.join();
}
}
void set_sync_target(const std::string &target_lcl)
{
if (sync_ctx.is_shutting_down)
return;
// Validate target lcl format.
uint64_t target_seq_no;
std::string target_hash;
if (extract_lcl(target_lcl, target_seq_no, target_hash) == -1)
{
LOG_ERROR << "lcl sync: Invalid target lcl " << target_seq_no;
return;
}
{
std::scoped_lock<std::mutex> lock(sync_ctx.target_lcl_mutex);
if (sync_ctx.target_lcl == target_lcl)
return;
sync_ctx.target_lcl = target_lcl;
sync_ctx.target_lcl_seq_no = target_seq_no;
sync_ctx.target_requested_on = 0;
sync_ctx.request_submissions = 0;
sync_ctx.is_syncing = true;
LOG_INFO << "lcl sync: Syncing for target:" << sync_ctx.target_lcl.substr(0, 15) << " (current:" << ctx.get_lcl().substr(0, 15) << ")";
}
}
/**
* Runs the lcl sync worker loop.
*/
void lcl_syncer_loop()
{
util::mask_signal();
LOG_INFO << "lcl sync: Worker started.";
while (!sync_ctx.is_shutting_down)
{
// Indicates whether any requests/responses were processed in the loop iteration.
bool processed = false;
// Perform lcl sync activities.
{
std::scoped_lock<std::mutex> lock(sync_ctx.target_lcl_mutex);
if (!sync_ctx.target_lcl.empty())
send_lcl_sync_request(); // Send lcl requests if needed (or abandon if sync timeout).
// Process any history responses from other nodes.
if (!sync_ctx.target_lcl.empty() && check_lcl_sync_responses() == 1)
processed = true;
}
// Serve any history requests from other nodes.
if (check_lcl_sync_requests() == 1)
processed = true;
// Wait a small delay if there were no requests/responses processed during previous iteration.
if (!processed)
util::sleep(SYNCER_IDLE_WAIT);
}
LOG_INFO << "lcl sync: Worker stopped.";
}
/**
* Submits/resubmits lcl history requests as needed. Abandons sync if threshold reached.
*/
void send_lcl_sync_request()
{
// If target lcl is genesis lcl, Clear the ledger history and reset target sequence number.
if (sync_ctx.target_lcl == GENESIS_LEDGER)
{
LOG_INFO << "lcl sync: Target is GENESIS. Clearing our history.";
clear_ledger();
sync_ctx.clear_target();
}
else
{
// Check whether we need to send any requests or abandon the sync due to timeout.
const uint64_t time_now = util::get_epoch_milliseconds();
if ((sync_ctx.target_requested_on == 0) || // Initial request.
(time_now - sync_ctx.target_requested_on) > REQUEST_RESUBMIT_TIMEOUT) // Request resubmission.
{
if (sync_ctx.request_submissions < ABANDON_THRESHOLD)
{
// Before first request, if full history mode is not enabled check the target lcl seq no to see whether
// it's too far ahead. That means no one probably has our lcl in their ledgers. So we should clear our
// entire ledger history before requesting from peers.
if (sync_ctx.target_requested_on == 0 && !conf::cfg.node.full_history && sync_ctx.target_lcl_seq_no > (ctx.get_seq_no() + MAX_LEDGER_SEQUENCE))
{
LOG_INFO << "lcl sync: Target " << sync_ctx.target_lcl.substr(0, 15) << " is too far ahead. Clearing our history.";
clear_ledger();
}
send_ledger_history_request(ctx.get_lcl(), sync_ctx.target_lcl);
sync_ctx.target_requested_on = time_now;
sync_ctx.request_submissions++;
}
else
{
LOG_INFO << "lcl sync: Resubmission threshold exceeded. Abandoning sync.";
sync_ctx.clear_target();
}
}
}
}
/**
* Processes any lcl responses we have received from other peers.
* @return 0 if no respones were processed. 1 if at least one response was processed.
*/
int check_lcl_sync_responses()
{
// Move over the collected responses to the local list.
std::list<p2p::history_response> history_responses;
{
std::scoped_lock<std::mutex>(sync_ctx.list_mutex);
history_responses.splice(history_responses.end(), sync_ctx.collected_history_responses);
}
const std::string current_lcl = ctx.get_lcl();
// Scan any queued lcl history responses.
// Only process the first successful item which matches with our current lcl.
for (const p2p::history_response &hr : history_responses)
{
if (hr.requester_lcl == current_lcl)
{
std::string new_lcl;
if (handle_ledger_history_response(hr, new_lcl) != -1)
{
LOG_INFO << "lcl sync: Sync complete. New lcl:" << new_lcl.substr(0, 15);
sync_ctx.clear_target();
break;
}
}
}
return history_responses.empty() ? 0 : 1;
}
/**
* Serves any lcl requests we have received from other peers.
* @return 0 if no requests were served. 1 if at least one request was served.
*/
int check_lcl_sync_requests()
{
// Move over the collected requests to the local list.
std::list<std::pair<std::string, p2p::history_request>> history_requests;
{
std::scoped_lock<std::mutex>(sync_ctx.list_mutex);
history_requests.splice(history_requests.end(), sync_ctx.collected_history_requests);
}
// Acquire lock so consensus does not update the ledger while we are reading the ledger.
std::scoped_lock<std::mutex> ledger_lock(ctx.ledger_mutex);
for (const auto &[session_id, hr] : history_requests)
{
// First check whether we have the required lcl available.
if (!check_required_lcl_availability(hr.required_lcl))
continue;
p2p::history_response resp;
if (ledger::retrieve_ledger_history(hr, resp) != -1)
{
flatbuffers::FlatBufferBuilder fbuf(1024);
p2pmsg::create_msg_from_history_response(fbuf, resp);
std::string_view msg = msg::fbuf::flatbuff_bytes_to_sv(fbuf.GetBufferPointer(), fbuf.GetSize());
// Find the peer that we should send the history response to.
std::scoped_lock<std::mutex> lock(p2p::ctx.peer_connections_mutex);
const auto peer_itr = p2p::ctx.peer_connections.find(session_id);
if (peer_itr != p2p::ctx.peer_connections.end())
{
comm::comm_session *session = peer_itr->second;
session->send(msg);
}
}
}
return history_requests.empty() ? 0 : 1;
}
/**
* Returns the current top ledger seq no and lcl.
*/
const std::pair<uint64_t, std::string> get_ledger_cache_top()
{
const auto latest_lcl_itr = ctx.cache.rbegin();
if (latest_lcl_itr == ctx.cache.rend())
return std::make_pair(0, GENESIS_LEDGER);
else
return std::make_pair(latest_lcl_itr->first, latest_lcl_itr->second);
}
/**
* Create and save ledger from the given proposal message. Called by consensus.
* @param proposal Consensus-reached Stage 3 proposal.
* @param raw_inputs Raw inputs that are going to store.
*/
int save_ledger(const p2p::proposal &proposal, const std::unordered_map<std::string, usr::raw_user_input> &raw_inputs)
{
// This is used as a sample to create ledger sqlite database,
// Later this callee method can be called directly from consensus on ledger storage implementations.
// ledger::ledger_sample::save_ledger(proposal);
uint64_t seq_no = 0;
std::string hash;
if (extract_lcl(proposal.lcl, seq_no, hash) == -1)
{
// lcl records should follow [ledger sequnce numer]-[lcl hex] format.
LOG_ERROR << "Invalid lcl name: " << proposal.lcl << " when saving ledger.";
return -1;
}
seq_no++; // New lcl sequence number.
// Serialize lcl using flatbuffer ledger block schema.
flatbuffers::FlatBufferBuilder builder(1024);
msg::fbuf::ledger::create_ledger_block_from_proposal(builder, proposal, seq_no);
// Get binary hash of the serialized lcl.
std::string_view ledger_str_buf = msg::fbuf::flatbuff_bytes_to_sv(builder.GetBufferPointer(), builder.GetSize());
const std::string lcl_hash = crypto::get_hash(ledger_str_buf);
// Get hex from binary hash.
const std::string lcl_hash_hex = util::to_hex(lcl_hash);
// Acquire lock so history request serving does not access the ledger while consensus is updating the ledger.
std::scoped_lock<std::mutex> ledger_lock(ctx.ledger_mutex);
// Construct lcl file name.
// lcl file name should follow [ledger sequnce numer]-lcl[lcl hex] format.
const std::string file_name = std::to_string(seq_no) + "-" + lcl_hash_hex;
if (write_ledger(file_name, builder.GetBufferPointer(), builder.GetSize()) == -1)
return -1;
ctx.set_lcl(seq_no, file_name);
ctx.cache.emplace(seq_no, std::move(file_name));
// Write full history to the full history directory if full history mode is on.
if (conf::cfg.node.full_history)
{
builder.Clear();
msg::fbuf::ledger::create_full_history_block_from_raw_input_map(builder, raw_inputs);
if (write_full_history(file_name, builder.GetBufferPointer(), builder.GetSize()) == -1)
return -1;
}
//Remove old ledgers that exceeds max sequence range.
if (seq_no > MAX_LEDGER_SEQUENCE)
remove_old_ledgers(seq_no - MAX_LEDGER_SEQUENCE);
return 0;
}
/**
* Remove old ledgers that exceeds max sequence range from file system and ledger history cache.
* @param led_seq_no minimum sequence number to be in history.
*/
void remove_old_ledgers(const uint64_t led_seq_no)
{
// Remove old ledgers if full history mode is not enabled.
if (!conf::cfg.node.full_history)
{
std::map<uint64_t, const std::string>::iterator itr;
for (itr = ctx.cache.begin();
itr != ctx.cache.lower_bound(led_seq_no + 1);
itr++)
{
const std::string file_path = conf::ctx.hist_dir + "/" + itr->second + ".lcl";
if (util::is_file_exists(file_path))
util::remove_file(file_path);
}
if (!ctx.cache.empty())
ctx.cache.erase(ctx.cache.begin(), ctx.cache.lower_bound(led_seq_no + 1));
}
}
/**
* Clears out entire ledger history.
*/
void clear_ledger()
{
util::clear_directory(conf::ctx.hist_dir);
ctx.cache.clear();
ctx.set_lcl(0, GENESIS_LEDGER);
}
/**
* Reads the specified ledger entry.
* @param file_path File path to read.
* @param buffer Buffer to populate with file contents.
* @return 0 on success. -1 on failure.
*/
int read_ledger(std::string_view file_path, std::vector<uint8_t> &buffer)
{
const int fd = open(file_path.data(), O_RDONLY);
if (fd == -1)
{
LOG_ERROR << errno << ": Error opening ledger file for read. " << file_path;
return -1;
}
struct stat st;
if (fstat(fd, &st) == -1)
{
close(fd);
LOG_ERROR << errno << ": Error in ledger file stat. " << file_path;
return -1;
}
buffer.resize(st.st_size);
if (read(fd, buffer.data(), buffer.size()) == -1)
{
close(fd);
LOG_ERROR << errno << ": Error reading ledger file. " << file_path;
return -1;
}
close(fd);
return 0;
}
/**
* Write ledger to file system.
* @param file_name current ledger sequence number.
* @param ledger_raw raw lcl data.
* @param ledger_size size of the raw lcl data.
*/
int write_ledger(const std::string &file_name, const uint8_t *ledger_raw, const size_t ledger_size)
{
// Create file path to save ledger.
// file name -> [ledger sequnce numer]-[lcl hex]
const std::string file_path = conf::ctx.hist_dir + "/" + file_name + ".lcl";
// Write ledger to file system
const int fd = open(file_path.data(), O_CREAT | O_RDWR, FILE_PERMS);
if (fd == -1)
{
LOG_ERROR << errno << ": Error creating ledger file. " << file_path;
return -1;
}
if (write(fd, ledger_raw, ledger_size) == -1)
{
LOG_ERROR << errno << ": Error writing to new ledger file. " << file_path;
close(fd);
return -1;
}
close(fd);
return 0;
}
/**
* Write full history to file system.
* @param file_name current ledger sequence number.
* @param full_history_raw raw full history data.
* @param full_history_size size of the raw full history data.
*/
int write_full_history(const std::string &file_name, const uint8_t *full_history_raw, const size_t full_history_size)
{
// Create file path to save full history.
// file name -> [ledger sequnce numer]-[lcl hex]
const std::string file_path = conf::ctx.full_hist_dir + "/" + file_name + ".flcl";
// Write full history to file system
const int fd = open(file_path.data(), O_CREAT | O_RDWR, FILE_PERMS);
if (fd == -1)
{
LOG_ERROR << errno << ": Error creating full history file. " << file_path;
return -1;
}
if (write(fd, full_history_raw, full_history_size) == -1)
{
LOG_ERROR << errno << ": Error writing to new full history file. " << file_path;
close(fd);
return -1;
}
close(fd);
return 0;
}
/**
* Delete ledger from file system.
* @param file_name name of ledger to be deleted.
*/
void remove_ledger(const std::string &file_name)
{
std::string file_path;
file_path.reserve(conf::ctx.hist_dir.size() + file_name.size() + 5);
file_path.append(conf::ctx.hist_dir)
.append("/")
.append(file_name)
.append(".lcl");
util::remove_file(file_path);
// Removing full history if exists.
file_path.clear();
file_path.reserve(conf::ctx.full_hist_dir.size() + file_name.size() + 6);
file_path.append(conf::ctx.full_hist_dir)
.append("/")
.append(file_name)
.append(".flcl");
if (util::is_file_exists(file_path))
util::remove_file(file_path);
}
/**
* Create and send ledger history request to random node from unl list.
* @param current_lcl Current lcl.
* @param required_lcl Required lcl.
*/
void send_ledger_history_request(std::string_view current_lcl, std::string_view required_lcl)
{
p2p::history_request hr;
hr.required_lcl = required_lcl;
hr.requester_lcl = current_lcl;
flatbuffers::FlatBufferBuilder fbuf(1024);
p2pmsg::create_msg_from_history_request(fbuf, hr);
std::string target_pubkey;
p2p::send_message_to_random_peer(fbuf, target_pubkey);
LOG_DEBUG << "Ledger history requested from [" << target_pubkey.substr(0, 10) << "]. Required lcl:" << required_lcl.substr(0, 15);
}
/**
* Check requested lcl is in node's lcl history cache.
* @param hr lcl history request information.
* @return true if requested lcl is in lcl history cache.
*/
bool check_required_lcl_availability(const std::string &required_lcl)
{
uint64_t req_seq_no = 0;
std::string hash;
if (extract_lcl(required_lcl, req_seq_no, hash) == -1)
{
LOG_DEBUG << "Required lcl parse error " << required_lcl;
return -1;
}
if (req_seq_no > 0)
{
const auto itr = ctx.cache.find(req_seq_no);
if (itr == ctx.cache.end())
{
LOG_DEBUG << "Required lcl seq no peer asked for is not in our lcl cache. " << required_lcl;
// Either this node is also not in consesnsus ledger or other node requesting a lcl that is older than node's current
// minimum lcl sequence becuase of maximum ledger history range.
return false;
}
else if (itr->second != required_lcl)
{
LOG_DEBUG << "Required lcl peer asked for is not in our lcl cache. " << required_lcl;
// Either this node or requesting node is in a fork condition.
return false;
}
}
else
{
return false; // Very rare case: Peer asking for the genisis lcl.
}
return true;
}
/**
* Retrieve lcl(last closed ledger) information from ledger history.
* @param hr lcl history request information.
* @param history_response Ledger history response to populate requested ledger details
* @return 0 on success. -1 on failure.
*/
int retrieve_ledger_history(const p2p::history_request &hr, p2p::history_response &history_response)
{
uint64_t min_seq_no = 0;
std::string hash;
if (extract_lcl(hr.requester_lcl, min_seq_no, hash) == -1)
{
LOG_DEBUG << "lcl serve: Invalid request. Requester lcl invalid:" << hr.requester_lcl;
return -1;
}
// We put the requester's own lcl back in the response so they can validate the liveliness of the response.
history_response.requester_lcl = hr.requester_lcl;
if (min_seq_no > 0)
{
const auto itr = ctx.cache.find(min_seq_no);
if (itr != ctx.cache.end()) // Requested minimum lcl was found in our lcl history cache
{
// Check whether requested minimum lcl hash is same as this node's.
// Evenhough sequence number are same, lcl hash can be changed if one of node is in a fork condition.
if (hr.requester_lcl != itr->second)
{
LOG_DEBUG << "lcl serve: Invalid minimum ledger. Requester lcl:" << hr.requester_lcl << " Node lcl:" << itr->second;
history_response.error = p2p::LEDGER_RESPONSE_ERROR::INVALID_MIN_LEDGER;
return 0;
}
min_seq_no = itr->first;
}
else if (min_seq_no > ctx.cache.rbegin()->first) //Recieved minimum lcl sequence is ahead of node's lcl sequence.
{
LOG_DEBUG << "lcl serve: Invalid minimum ledger. Requester lcl " << hr.requester_lcl << " is ahead of us.";
history_response.error = p2p::LEDGER_RESPONSE_ERROR::INVALID_MIN_LEDGER;
return 0;
}
else
{
LOG_DEBUG << "lcl serve: Requester lcl is not in our lcl cache. Sending our entire history.";
min_seq_no = ctx.cache.begin()->first;
}
}
else
{
LOG_DEBUG << "lcl serve: Requester lcl is GENSIS. Sending our entire history.";
min_seq_no = ctx.cache.begin()->first;
}
//copy current history cache.
std::map<uint64_t, const std::string> led_cache = ctx.cache;
//filter out cache and get raw files here.
led_cache.erase(
led_cache.begin(),
led_cache.lower_bound(min_seq_no));
//Get raw content of lcls that going to be send.
for (const auto &[seq_no, lcl] : led_cache)
{
p2p::history_ledger_block ledger_block;
ledger_block.lcl = lcl;
// Read lcl file.
const std::string file_path = conf::ctx.hist_dir + "/" + lcl + ".lcl";
if (read_ledger(file_path, ledger_block.block_buffer) == -1)
{
LOG_DEBUG << "lcl serve: Error when reading ledger file.";
return -1;
}
history_response.hist_ledger_blocks.emplace(seq_no, std::move(ledger_block));
}
return 0;
}
/**
* Handle recieved ledger history response.
* @param hr lcl history request information.
* @return 0 on successful lcl update. -1 on failure.
*/
int handle_ledger_history_response(const p2p::history_response &hr, std::string &new_lcl)
{
if (hr.error == p2p::LEDGER_RESPONSE_ERROR::INVALID_MIN_LEDGER)
{
// This means we are in a fork ledger. Remove/rollback current top ledger.
// Basically in the long run we'll rolback one by one untill we catch up to valid minimum ledger.
remove_ledger(ctx.get_lcl());
ctx.cache.erase(ctx.cache.rbegin()->first);
const auto [seq_no, lcl] = get_ledger_cache_top();
ctx.set_lcl(seq_no, lcl);
new_lcl = lcl;
LOG_INFO << "lcl sync: Fork detected. Removed last ledger. New lcl:" << lcl.substr(0, 15);
return 0;
}
else
{
// Check whether recieved lcl history contains the current lcl node required.
bool contains_requested_lcl = false;
for (auto &[seq_no, ledger] : hr.hist_ledger_blocks)
{
if (sync_ctx.target_lcl == ledger.lcl)
{
contains_requested_lcl = true;
break;
}
}
if (!contains_requested_lcl)
{
LOG_INFO << "lcl sync: Peer sent us a history response but not containing the lcl we asked for.";
return -1;
}
// Check integrity of recieved lcl list.
// By checking recieved lcl hashes matches lcl content by applying hashing for each raw content.
std::string previous_history_block_lcl;
uint64_t previous_history_block_seq_no;
for (auto &[seq_no, ledger] : hr.hist_ledger_blocks)
{
// Individually check each ledger entry's integrity before the chain check.
uint64_t lcl_seq_no;
std::string lcl_hash;
if (extract_lcl(ledger.lcl, lcl_seq_no, lcl_hash) == -1)
{
LOG_INFO << "lcl sync: Error when parsing lcl " << ledger.lcl;
return -1;
}
if (!check_block_integrity(lcl_hash, ledger.block_buffer))
{
LOG_INFO << "lcl sync: Peer sent us a history response but the ledger data does not match the hashes.";
// todo: we should penalize peer who sent this.
return -1;
}
// Ledger chain integrity check.
if (!previous_history_block_lcl.empty())
{
const p2p::proposal proposal = msg::fbuf::ledger::create_proposal_from_ledger_block(ledger.block_buffer);
if ((seq_no - previous_history_block_seq_no != 1) && (previous_history_block_lcl != proposal.lcl))
{
LOG_INFO << "Ledger block chain-link verification failed. " << ledger.lcl;
return -1;
}
}
previous_history_block_lcl = ledger.lcl;
previous_history_block_seq_no = seq_no;
}
}
// Performing ledger history joining check.
if (!ctx.cache.empty())
{
const auto history_itr = hr.hist_ledger_blocks.begin();
const p2p::proposal history_first_proposal = msg::fbuf::ledger::create_proposal_from_ledger_block(history_itr->second.block_buffer);
// Removing ledger blocks upto the received histroy response starting point.
const uint64_t joining_seq_no = history_itr->first;
if (ctx.cache.count(joining_seq_no) == 1)
{
// If cache ledger and history ledger are overlapping, remove blocks from end until the
// cache end at the state where history ledger can be straightly joined.
auto it = ctx.cache.rbegin();
while (it != ctx.cache.rend() && it->first >= joining_seq_no)
{
remove_ledger(it->second);
// Erase and advance the reverse iterator.
ctx.cache.erase((--it.base()));
}
auto &[cache_seq_no, cache_lcl] = get_ledger_cache_top();
ctx.set_lcl(cache_seq_no, cache_lcl);
// Comparing the sequence number and the lcl to validate the joining point.
if ((history_itr->first - cache_seq_no != 1) || (history_first_proposal.lcl != cache_lcl))
{
LOG_ERROR << "lcl sync: Ledger integrity check at history joining point failed.";
return -1;
}
}
}
// Execution to here means the history data sent checks out.
// Save recieved lcl in file system and update lcl history cache.
for (auto &[seq_no, ledger] : hr.hist_ledger_blocks)
{
write_ledger(ledger.lcl, ledger.block_buffer.data(), ledger.block_buffer.size());
ctx.cache.emplace(seq_no, ledger.lcl);
}
const auto [seq_no, lcl] = get_ledger_cache_top();
ctx.set_lcl(seq_no, lcl);
new_lcl = lcl;
return 0;
}
/**
* Check the integrity of the given ledger.
* @param supplied_hash supplied hash hex of the ledger block.
* @param raw_ledger ledger.
* @return true if the integrity check passes and false otherwise.
*/
bool check_block_integrity(std::string_view supplied_hash, const std::vector<uint8_t> &block_buffer)
{
// Get binary hash of the serialized lcl.
const std::string binary_block_hash = crypto::get_hash(block_buffer.data(), block_buffer.size());
// Get hex from binary hash.
const std::string block_hash = util::to_hex(binary_block_hash);
return block_hash == supplied_hash;
}
/**
* Sorting given lcl filename list in sequence number order and validate filenames.
* @param list List of lcl filenames.
* @return 0 if success and -1 on error.
*/
int sort_lcl_filenames_and_validate(std::list<std::string> &list)
{
try
{
list.sort([](std::string &a, std::string &b) {
uint64_t seq_no_a, seq_no_b;
if (util::stoull(a.substr(0, a.find("-")), seq_no_a) == -1)
{
throw "Lcl file parsing error in file " + a + " in " + conf::ctx.hist_dir;
}
if (util::stoull(b.substr(0, b.find("-")), seq_no_b) == -1)
{
throw "Lcl file parsing error in file " + b + " in " + conf::ctx.hist_dir;
}
const std::string_view extension_a = util::fetch_file_extension(a);
if (extension_a != ".lcl")
{
throw "Found invalid file extension: " + std::string(extension_a) + " for lcl file " + a + " in " + conf::ctx.hist_dir;
}
const std::string_view extension_b = util::fetch_file_extension(b);
if (extension_b != ".lcl")
{
throw "Found invalid file extension: " + std::string(extension_b) + " for lcl file " + b + " in " + conf::ctx.hist_dir;
}
return seq_no_a < seq_no_b;
});
return 0;
}
catch (std::string message)
{
LOG_ERROR << message;
return -1;
}
}
int extract_lcl(const std::string &lcl, uint64_t &seq_no, std::string &hash)
{
if (lcl == GENESIS_LEDGER)
{
seq_no = 0;
hash = lcl.substr(2);
return 0;
}
const size_t pos = lcl.find("-");
if (pos == std::string::npos)
return -1;
if (util::stoull(lcl.substr(0, pos), seq_no) == -1)
return -1;
hash = lcl.substr(pos + 1);
if (hash.size() != 64)
return -1;
return 0;
}
} // namespace ledger

126
src/ledger.hpp
View File

@@ -1,126 +0,0 @@
#ifndef _HP_LEDGER_
#define _HP_LEDGER_
#include "pchheader.hpp"
#include "p2p/p2p.hpp"
namespace ledger
{
constexpr const char *GENESIS_LEDGER = "0-genesis";
constexpr uint16_t HISTORY_REQ_LIST_CAP = 64; // Maximum history request count.
constexpr uint16_t HISTORY_RES_LIST_CAP = 64; // Maximum history response count.
struct sync_context
{
// The current target lcl that we are syncing towards.
std::string target_lcl;
uint64_t target_lcl_seq_no = 0;
uint64_t target_requested_on = 0;
uint16_t request_submissions = 0;
std::mutex target_lcl_mutex;
// Lists holding history requests and responses collected from incoming p2p messages.
std::list<std::pair<std::string, p2p::history_request>> collected_history_requests;
std::list<p2p::history_response> collected_history_responses;
std::mutex list_mutex;
std::thread lcl_sync_thread;
std::atomic<bool> is_syncing = false;
std::atomic<bool> is_shutting_down = false;
void clear_target()
{
target_lcl.clear();
target_lcl_seq_no = 0;
target_requested_on = 0;
request_submissions = 0;
is_syncing = false;
}
};
struct ledger_context
{
private:
std::string lcl;
uint64_t seq_no = 0;
std::shared_mutex lcl_mutex;
public:
// Map of closed ledgers (lcl string) with sequence number as map key.
// Contains closed ledgers from oldest to latest - MAX_LEDGER_SEQUENCE.
// This is loaded when node started and updated throughout consensus.
// If full history mode is not enabled, deletes ledgers that falls behind MAX_LEDGER_SEQUENCE range.
std::map<uint64_t, const std::string> cache;
std::mutex ledger_mutex;
const std::string get_lcl()
{
std::shared_lock lock(lcl_mutex);
return lcl;
}
uint64_t get_seq_no()
{
std::shared_lock lock(lcl_mutex);
return seq_no;
}
void set_lcl(const uint64_t new_seq_no, std::string_view new_lcl)
{
std::unique_lock lock(lcl_mutex);
lcl = new_lcl;
seq_no = new_seq_no;
}
};
extern sync_context sync_ctx;
extern ledger_context ctx;
int init();
void deinit();
void set_sync_target(const std::string &target_lcl);
void lcl_syncer_loop();
void send_lcl_sync_request();
int check_lcl_sync_responses();
int check_lcl_sync_requests();
const std::pair<uint64_t, std::string> get_ledger_cache_top();
int save_ledger(const p2p::proposal &proposal, const std::unordered_map<std::string, usr::raw_user_input> &raw_inputs);
void remove_old_ledgers(const uint64_t led_seq_no);
void clear_ledger();
int read_ledger(std::string_view file_path, std::vector<uint8_t> &buffer);
int write_ledger(const std::string &file_name, const uint8_t *ledger_raw, const size_t ledger_size);
int write_full_history(const std::string &file_name, const uint8_t *full_history_raw, const size_t full_history_size);
void remove_ledger(const std::string &file_name);
void send_ledger_history_request(std::string_view current_lcl, std::string_view required_lcl);
bool check_required_lcl_availability(const std::string &required_lcl);
int retrieve_ledger_history(const p2p::history_request &hr, p2p::history_response &history_response);
int handle_ledger_history_response(const p2p::history_response &hr, std::string &new_lcl);
bool check_block_integrity(std::string_view hash, const std::vector<uint8_t> &block_buffer);
int sort_lcl_filenames_and_validate(std::list<std::string> &list);
int extract_lcl(const std::string &lcl, uint64_t &seq_no, std::string &hash);
} // namespace ledger
#endif

527
src/ledger/ledger.cpp Normal file
View File

@@ -0,0 +1,527 @@
#include "ledger.hpp"
#include "../crypto.hpp"
#include "../conf.hpp"
#include "../util/util.hpp"
#include "../msg/fbuf/ledger_helpers.hpp"
#include "../msg/fbuf/common_helpers.hpp"
#include "ledger_serve.hpp"
#define LEDGER_CREATE_ERROR \
{ \
if (db != NULL) \
sqlite::close_db(&db); \
ledger_fs.release_rw_session(); \
return -1; \
}
namespace ledger
{
ledger_context ctx;
constexpr uint32_t LEDGER_FS_ID = 1;
ledger::ledger_mount ledger_fs; // Global ledger file system instance.
ledger::ledger_sync ledger_sync_worker; // Global ledger file system sync instance.
ledger::ledger_serve ledger_server; // Ledger file server instance.
std::shared_mutex primary_index_file_mutex;
/**
* Perform ledger related initializations.
*/
int init()
{
if (ledger_fs.init(LEDGER_FS_ID, conf::ctx.ledger_hpfs_dir, conf::ctx.ledger_hpfs_mount_dir, conf::ctx.ledger_hpfs_rw_dir, conf::cfg.node.full_history) == -1)
{
LOG_ERROR << "Ledger file system initialization failed.";
return -1;
}
if (ledger_server.init("ledger", &ledger_fs) == -1)
{
LOG_ERROR << "Ledger file system serve worker initialization failed.";
return -1;
}
if (ledger_sync_worker.init("ledger", &ledger_fs) == -1)
{
LOG_ERROR << "Ledger file system sync worker initialization failed.";
return -1;
}
if (get_last_ledger_and_update_context() == -1)
{
LOG_ERROR << "Getting last ledger faild.";
return -1;
}
return 0;
}
/**
* Perform deinit tasks related to ledger.
*/
void deinit()
{
ledger_sync_worker.deinit();
ledger_server.deinit();
ledger_fs.deinit();
}
/**
* Create and save ledger record from the given proposal message.
* @param proposal Consensus-reached Stage 3 proposal.
* @param candidate_user_inputs Raw inputs received in this consensus round.
* @param generated_user_outputs Generated raw outputs in this consensus round.
* @return Returns 0 on success -1 on error.
*/
int save_ledger(const p2p::proposal &proposal, const std::map<std::string, consensus::candidate_user_input> &candidate_user_inputs,
const std::map<std::string, consensus::generated_user_output> &generated_user_outputs)
{
uint64_t seq_no = 0;
std::string prev_ledger_hash_hex;
if (extract_lcl(proposal.lcl, seq_no, prev_ledger_hash_hex) == -1)
{
// lcl records should follow [ledger sequnce numer]-[lcl hex] format.
LOG_ERROR << "Invalid lcl name: " << proposal.lcl << " when saving ledger.";
return -1;
}
seq_no++; // New lcl sequence number.
// Aqure hpfs rw session before accessing shards and insert ledger records.
if (ledger_fs.acquire_rw_session() == -1)
return -1;
sqlite3 *db = NULL;
// Prepare shard folders and database and get the primary shard sequence number.
uint64_t primary_shard_seq_no;
if (prepare_shard(&db, primary_shard_seq_no, seq_no) == -1)
LEDGER_CREATE_ERROR;
// Combined binary hash of consensus user binary pub keys.
const std::string user_hash = crypto::get_hash(proposal.users);
// Combined binary hash of consensus input hashes.
const std::string input_hash = crypto::get_hash(proposal.input_hashes);
uint8_t seq_no_byte_str[8], time_byte_str[8];
util::uint64_to_bytes(seq_no_byte_str, seq_no);
util::uint64_to_bytes(time_byte_str, proposal.time);
// Contruct binary string for data hash.
std::string data;
data.reserve(sizeof(seq_no_byte_str) + sizeof(time_byte_str) + (sizeof(util::h32) * 5));
data.append((char *)seq_no_byte_str);
data.append((char *)time_byte_str);
data.append(proposal.state_hash.to_string_view());
data.append(proposal.patch_hash.to_string_view());
data.append(user_hash);
data.append(input_hash);
data.append(proposal.output_hash);
// Combined binary hash of data fields. blake3(seq_no + time + state_hash + patch_hash + user_hash + input_hash + output_hash)
const std::string data_hash = crypto::get_hash(data);
const std::string prev_ledger_hash = util::to_bin(prev_ledger_hash_hex);
// Ledger hash is the combined hash of previous ledger hash and the new data hash.
const std::string ledger_hash = crypto::get_hash(prev_ledger_hash, data_hash);
const std::string ledger_hash_hex = util::to_hex(ledger_hash);
// Construct ledger struct.
// Hashes are stored as hex string;
const sqlite::ledger ledger(
seq_no,
proposal.time,
ledger_hash_hex,
prev_ledger_hash_hex,
util::to_hex(data_hash),
util::to_hex(proposal.state_hash.to_string_view()),
util::to_hex(proposal.patch_hash.to_string_view()),
util::to_hex(user_hash),
util::to_hex(input_hash),
util::to_hex(proposal.output_hash)); // Merkle root output hash.
if (sqlite::insert_ledger_row(db, ledger) == -1)
{
LOG_ERROR << errno << ": Error creating the ledger, shard: " << std::to_string(primary_shard_seq_no);
LEDGER_CREATE_ERROR;
}
if ((!candidate_user_inputs.empty() || !generated_user_outputs.empty()) && save_ledger_blob(ledger_hash, candidate_user_inputs, generated_user_outputs) == -1)
{
LOG_ERROR << errno << ": Error saving the raw inputs/outputs, shard: " << std::to_string(primary_shard_seq_no);
LEDGER_CREATE_ERROR;
}
// Update the seq_no and lcl when ledger is created.
const std::string new_lcl = std::string(std::to_string(seq_no)).append("-").append(ledger_hash_hex);
ctx.set_lcl(seq_no, new_lcl);
const std::string shard_vpath = std::string(ledger::PRIMARY_DIR).append("/").append(std::to_string(primary_shard_seq_no));
util::h32 last_primary_shard_hash;
if (ledger_fs.get_hash(last_primary_shard_hash, hpfs::RW_SESSION_NAME, shard_vpath) == -1)
{
LOG_ERROR << errno << ": Error reading shard hash: " << std::to_string(primary_shard_seq_no);
LEDGER_CREATE_ERROR;
}
// Update the last shard hash and shard seqence number tracker when a new ledger is created.
ctx.set_last_primary_shard_id(p2p::sequence_hash{primary_shard_seq_no, last_primary_shard_hash});
//Remove old shards that exceeds max shard range.
if (conf::cfg.node.max_shards > 0 && primary_shard_seq_no >= conf::cfg.node.max_shards)
{
remove_old_shards(primary_shard_seq_no - conf::cfg.node.max_shards + 1, PRIMARY_DIR);
}
sqlite::close_db(&db);
return ledger_fs.release_rw_session();
}
/**
* Opens a db connection to a shard and populates the shard_seq_no.
* @param db Database connection to be openned.
* @param ledger_seq_no Ledger sequence number.
* @return Returns 0 on success -1 on failure.
*/
int prepare_shard(sqlite3 **db, uint64_t &shard_seq_no, const uint64_t ledger_seq_no)
{
// Construct shard path.
shard_seq_no = (ledger_seq_no - 1) / PRIMARY_SHARD_SIZE;
const std::string shard_path = ledger_fs.physical_path(hpfs::RW_SESSION_NAME, std::string(ledger::PRIMARY_DIR).append("/").append(std::to_string(shard_seq_no)));
// If (seq_no - 1) % PRIMARY_SHARD_SIZE == 0 means this is the first ledger of the shard.
// So create the shard folder and ledger table.
if ((ledger_seq_no - 1) % PRIMARY_SHARD_SIZE == 0)
{
// Creating the directory.
if (util::create_dir_tree_recursive(shard_path) == -1)
{
LOG_ERROR << errno << ": Error creating the shard, shard: " << std::to_string(shard_seq_no);
return -1;
}
// Creating ledger database and open a database connection.
if (sqlite::open_db(shard_path + "/" + DATEBASE, db) == -1)
{
LOG_ERROR << errno << ": Error openning the shard database, shard: " << std::to_string(shard_seq_no);
return -1;
}
// Creating the ledger table.
if (sqlite::create_ledger_table(*db) == -1)
{
LOG_ERROR << errno << ": Error creating the shard table, shard: " << std::to_string(shard_seq_no);
return -1;
}
util::h32 prev_shard_hash;
if (shard_seq_no > 0)
{
const std::string prev_shard_vpath = std::string(ledger::PRIMARY_DIR).append("/").append(std::to_string(shard_seq_no - 1));
if (ledger_fs.get_hash(prev_shard_hash, hpfs::RW_SESSION_NAME, prev_shard_vpath) < 1)
{
LOG_ERROR << errno << ": Error getting shard hash in vpath: " << prev_shard_vpath << " for previous shard hash.";
return -1;
}
}
// Write the prev_shard.hash to the new folder.
const std::string shard_hash_file_path = shard_path + PREV_SHARD_HASH_FILENAME;
const int fd = open(shard_hash_file_path.data(), O_CREAT | O_RDWR, FILE_PERMS);
if (fd == -1)
{
LOG_ERROR << errno << ": Error creating prev_shard.hash file in shard " << std::to_string(shard_seq_no);
return -1;
}
if (write(fd, &prev_shard_hash, sizeof(util::h32)) == -1)
{
LOG_ERROR << errno << ": Error writing to " << shard_hash_file_path << ".";
close(fd);
return -1;
}
close(fd);
}
else if (sqlite::open_db(shard_path + "/" + DATEBASE, db) == -1)
{
LOG_ERROR << errno << ": Error openning the shard database, shard: " << std::to_string(shard_seq_no);
return -1;
}
return 0;
}
/**
* Remove old shards that exceeds max shard range from file system.
* @param led_shard_no minimum shard number to be in history.
*/
void remove_old_shards(const uint64_t led_shard_no, std::string_view shard_parent_dir)
{
// Remove old shards if full history mode is not enabled.
if (!conf::cfg.node.full_history)
{
const std::string shard_dir_path = ledger_fs.physical_path(hpfs::RW_SESSION_NAME, shard_parent_dir);
std::list<std::string> shards = util::fetch_dir_entries(shard_dir_path);
for (const std::string shard : shards)
{
uint64_t primary_shard_seq_no;
util::stoull(shard, primary_shard_seq_no);
if (primary_shard_seq_no < led_shard_no)
{
const std::string shard_path = std::string(shard_dir_path).append("/").append(shard);
if (util::is_dir_exists(shard_path) && util::remove_directory_recursively(shard_path) == -1)
{
LOG_ERROR << errno << ": Error deleting shard: " << shard;
}
}
}
}
}
/**
* Extract seq_no and hash from lcl.
* @param lcl lcl to be extracted.
* @param seq_no Extracted sequence number.
* @param hash Extracted hash.
* @return Returns 0 on success -1 on error.
*/
int extract_lcl(const std::string &lcl, uint64_t &seq_no, std::string &hash)
{
if (lcl == GENESIS_LEDGER)
{
seq_no = 0;
hash = lcl.substr(2);
return 0;
}
const size_t pos = lcl.find("-");
if (pos == std::string::npos)
return -1;
if (util::stoull(lcl.substr(0, pos), seq_no) == -1)
return -1;
hash = lcl.substr(pos + 1);
if (hash.size() != 64)
return -1;
return 0;
}
/**
* Save raw data from the consensused proposal. A blob file is only created if there is any user inputs or contract outputs
* to save disk space.
* @param ledger_hash Hash of this ledger we are saving.
* @param candidate_user_inputs Raw inputs received in this consensus round.
* @param generated_user_outputs Generated raw outputs in this consensus round.
* @return Returns 0 on success -1 on error.
*/
int save_ledger_blob(std::string_view ledger_hash, const std::map<std::string, consensus::candidate_user_input> &candidate_user_inputs,
const std::map<std::string, consensus::generated_user_output> &generated_user_outputs)
{
// Construct shard path.
uint64_t last_blob_shard_seq_no = ctx.get_last_blob_shard_id().seq_no;
std::string shard_vpath = std::string(ledger::BLOB_DIR).append("/").append(std::to_string(last_blob_shard_seq_no));
std::string shard_path = ledger_fs.physical_path(hpfs::RW_SESSION_NAME, shard_vpath);
bool should_create_folder = false;
if (util::is_dir_exists(shard_path))
{
if ((util::fetch_dir_entries(shard_path).size() - 1) >= BLOB_SHARD_SIZE)
{
should_create_folder = true;
last_blob_shard_seq_no++;
shard_vpath = std::string(ledger::BLOB_DIR).append("/").append(std::to_string(last_blob_shard_seq_no));
shard_path = ledger_fs.physical_path(hpfs::RW_SESSION_NAME, shard_vpath);
}
}
else
{
should_create_folder = true;
}
// Create the required shard folder if not already existing.
if (should_create_folder)
{
// Creating the directory.
if (util::create_dir_tree_recursive(shard_path) == -1)
{
LOG_ERROR << errno << ": Error creating the blob shard, shard: " << std::to_string(last_blob_shard_seq_no);
ledger_fs.release_rw_session();
return -1;
}
util::h32 prev_shard_hash;
if (last_blob_shard_seq_no > 0)
{
const std::string prev_shard_vpath = std::string(ledger::BLOB_DIR).append("/").append(std::to_string(last_blob_shard_seq_no - 1));
if (ledger_fs.get_hash(prev_shard_hash, hpfs::RW_SESSION_NAME, prev_shard_vpath) < 1)
{
LOG_ERROR << errno << ": Error getting blob shard hash in vpath: " << prev_shard_vpath << " for previous shard hash.";
ledger_fs.release_rw_session();
return -1;
}
}
// Write the prev_shard.hash to the new folder.
const std::string shard_hash_file_path = shard_path + PREV_SHARD_HASH_FILENAME;
const int fd = open(shard_hash_file_path.data(), O_CREAT | O_RDWR, FILE_PERMS);
if (fd == -1)
{
LOG_ERROR << errno << ": Error creating prev_shard.hash file in blob shard " << std::to_string(last_blob_shard_seq_no);
return -1;
}
if (write(fd, &prev_shard_hash, sizeof(util::h32)) == -1)
{
LOG_ERROR << errno << ": Error writing to " << shard_hash_file_path << ".";
close(fd);
return -1;
}
close(fd);
}
ledger_blob blob;
blob.ledger_hash = ledger_hash;
for (const auto &[hash, user_input] : candidate_user_inputs)
{
std::string input;
if (usr::input_store.read_buf(user_input.input, input) != -1)
{
const auto itr = blob.inputs.find(user_input.userpubkey);
if (itr == blob.inputs.end())
blob.inputs.emplace(user_input.userpubkey, std::vector<std::string>());
blob.inputs[user_input.userpubkey].push_back(input);
}
}
for (const auto &[hash, user_output] : generated_user_outputs)
{
std::vector<std::string> outputs;
for (const auto &output : user_output.outputs)
{
outputs.push_back(output.message);
}
blob.outputs.emplace(user_output.userpubkey, outputs);
}
flatbuffers::FlatBufferBuilder builder(1024);
msg::fbuf::ledgermsg::create_ledger_blob_msg_from_ledger_blob(builder, blob);
const std::string file_path = shard_path + "/" + util::to_hex(ledger_hash) + ".blob";
const int fd = open(file_path.data(), O_CREAT | O_RDWR, FILE_PERMS);
if (fd == -1)
{
LOG_ERROR << errno << ": Error creating ledger blob file. " << file_path;
return -1;
}
if (write(fd, builder.GetBufferPointer(), builder.GetSize()) == -1)
{
LOG_ERROR << errno << ": Error writing to ledger blob file. " << file_path;
close(fd);
return -1;
}
close(fd);
util::h32 last_shard_hash;
if (ledger_fs.get_hash(last_shard_hash, hpfs::RW_SESSION_NAME, shard_vpath) == -1)
{
LOG_ERROR << errno << ": Error reading blob shard hash: " << std::to_string(last_blob_shard_seq_no);
ledger_fs.release_rw_session();
return -1;
}
// Update the last blob shard hash and blob shard seqence number tracker when a new ledger is created.
ctx.set_last_blob_shard_id(p2p::sequence_hash{last_blob_shard_seq_no, last_shard_hash});
//Remove old shards that exceeds max shard range.
if (last_blob_shard_seq_no >= MAX_BLOB_SHARDS)
{
remove_old_shards(last_blob_shard_seq_no - MAX_BLOB_SHARDS + 1, BLOB_DIR);
}
return 0;
}
/**
* Get last ledger and update the context.
* @return Returns 0 on success -1 on error.
*/
int get_last_ledger_and_update_context()
{
// Aquire hpfs rw session before accessing shards and insert ledger records.
if (ledger_fs.acquire_rw_session() == -1)
return -1;
const std::string shard_dir_path = ledger_fs.physical_path(hpfs::RW_SESSION_NAME, ledger::PRIMARY_DIR);
std::list<std::string> shards = util::fetch_dir_entries(shard_dir_path);
if (shards.size() == 0)
{
ledger_fs.release_rw_session();
ctx.set_lcl(0, GENESIS_LEDGER);
}
else
{
sqlite3 *db = NULL;
shards.sort([](std::string &a, std::string &b) {
uint64_t seq_no_a, seq_no_b;
util::stoull(a, seq_no_a);
util::stoull(b, seq_no_b);
return seq_no_a > seq_no_b;
});
uint64_t last_primary_shard_seq_no;
util::stoull(shards.front(), last_primary_shard_seq_no);
const std::string shard_path = std::string(shard_dir_path).append("/").append(shards.front());
// Open a database connection.
if (sqlite::open_db(shard_path + "/" + DATEBASE, &db) == -1)
{
LOG_ERROR << errno << ": Error openning the shard database, shard: " << shards.front();
ledger_fs.release_rw_session();
return -1;
}
sqlite::ledger last_ledger = sqlite::get_last_ledger(db);
sqlite::close_db(&db);
ctx.set_lcl(last_ledger.seq_no, std::to_string(last_ledger.seq_no) + "-" + last_ledger.ledger_hash_hex);
ledger_fs.release_rw_session();
}
return 0;
}
/**
* Get the hash and shard sequence number of the last shard in the ledger primary directory.
* @param session_name Hpfs session name.
* @param last_shard_id Struct which holds last shard data. (sequence number and hash).
* @param shard_parent_dir Parent director vpath of the shards.
* @return
*/
int get_last_shard_info(std::string_view session_name, p2p::sequence_hash &last_shard_id, std::string_view shard_parent_dir)
{
const std::string shard_dir_path = ledger_fs.physical_path(session_name, shard_parent_dir);
std::list<std::string> shards = util::fetch_dir_entries(shard_dir_path);
if (shards.size() > 0)
{
shards.sort([](std::string &a, std::string &b) {
uint64_t seq_no_a, seq_no_b;
util::stoull(a, seq_no_a);
util::stoull(b, seq_no_b);
return seq_no_a > seq_no_b;
});
const std::string shard_path = std::string(shard_parent_dir).append("/").append(shards.front());
if (ledger_fs.get_hash(last_shard_id.hash, session_name, shard_path) == -1 || util::stoull(shards.front(), last_shard_id.seq_no) == -1)
{
LOG_ERROR << "Error reading last shard hash in " << shard_path;
return -1;
}
}
return 0;
}
} // namespace ledger

108
src/ledger/ledger.hpp Normal file
View File

@@ -0,0 +1,108 @@
#ifndef _HP_LEDGER_LEDGER_
#define _HP_LEDGER_LEDGER_
#include "../p2p/p2p.hpp"
#include "sqlite.hpp"
#include "../consensus.hpp"
#include "ledger_sync.hpp"
#include "ledger_mount.hpp"
namespace ledger
{
constexpr const char *GENESIS_LEDGER = "0-genesis";
constexpr const char *DATEBASE = "ledger.sqlite";
constexpr uint64_t PRIMARY_SHARD_SIZE = 4096;
constexpr uint64_t BLOB_SHARD_SIZE = 4096;
constexpr int FILE_PERMS = 0644;
constexpr uint64_t MAX_BLOB_SHARDS = 4;
struct ledger_context
{
private:
std::shared_mutex lcl_mutex;
std::string lcl;
uint64_t seq_no = 0;
std::shared_mutex last_primary_shard_mutex;
p2p::sequence_hash last_primary_shard_id;
std::shared_mutex last_blob_shard_mutex;
p2p::sequence_hash last_blob_shard_id;
public:
const std::string get_lcl()
{
std::shared_lock lock(lcl_mutex);
return lcl;
}
uint64_t get_seq_no()
{
std::shared_lock lock(lcl_mutex);
return seq_no;
}
void set_lcl(const uint64_t new_seq_no, std::string_view new_lcl)
{
std::unique_lock lock(lcl_mutex);
lcl = new_lcl;
seq_no = new_seq_no;
}
const p2p::sequence_hash get_last_primary_shard_id()
{
std::shared_lock lock(last_primary_shard_mutex);
return last_primary_shard_id;
}
void set_last_primary_shard_id(const p2p::sequence_hash &sequence_hash_id)
{
std::unique_lock lock(last_primary_shard_mutex);
last_primary_shard_id = sequence_hash_id;
}
const p2p::sequence_hash get_last_blob_shard_id()
{
std::shared_lock lock(last_blob_shard_mutex);
return last_blob_shard_id;
}
void set_last_blob_shard_id(const p2p::sequence_hash &sequence_hash_id)
{
std::unique_lock lock(last_blob_shard_mutex);
last_blob_shard_id = sequence_hash_id;
}
};
struct ledger_blob
{
std::string ledger_hash;
std::map<std::string, std::vector<std::string>> inputs;
std::map<std::string, std::vector<std::string>> outputs;
};
extern ledger_context ctx;
extern ledger::ledger_mount ledger_fs; // Global ledger file system instance.
extern ledger::ledger_sync ledger_sync_worker; // Global ledger file system sync instance.
int init();
void deinit();
int save_ledger(const p2p::proposal &proposal, const std::map<std::string, consensus::candidate_user_input> &candidate_user_inputs,
const std::map<std::string, consensus::generated_user_output> &generated_user_outputs);
int prepare_shard(sqlite3 **db, uint64_t &shard_seq_no, const uint64_t ledger_seq_no);
int save_ledger_blob(std::string_view ledger_hash, const std::map<std::string, consensus::candidate_user_input> &candidate_user_inputs,
const std::map<std::string, consensus::generated_user_output> &generated_user_outputs);
void remove_old_shards(const uint64_t led_shard_no, std::string_view shard_parent_dir);
int extract_lcl(const std::string &lcl, uint64_t &seq_no, std::string &hash);
int get_last_ledger_and_update_context();
int get_last_shard_info(std::string_view session_name, p2p::sequence_hash &last_shard_id, std::string_view shard_parent_dir);
} // namespace ledger
#endif

20
src/ledger/ledger_mount.cpp
View File

@@ -1,4 +1,5 @@
#include "./ledger_mount.hpp"
#include "ledger.hpp"
namespace ledger
{
@@ -9,6 +10,25 @@ namespace ledger
int ledger_mount::prepare_fs()
{
// Add ledger fs preparation logic here.
p2p::sequence_hash last_primary_shard_id;
p2p::sequence_hash last_blob_shard_id;
constexpr const char *session_name = "ro_ledger_prepare_fs";
if (start_ro_session(session_name, true) == -1 ||
get_last_shard_info(session_name, last_primary_shard_id, PRIMARY_DIR) == -1 ||
get_last_shard_info(session_name, last_blob_shard_id, BLOB_DIR) == -1 ||
stop_ro_session(session_name) == -1)
{
LOG_ERROR << "Failed to prepare initial fs at mount " << mount_dir << ".";
return -1;
}
LOG_INFO << "Initial primary: " << last_primary_shard_id << " | blob: " << last_blob_shard_id;
// Update last shard hash and shard number tracker.
ctx.set_last_primary_shard_id(last_primary_shard_id);
// Update last blob shard hash and blob shard number tracker.
ctx.set_last_blob_shard_id(last_blob_shard_id);
return 0;
}

3
src/ledger/ledger_mount.hpp
View File

@@ -8,6 +8,9 @@
namespace ledger
{
constexpr const char *PRIMARY_DIR = "/primary"; // Ledger primary directory name.
constexpr const char *BLOB_DIR = "/blob"; // Ledger blob directory name.
constexpr const char *PREV_SHARD_HASH_FILENAME = "/prev_shard.hash"; // Previous shard hash file name.
/**
* Represents ledger file system mount.
*/

161
src/ledger/ledger_sample.cpp
View File

@@ -1,161 +0,0 @@
#include "ledger_sample.hpp"
#include "../crypto.hpp"
#include "../util/util.hpp"
#include "../msg/fbuf/ledger_helpers.hpp"
#include "../msg/fbuf/common_helpers.hpp"
#include "ledger_serve.hpp"
// Currently this namespace is added for sqlite testing, later this can be modified and renamed as 'ledger::ledger_sample' -> 'ledger' for ledger implementations.
namespace ledger::ledger_sample
{
constexpr uint32_t LEDGER_FS_ID = 1;
ledger::ledger_mount ledger_fs; // Global ledger file system instance.
ledger::ledger_sync ledger_sync_worker; // Global ledger file system sync instance.
ledger::ledger_serve ledger_server; // Ledger file server instance.
/**
* Perform ledger related initializations.
*/
int init()
{
if (ledger_fs.init(LEDGER_FS_ID, conf::ctx.ledger_hpfs_dir, conf::ctx.ledger_hpfs_mount_dir, conf::ctx.ledger_hpfs_rw_dir, conf::cfg.node.full_history) == -1)
{
LOG_ERROR << "Ledger file system initialization failed.";
return -1;
}
if (ledger_server.init("ledger", &ledger_fs) == -1)
{
LOG_ERROR << "Ledger file system serve worker initialization failed.";
return -1;
}
if (ledger_sync_worker.init("ledger", &ledger_fs) == -1)
{
LOG_ERROR << "Ledger file system sync worker initialization failed.";
return -1;
}
return 0;
}
/**
* Perform deinit tasks related to ledger.
*/
void deinit()
{
ledger_fs.deinit();
ledger_server.deinit();
ledger_sync_worker.deinit();
}
/**
* Create and save ledger record from the given proposal message.
* @param proposal Consensus-reached Stage 3 proposal.
*/
int save_ledger(const p2p::proposal &proposal)
{
sqlite3 *db;
// For testing purpose a database file is created in directory root.
if (sqlite::open_db("ledger.sqlite", &db) == -1)
{
sqlite3_close(db);
return -1;
}
if (!sqlite::is_ledger_table_exist(db) && sqlite::create_ledger_table(db) == -1)
{
sqlite3_close(db);
return -1;
}
uint64_t seq_no = 0;
std::string hash;
if (extract_lcl(proposal.lcl, seq_no, hash) == -1)
{
// lcl records should follow [ledger sequnce numer]-[lcl hex] format.
LOG_ERROR << "Invalid lcl name: " << proposal.lcl << " when saving ledger.";
return -1;
}
seq_no++; // New lcl sequence number.
// Serialize lcl using flatbuffer ledger block schema.
flatbuffers::FlatBufferBuilder builder(1024);
msg::fbuf::ledger::create_ledger_block_from_proposal(builder, proposal, seq_no);
// Get binary hash of the serialized lcl.
std::string_view ledger_str_buf = msg::fbuf::flatbuff_bytes_to_sv(builder.GetBufferPointer(), builder.GetSize());
const std::string lcl_hash = crypto::get_hash(ledger_str_buf);
// Get binary hash of users and inputs.
const std::string user_hash = crypto::get_hash(proposal.users);
const std::string input_hash = crypto::get_hash(proposal.input_hashes);
const std::string seq_no_str = std::to_string(seq_no);
const std::string time_str = std::to_string(proposal.time);
// Contruct binary string for data hash.
std::string data;
data.reserve(seq_no_str.size() + time_str.size() + (32 * 5));
data.append(seq_no_str);
data.append(time_str);
data.append(proposal.state_hash.to_string_view());
data.append(proposal.patch_hash.to_string_view());
data.append(user_hash);
data.append(input_hash);
data.append(proposal.output_hash);
// Get binary hash of data.
const std::string data_hash = crypto::get_hash(data);
// Construct ledger struct.
// Hashes are stored as hex string;
const sqlite::ledger ledger(
seq_no,
proposal.time,
util::to_hex(lcl_hash),
hash,
util::to_hex(data_hash),
util::to_hex(proposal.state_hash.to_string_view()),
util::to_hex(proposal.patch_hash.to_string_view()),
util::to_hex(user_hash),
util::to_hex(input_hash),
util::to_hex(proposal.output_hash));
if (sqlite::insert_ledger_row(db, ledger) == -1)
{
sqlite3_close(db);
return -1;
}
sqlite3_close(db);
return 0;
}
int extract_lcl(const std::string &lcl, uint64_t &seq_no, std::string &hash)
{
if (lcl == GENESIS_LEDGER)
{
seq_no = 0;
hash = lcl.substr(2);
return 0;
}
const size_t pos = lcl.find("-");
if (pos == std::string::npos)
return -1;
if (util::stoull(lcl.substr(0, pos), seq_no) == -1)
return -1;
hash = lcl.substr(pos + 1);
if (hash.size() != 64)
return -1;
return 0;
}
} // namespace ledger::ledger_sample

21
src/ledger/ledger_sample.hpp
View File

@@ -1,21 +0,0 @@
#include "../p2p/p2p.hpp"
#include "sqlite.hpp"
#include "ledger_sync.hpp"
#include "ledger_mount.hpp"
namespace ledger::ledger_sample
{
constexpr const char *GENESIS_LEDGER = "0-genesis";
extern ledger::ledger_mount ledger_fs; // Global ledger file system instance.
extern ledger::ledger_sync ledger_sync_worker; // Global ledger file system sync instance.
int init();
void deinit();
int save_ledger(const p2p::proposal &proposal);
int extract_lcl(const std::string &lcl, uint64_t &seq_no, std::string &hash);
} // namespace ledger::ledger_sample

119
src/ledger/ledger_sync.cpp
View File

@@ -1,11 +1,125 @@
#include "./ledger_sync.hpp"
#include "ledger.hpp"
namespace ledger
{
void ledger_sync::on_current_sync_state_acheived(const util::h32 &acheived_hash)
constexpr const char *HPFS_SESSION_NAME = "ro_shard_sync_status";
void ledger_sync::on_current_sync_state_acheived(const hpfs::sync_target &synced_target)
{
// Logic when a sync state is acheived can be performed here.
const std::string shard_hash_file_path = fs_mount->physical_path(hpfs::RW_SESSION_NAME, synced_target.vpath) + PREV_SHARD_HASH_FILENAME;
const int fd = open(shard_hash_file_path.c_str(), O_RDONLY | O_CLOEXEC);
if (fd == -1)
{
LOG_DEBUG << "Cannot read " << shard_hash_file_path;
return;
}
util::h32 prev_shard_hash_from_file;
const int res = read(fd, &prev_shard_hash_from_file, sizeof(util::h32));
close(fd);
if (res == -1)
{
LOG_ERROR << errno << ": Error reading hash file. " << shard_hash_file_path;
return;
}
const size_t pos = synced_target.vpath.find_last_of("/");
if (pos == std::string::npos)
{
LOG_ERROR << "Error retreiving shard no from " << synced_target.vpath;
return;
}
const std::string synced_shard_seq_no_str = synced_target.vpath.substr(pos + 1);
uint64_t synced_shard_seq_no;
if (util::stoull(synced_shard_seq_no_str, synced_shard_seq_no) == -1)
{
LOG_ERROR << "Error converting shard no from string. " << synced_shard_seq_no_str;
return;
}
util::h32 prev_shard_hash_from_hpfs;
const std::string shard_parent_dir = synced_target.vpath.substr(0, pos);
if (shard_parent_dir == PRIMARY_DIR)
{
// If the synced shard sequence number is equal or greater than the current shard seq number,
// then the context information should be updated.
uint64_t last_primary_shard_seq_no = ctx.get_last_primary_shard_id().seq_no;
if (last_primary_shard_seq_no <= synced_shard_seq_no)
{
if (get_last_ledger_and_update_context() == -1)
{
LOG_ERROR << "Error updating context from the synced shard " << synced_target.name;
return;
}
last_primary_shard_seq_no = synced_shard_seq_no;
ctx.set_last_primary_shard_id(p2p::sequence_hash{synced_shard_seq_no, synced_target.hash});
}
if (conf::cfg.node.max_shards == 0 || // Sync all shards if this is a full history node.
last_primary_shard_seq_no - synced_shard_seq_no + 1 < conf::cfg.node.max_shards)
{
// Check whether the hash of the previous shard matches with the hash in the prev_shard.hash file.
const std::string prev_shard_vpath = std::string(PRIMARY_DIR).append("/").append(std::to_string(--synced_shard_seq_no));
fs_mount->get_hash(prev_shard_hash_from_hpfs, hpfs::RW_SESSION_NAME, prev_shard_vpath);
if (prev_shard_hash_from_file != util::h32_empty // Hash in the prev_shard.hash of the 0th shard is h32 empty. Syncing should be stopped then.
&& prev_shard_hash_from_file != prev_shard_hash_from_hpfs) // Continue to sync backwards if the hash from prev_shard.hash is not matching with the shard hash from hpfs.
{
const std::string sync_name = "primary shard " + std::to_string(synced_shard_seq_no);
const std::string shard_path = std::string(PRIMARY_DIR).append("/").append(std::to_string(synced_shard_seq_no));
set_target_push_back(hpfs::sync_target{sync_name, prev_shard_hash_from_file, shard_path, hpfs::BACKLOG_ITEM_TYPE::DIR});
}
else if (conf::cfg.node.max_shards != 0 && last_primary_shard_seq_no >= conf::cfg.node.max_shards)
{
// When there are no more shards to sync, Remove old shards that exceeds max shard range.
remove_old_shards(last_primary_shard_seq_no - conf::cfg.node.max_shards + 1, PRIMARY_DIR);
}
}
else if (last_primary_shard_seq_no >= conf::cfg.node.max_shards)
{
// When there are no more shards to sync, Remove old shards that exceeds max shard range.
remove_old_shards(last_primary_shard_seq_no - conf::cfg.node.max_shards + 1, PRIMARY_DIR);
}
}
else if (shard_parent_dir == BLOB_DIR)
{
// If the synced blob shard sequence number is equal or greater than the current blob shard seq number,
// then the context information should be updated.
uint64_t last_blob_shard_seq_no = ctx.get_last_blob_shard_id().seq_no;
if (last_blob_shard_seq_no <= synced_shard_seq_no)
{
last_blob_shard_seq_no = synced_shard_seq_no;
ctx.set_last_blob_shard_id(p2p::sequence_hash{synced_shard_seq_no, synced_target.hash});
}
if (MAX_BLOB_SHARDS == 0 || // Sync all blob shards if this is a full history node.
last_blob_shard_seq_no - synced_shard_seq_no + 1 < MAX_BLOB_SHARDS)
{
// Check whether the blob hash of the previous blob shard matches with the hash in the prev_shard.hash file.
const std::string prev_shard_vpath = std::string(BLOB_DIR).append("/").append(std::to_string(--synced_shard_seq_no));
fs_mount->get_hash(prev_shard_hash_from_hpfs, hpfs::RW_SESSION_NAME, prev_shard_vpath);
if (prev_shard_hash_from_file != util::h32_empty // Hash in the prev_shard.hash of the 0th shard is h32 empty. Syncing should be stopped then.
&& prev_shard_hash_from_file != prev_shard_hash_from_hpfs) // Continue to sync backwards if the hash from prev_shard.hash is not matching with the shard hash from hpfs.
{
const std::string sync_name = "blob shard " + std::to_string(synced_shard_seq_no);
const std::string shard_path = std::string(BLOB_DIR).append("/").append(std::to_string(synced_shard_seq_no));
set_target_push_back(hpfs::sync_target{sync_name, prev_shard_hash_from_file, shard_path, hpfs::BACKLOG_ITEM_TYPE::DIR});
}
else if (MAX_BLOB_SHARDS != 0 && last_blob_shard_seq_no >= MAX_BLOB_SHARDS)
{
// When there are no more shards to sync, Remove old shards that exceeds max shard range.
remove_old_shards(last_blob_shard_seq_no - MAX_BLOB_SHARDS + 1, BLOB_DIR);
}
}
else if (last_blob_shard_seq_no >= MAX_BLOB_SHARDS)
{
// When there are no more shards to sync, Remove old shards that exceeds max shard range.
remove_old_shards(last_blob_shard_seq_no - MAX_BLOB_SHARDS + 1, BLOB_DIR);
}
}
}
void ledger_sync::swap_collected_responses()
@@ -16,4 +130,5 @@ namespace ledger
if (!p2p::ctx.collected_msgs.ledger_hpfs_responses.empty())
candidate_hpfs_responses.splice(candidate_hpfs_responses.end(), p2p::ctx.collected_msgs.ledger_hpfs_responses);
}
} // namespace ledger

2
src/ledger/ledger_sync.hpp
View File

@@ -11,8 +11,8 @@ namespace ledger
class ledger_sync : public hpfs::hpfs_sync
{
private:
void on_current_sync_state_acheived(const util::h32 &acheived_hash);
void swap_collected_responses();
void on_current_sync_state_acheived(const hpfs::sync_target &synced_target);
};
} // namespace ledger
#endif

63
src/ledger/sqlite.cpp
View File

@@ -5,7 +5,7 @@ namespace ledger::sqlite
constexpr const char *LEDGER_TABLE = "ledger";
constexpr const char *LEDGER_COLUMNS = "seq_no, time, ledger_hash, prev_ledger_hash, data_hash, state_hash, patch_hash, user_hash, input_hash, output_hash";
constexpr const char *COLUMN_DATA_TYPES[]{"INT", "TEXT"};
constexpr const char *CREATE_TABLE = "CREATE TABLE ";
constexpr const char *CREATE_TABLE = "CREATE TABLE IF NOT EXISTS ";
constexpr const char *INSERT_INTO = "INSERT INTO ";
constexpr const char *PRIMARY_KEY = "PRIMARY KEY";
constexpr const char *NOT_NULL = "NOT NULL";
@@ -13,6 +13,7 @@ namespace ledger::sqlite
constexpr const char *SELECT_ALL = "SELECT * FROM ";
constexpr const char *SQLITE_MASTER = "sqlite_master";
constexpr const char *WHERE = " WHERE ";
constexpr const char *ORDER_BY = " ORDER BY ";
constexpr const char *AND = " AND ";
/**
@@ -25,6 +26,7 @@ namespace ledger::sqlite
{
if (sqlite3_open(db_name.data(), db) != SQLITE_OK)
{
*db = NULL;
LOG_ERROR << "Can't open database: " << sqlite3_errmsg(*db);
return -1;
}
@@ -179,14 +181,33 @@ namespace ledger::sqlite
if (sqlite3_prepare_v2(db, sql.data(), -1, &stmt, 0) == SQLITE_OK &&
stmt != NULL && sqlite3_step(stmt) == SQLITE_ROW)
{
sqlite3_reset(stmt);
// Finalize and distroys the statement.
sqlite3_finalize(stmt);
return true;
}
sqlite3_reset(stmt);
// Finalize and distroys the statement.
sqlite3_finalize(stmt);
return false;
}
/**
* Closes a connection to a given databse.
* @param db Pointer to the db.
* @returns returns 0 on success, or -1 on error.
*/
int close_db(sqlite3 **db)
{
if (sqlite3_close(*db) != SQLITE_OK)
{
LOG_ERROR << "Can't close database: " << sqlite3_errmsg(*db);
return -1;
}
*db = NULL;
return 0;
}
/**
* Creates a table for ledger records.
* @param db Pointer to the db.
@@ -246,4 +267,40 @@ namespace ledger::sqlite
{
return is_table_exists(db, LEDGER_TABLE);
}
/**
* Get the last ledger record of the given db.
* @param db Pointer to the db.
* @returns returns the last ledger as a struct.
*/
ledger get_last_ledger(sqlite3 *db)
{
std::string sql;
sql.append(SELECT_ALL);
sql.append(LEDGER_TABLE);
sql.append(ORDER_BY);
sql.append("seq_no DESC LIMIT 1");
sqlite3_stmt *stmt;
sqlite::ledger ledger;
if (sqlite3_prepare_v2(db, sql.data(), -1, &stmt, 0) == SQLITE_OK &&
stmt != NULL && sqlite3_step(stmt) == SQLITE_ROW)
{
// Finalize and distroys the statement.
ledger.seq_no = sqlite3_column_int64(stmt, 0);
ledger.time = sqlite3_column_int64(stmt, 1);
ledger.ledger_hash_hex = std::string((char *)sqlite3_column_text(stmt, 2));
ledger.prev_ledger_hash_hex = std::string((char *)sqlite3_column_text(stmt, 3));
ledger.data_hash_hex = std::string((char *)sqlite3_column_text(stmt, 4));
ledger.state_hash_hex = std::string((char *)sqlite3_column_text(stmt, 5));
ledger.patch_hash_hex = std::string((char *)sqlite3_column_text(stmt, 6));
ledger.user_hash_hex = std::string((char *)sqlite3_column_text(stmt, 7));
ledger.input_hash_hex = std::string((char *)sqlite3_column_text(stmt, 8));
ledger.output_hash_hex = std::string((char *)sqlite3_column_text(stmt, 9));
}
sqlite3_finalize(stmt);
return ledger;
}
} // namespace ledger::sqlite

8
src/ledger/sqlite.hpp
View File

@@ -54,6 +54,8 @@ namespace ledger::sqlite
std::string input_hash_hex;
std::string output_hash_hex;
ledger() {};
ledger(
const uint64_t seq_no,
const uint64_t time,
@@ -92,13 +94,17 @@ namespace ledger::sqlite
bool is_table_exists(sqlite3 *db, std::string_view table_name);
int close_db(sqlite3 **db);
// Ledger specific methdods.
int create_ledger_table(sqlite3 *db);
int insert_ledger_row(sqlite3 *db, const ledger &ledger);
bool is_ledger_table_exist(sqlite3 *db);
ledger get_last_ledger(sqlite3 *db);
} // namespace ledger::sqlite
#endif

5
src/main.cpp
View File

@@ -12,8 +12,7 @@
#include "usr/read_req.hpp"
#include "p2p/p2p.hpp"
#include "consensus.hpp"
#include "ledger.hpp"
#include "ledger/ledger_sample.hpp"
#include "ledger/ledger.hpp"
#include "unl.hpp"
/**
@@ -71,7 +70,6 @@ void deinit()
p2p::deinit();
read_req::deinit();
consensus::deinit();
ledger::ledger_sample::deinit(); // Deinit method in new ledger implementation.
sc::deinit();
ledger::deinit();
conf::deinit();
@@ -197,7 +195,6 @@ int main(int argc, char **argv)
LOG_INFO << "Contract: " << conf::cfg.contract.id << " (" << conf::cfg.contract.version << ")";
if (sc::init() == -1 ||
ledger::ledger_sample::init() == -1 || // Init method of new ledger implementaiton.
ledger::init() == -1 ||
unl::init() == -1 ||
consensus::init() == -1 ||

14
src/msg/fbuf/fullhistory_schema.fbs
View File

@@ -1,14 +0,0 @@
namespace msg.fbuf.ledger;
table FullHistoryBlock {
version:string;
raw_inputs: [RawInput];
}
table RawInput {
hash:[ubyte];
pubkey:[ubyte];
input:[ubyte];
}
root_type FullHistoryBlock;

217
src/msg/fbuf/fullhistory_schema_generated.h
View File

@@ -1,217 +0,0 @@
// automatically generated by the FlatBuffers compiler, do not modify
#ifndef FLATBUFFERS_GENERATED_FULLHISTORYSCHEMA_MSG_FBUF_LEDGER_H_
#define FLATBUFFERS_GENERATED_FULLHISTORYSCHEMA_MSG_FBUF_LEDGER_H_
#include "flatbuffers/flatbuffers.h"
namespace msg {
namespace fbuf {
namespace ledger {
struct FullHistoryBlock;
struct FullHistoryBlockBuilder;
struct RawInput;
struct RawInputBuilder;
struct FullHistoryBlock FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
typedef FullHistoryBlockBuilder Builder;
enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
VT_VERSION = 4,
VT_RAW_INPUTS = 6
};
const flatbuffers::String *version() const {
return GetPointer<const flatbuffers::String *>(VT_VERSION);
}
flatbuffers::String *mutable_version() {
return GetPointer<flatbuffers::String *>(VT_VERSION);
}
const flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ledger::RawInput>> *raw_inputs() const {
return GetPointer<const flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ledger::RawInput>> *>(VT_RAW_INPUTS);
}
flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ledger::RawInput>> *mutable_raw_inputs() {
return GetPointer<flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ledger::RawInput>> *>(VT_RAW_INPUTS);
}
bool Verify(flatbuffers::Verifier &verifier) const {
return VerifyTableStart(verifier) &&
VerifyOffset(verifier, VT_VERSION) &&
verifier.VerifyString(version()) &&
VerifyOffset(verifier, VT_RAW_INPUTS) &&
verifier.VerifyVector(raw_inputs()) &&
verifier.VerifyVectorOfTables(raw_inputs()) &&
verifier.EndTable();
}
};
struct FullHistoryBlockBuilder {
typedef FullHistoryBlock Table;
flatbuffers::FlatBufferBuilder &fbb_;
flatbuffers::uoffset_t start_;
void add_version(flatbuffers::Offset<flatbuffers::String> version) {
fbb_.AddOffset(FullHistoryBlock::VT_VERSION, version);
}
void add_raw_inputs(flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ledger::RawInput>>> raw_inputs) {
fbb_.AddOffset(FullHistoryBlock::VT_RAW_INPUTS, raw_inputs);
}
explicit FullHistoryBlockBuilder(flatbuffers::FlatBufferBuilder &_fbb)
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
flatbuffers::Offset<FullHistoryBlock> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<FullHistoryBlock>(end);
return o;
}
};
inline flatbuffers::Offset<FullHistoryBlock> CreateFullHistoryBlock(
flatbuffers::FlatBufferBuilder &_fbb,
flatbuffers::Offset<flatbuffers::String> version = 0,
flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ledger::RawInput>>> raw_inputs = 0) {
FullHistoryBlockBuilder builder_(_fbb);
builder_.add_raw_inputs(raw_inputs);
builder_.add_version(version);
return builder_.Finish();
}
inline flatbuffers::Offset<FullHistoryBlock> CreateFullHistoryBlockDirect(
flatbuffers::FlatBufferBuilder &_fbb,
const char *version = nullptr,
const std::vector<flatbuffers::Offset<msg::fbuf::ledger::RawInput>> *raw_inputs = nullptr) {
auto version__ = version ? _fbb.CreateString(version) : 0;
auto raw_inputs__ = raw_inputs ? _fbb.CreateVector<flatbuffers::Offset<msg::fbuf::ledger::RawInput>>(*raw_inputs) : 0;
return msg::fbuf::ledger::CreateFullHistoryBlock(
_fbb,
version__,
raw_inputs__);
}
struct RawInput FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
typedef RawInputBuilder Builder;
enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
VT_HASH = 4,
VT_PUBKEY = 6,
VT_INPUT = 8
};
const flatbuffers::Vector<uint8_t> *hash() const {
return GetPointer<const flatbuffers::Vector<uint8_t> *>(VT_HASH);
}
flatbuffers::Vector<uint8_t> *mutable_hash() {
return GetPointer<flatbuffers::Vector<uint8_t> *>(VT_HASH);
}
const flatbuffers::Vector<uint8_t> *pubkey() const {
return GetPointer<const flatbuffers::Vector<uint8_t> *>(VT_PUBKEY);
}
flatbuffers::Vector<uint8_t> *mutable_pubkey() {
return GetPointer<flatbuffers::Vector<uint8_t> *>(VT_PUBKEY);
}
const flatbuffers::Vector<uint8_t> *input() const {
return GetPointer<const flatbuffers::Vector<uint8_t> *>(VT_INPUT);
}
flatbuffers::Vector<uint8_t> *mutable_input() {
return GetPointer<flatbuffers::Vector<uint8_t> *>(VT_INPUT);
}
bool Verify(flatbuffers::Verifier &verifier) const {
return VerifyTableStart(verifier) &&
VerifyOffset(verifier, VT_HASH) &&
verifier.VerifyVector(hash()) &&
VerifyOffset(verifier, VT_PUBKEY) &&
verifier.VerifyVector(pubkey()) &&
VerifyOffset(verifier, VT_INPUT) &&
verifier.VerifyVector(input()) &&
verifier.EndTable();
}
};
struct RawInputBuilder {
typedef RawInput Table;
flatbuffers::FlatBufferBuilder &fbb_;
flatbuffers::uoffset_t start_;
void add_hash(flatbuffers::Offset<flatbuffers::Vector<uint8_t>> hash) {
fbb_.AddOffset(RawInput::VT_HASH, hash);
}
void add_pubkey(flatbuffers::Offset<flatbuffers::Vector<uint8_t>> pubkey) {
fbb_.AddOffset(RawInput::VT_PUBKEY, pubkey);
}
void add_input(flatbuffers::Offset<flatbuffers::Vector<uint8_t>> input) {
fbb_.AddOffset(RawInput::VT_INPUT, input);
}
explicit RawInputBuilder(flatbuffers::FlatBufferBuilder &_fbb)
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
flatbuffers::Offset<RawInput> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<RawInput>(end);
return o;
}
};
inline flatbuffers::Offset<RawInput> CreateRawInput(
flatbuffers::FlatBufferBuilder &_fbb,
flatbuffers::Offset<flatbuffers::Vector<uint8_t>> hash = 0,
flatbuffers::Offset<flatbuffers::Vector<uint8_t>> pubkey = 0,
flatbuffers::Offset<flatbuffers::Vector<uint8_t>> input = 0) {
RawInputBuilder builder_(_fbb);
builder_.add_input(input);
builder_.add_pubkey(pubkey);
builder_.add_hash(hash);
return builder_.Finish();
}
inline flatbuffers::Offset<RawInput> CreateRawInputDirect(
flatbuffers::FlatBufferBuilder &_fbb,
const std::vector<uint8_t> *hash = nullptr,
const std::vector<uint8_t> *pubkey = nullptr,
const std::vector<uint8_t> *input = nullptr) {
auto hash__ = hash ? _fbb.CreateVector<uint8_t>(*hash) : 0;
auto pubkey__ = pubkey ? _fbb.CreateVector<uint8_t>(*pubkey) : 0;
auto input__ = input ? _fbb.CreateVector<uint8_t>(*input) : 0;
return msg::fbuf::ledger::CreateRawInput(
_fbb,
hash__,
pubkey__,
input__);
}
inline const msg::fbuf::ledger::FullHistoryBlock *GetFullHistoryBlock(const void *buf) {
return flatbuffers::GetRoot<msg::fbuf::ledger::FullHistoryBlock>(buf);
}
inline const msg::fbuf::ledger::FullHistoryBlock *GetSizePrefixedFullHistoryBlock(const void *buf) {
return flatbuffers::GetSizePrefixedRoot<msg::fbuf::ledger::FullHistoryBlock>(buf);
}
inline FullHistoryBlock *GetMutableFullHistoryBlock(void *buf) {
return flatbuffers::GetMutableRoot<FullHistoryBlock>(buf);
}
inline bool VerifyFullHistoryBlockBuffer(
flatbuffers::Verifier &verifier) {
return verifier.VerifyBuffer<msg::fbuf::ledger::FullHistoryBlock>(nullptr);
}
inline bool VerifySizePrefixedFullHistoryBlockBuffer(
flatbuffers::Verifier &verifier) {
return verifier.VerifySizePrefixedBuffer<msg::fbuf::ledger::FullHistoryBlock>(nullptr);
}
inline void FinishFullHistoryBlockBuffer(
flatbuffers::FlatBufferBuilder &fbb,
flatbuffers::Offset<msg::fbuf::ledger::FullHistoryBlock> root) {
fbb.Finish(root);
}
inline void FinishSizePrefixedFullHistoryBlockBuffer(
flatbuffers::FlatBufferBuilder &fbb,
flatbuffers::Offset<msg::fbuf::ledger::FullHistoryBlock> root) {
fbb.FinishSizePrefixed(root);
}
} // namespace ledger
} // namespace fbuf
} // namespace msg
#endif // FLATBUFFERS_GENERATED_FULLHISTORYSCHEMA_MSG_FBUF_LEDGER_H_

27
src/msg/fbuf/ledger_blob_schema.fbs Normal file
View File

@@ -0,0 +1,27 @@
namespace msg.fbuf.ledgermsg;
table LedgerBlob {
ledger_hash:[ubyte];
raw_inputs:[RawInputCollection];
raw_outputs:[RawOutputCollection];
}
table RawInputCollection {
pubkey:[ubyte];
inputs:[RawInput];
}
table RawInput {
input: [ubyte];
}
table RawOutputCollection {
pubkey:[ubyte];
outputs:[RawOutput];
}
table RawOutput {
output: [ubyte];
}
root_type LedgerBlob;

408
src/msg/fbuf/ledger_blob_schema_generated.h Normal file
View File

@@ -0,0 +1,408 @@
// automatically generated by the FlatBuffers compiler, do not modify
#ifndef FLATBUFFERS_GENERATED_LEDGERBLOBSCHEMA_MSG_FBUF_LEDGERMSG_H_
#define FLATBUFFERS_GENERATED_LEDGERBLOBSCHEMA_MSG_FBUF_LEDGERMSG_H_
#include "flatbuffers/flatbuffers.h"
namespace msg {
namespace fbuf {
namespace ledgermsg {
struct LedgerBlob;
struct LedgerBlobBuilder;
struct RawInputCollection;
struct RawInputCollectionBuilder;
struct RawInput;
struct RawInputBuilder;
struct RawOutputCollection;
struct RawOutputCollectionBuilder;
struct RawOutput;
struct RawOutputBuilder;
struct LedgerBlob FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
typedef LedgerBlobBuilder Builder;
enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
VT_LEDGER_HASH = 4,
VT_RAW_INPUTS = 6,
VT_RAW_OUTPUTS = 8
};
const flatbuffers::Vector<uint8_t> *ledger_hash() const {
return GetPointer<const flatbuffers::Vector<uint8_t> *>(VT_LEDGER_HASH);
}
flatbuffers::Vector<uint8_t> *mutable_ledger_hash() {
return GetPointer<flatbuffers::Vector<uint8_t> *>(VT_LEDGER_HASH);
}
const flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ledgermsg::RawInputCollection>> *raw_inputs() const {
return GetPointer<const flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ledgermsg::RawInputCollection>> *>(VT_RAW_INPUTS);
}
flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ledgermsg::RawInputCollection>> *mutable_raw_inputs() {
return GetPointer<flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ledgermsg::RawInputCollection>> *>(VT_RAW_INPUTS);
}
const flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ledgermsg::RawOutputCollection>> *raw_outputs() const {
return GetPointer<const flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ledgermsg::RawOutputCollection>> *>(VT_RAW_OUTPUTS);
}
flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ledgermsg::RawOutputCollection>> *mutable_raw_outputs() {
return GetPointer<flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ledgermsg::RawOutputCollection>> *>(VT_RAW_OUTPUTS);
}
bool Verify(flatbuffers::Verifier &verifier) const {
return VerifyTableStart(verifier) &&
VerifyOffset(verifier, VT_LEDGER_HASH) &&
verifier.VerifyVector(ledger_hash()) &&
VerifyOffset(verifier, VT_RAW_INPUTS) &&
verifier.VerifyVector(raw_inputs()) &&
verifier.VerifyVectorOfTables(raw_inputs()) &&
VerifyOffset(verifier, VT_RAW_OUTPUTS) &&
verifier.VerifyVector(raw_outputs()) &&
verifier.VerifyVectorOfTables(raw_outputs()) &&
verifier.EndTable();
}
};
struct LedgerBlobBuilder {
typedef LedgerBlob Table;
flatbuffers::FlatBufferBuilder &fbb_;
flatbuffers::uoffset_t start_;
void add_ledger_hash(flatbuffers::Offset<flatbuffers::Vector<uint8_t>> ledger_hash) {
fbb_.AddOffset(LedgerBlob::VT_LEDGER_HASH, ledger_hash);
}
void add_raw_inputs(flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ledgermsg::RawInputCollection>>> raw_inputs) {
fbb_.AddOffset(LedgerBlob::VT_RAW_INPUTS, raw_inputs);
}
void add_raw_outputs(flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ledgermsg::RawOutputCollection>>> raw_outputs) {
fbb_.AddOffset(LedgerBlob::VT_RAW_OUTPUTS, raw_outputs);
}
explicit LedgerBlobBuilder(flatbuffers::FlatBufferBuilder &_fbb)
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
flatbuffers::Offset<LedgerBlob> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<LedgerBlob>(end);
return o;
}
};
inline flatbuffers::Offset<LedgerBlob> CreateLedgerBlob(
flatbuffers::FlatBufferBuilder &_fbb,
flatbuffers::Offset<flatbuffers::Vector<uint8_t>> ledger_hash = 0,
flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ledgermsg::RawInputCollection>>> raw_inputs = 0,
flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ledgermsg::RawOutputCollection>>> raw_outputs = 0) {
LedgerBlobBuilder builder_(_fbb);
builder_.add_raw_outputs(raw_outputs);
builder_.add_raw_inputs(raw_inputs);
builder_.add_ledger_hash(ledger_hash);
return builder_.Finish();
}
inline flatbuffers::Offset<LedgerBlob> CreateLedgerBlobDirect(
flatbuffers::FlatBufferBuilder &_fbb,
const std::vector<uint8_t> *ledger_hash = nullptr,
const std::vector<flatbuffers::Offset<msg::fbuf::ledgermsg::RawInputCollection>> *raw_inputs = nullptr,
const std::vector<flatbuffers::Offset<msg::fbuf::ledgermsg::RawOutputCollection>> *raw_outputs = nullptr) {
auto ledger_hash__ = ledger_hash ? _fbb.CreateVector<uint8_t>(*ledger_hash) : 0;
auto raw_inputs__ = raw_inputs ? _fbb.CreateVector<flatbuffers::Offset<msg::fbuf::ledgermsg::RawInputCollection>>(*raw_inputs) : 0;
auto raw_outputs__ = raw_outputs ? _fbb.CreateVector<flatbuffers::Offset<msg::fbuf::ledgermsg::RawOutputCollection>>(*raw_outputs) : 0;
return msg::fbuf::ledgermsg::CreateLedgerBlob(
_fbb,
ledger_hash__,
raw_inputs__,
raw_outputs__);
}
struct RawInputCollection FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
typedef RawInputCollectionBuilder Builder;
enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
VT_PUBKEY = 4,
VT_INPUTS = 6
};
const flatbuffers::Vector<uint8_t> *pubkey() const {
return GetPointer<const flatbuffers::Vector<uint8_t> *>(VT_PUBKEY);
}
flatbuffers::Vector<uint8_t> *mutable_pubkey() {
return GetPointer<flatbuffers::Vector<uint8_t> *>(VT_PUBKEY);
}
const flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ledgermsg::RawInput>> *inputs() const {
return GetPointer<const flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ledgermsg::RawInput>> *>(VT_INPUTS);
}
flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ledgermsg::RawInput>> *mutable_inputs() {
return GetPointer<flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ledgermsg::RawInput>> *>(VT_INPUTS);
}
bool Verify(flatbuffers::Verifier &verifier) const {
return VerifyTableStart(verifier) &&
VerifyOffset(verifier, VT_PUBKEY) &&
verifier.VerifyVector(pubkey()) &&
VerifyOffset(verifier, VT_INPUTS) &&
verifier.VerifyVector(inputs()) &&
verifier.VerifyVectorOfTables(inputs()) &&
verifier.EndTable();
}
};
struct RawInputCollectionBuilder {
typedef RawInputCollection Table;
flatbuffers::FlatBufferBuilder &fbb_;
flatbuffers::uoffset_t start_;
void add_pubkey(flatbuffers::Offset<flatbuffers::Vector<uint8_t>> pubkey) {
fbb_.AddOffset(RawInputCollection::VT_PUBKEY, pubkey);
}
void add_inputs(flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ledgermsg::RawInput>>> inputs) {
fbb_.AddOffset(RawInputCollection::VT_INPUTS, inputs);
}
explicit RawInputCollectionBuilder(flatbuffers::FlatBufferBuilder &_fbb)
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
flatbuffers::Offset<RawInputCollection> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<RawInputCollection>(end);
return o;
}
};
inline flatbuffers::Offset<RawInputCollection> CreateRawInputCollection(
flatbuffers::FlatBufferBuilder &_fbb,
flatbuffers::Offset<flatbuffers::Vector<uint8_t>> pubkey = 0,
flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ledgermsg::RawInput>>> inputs = 0) {
RawInputCollectionBuilder builder_(_fbb);
builder_.add_inputs(inputs);
builder_.add_pubkey(pubkey);
return builder_.Finish();
}
inline flatbuffers::Offset<RawInputCollection> CreateRawInputCollectionDirect(
flatbuffers::FlatBufferBuilder &_fbb,
const std::vector<uint8_t> *pubkey = nullptr,
const std::vector<flatbuffers::Offset<msg::fbuf::ledgermsg::RawInput>> *inputs = nullptr) {
auto pubkey__ = pubkey ? _fbb.CreateVector<uint8_t>(*pubkey) : 0;
auto inputs__ = inputs ? _fbb.CreateVector<flatbuffers::Offset<msg::fbuf::ledgermsg::RawInput>>(*inputs) : 0;
return msg::fbuf::ledgermsg::CreateRawInputCollection(
_fbb,
pubkey__,
inputs__);
}
struct RawInput FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
typedef RawInputBuilder Builder;
enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
VT_INPUT = 4
};
const flatbuffers::Vector<uint8_t> *input() const {
return GetPointer<const flatbuffers::Vector<uint8_t> *>(VT_INPUT);
}
flatbuffers::Vector<uint8_t> *mutable_input() {
return GetPointer<flatbuffers::Vector<uint8_t> *>(VT_INPUT);
}
bool Verify(flatbuffers::Verifier &verifier) const {
return VerifyTableStart(verifier) &&
VerifyOffset(verifier, VT_INPUT) &&
verifier.VerifyVector(input()) &&
verifier.EndTable();
}
};
struct RawInputBuilder {
typedef RawInput Table;
flatbuffers::FlatBufferBuilder &fbb_;
flatbuffers::uoffset_t start_;
void add_input(flatbuffers::Offset<flatbuffers::Vector<uint8_t>> input) {
fbb_.AddOffset(RawInput::VT_INPUT, input);
}
explicit RawInputBuilder(flatbuffers::FlatBufferBuilder &_fbb)
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
flatbuffers::Offset<RawInput> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<RawInput>(end);
return o;
}
};
inline flatbuffers::Offset<RawInput> CreateRawInput(
flatbuffers::FlatBufferBuilder &_fbb,
flatbuffers::Offset<flatbuffers::Vector<uint8_t>> input = 0) {
RawInputBuilder builder_(_fbb);
builder_.add_input(input);
return builder_.Finish();
}
inline flatbuffers::Offset<RawInput> CreateRawInputDirect(
flatbuffers::FlatBufferBuilder &_fbb,
const std::vector<uint8_t> *input = nullptr) {
auto input__ = input ? _fbb.CreateVector<uint8_t>(*input) : 0;
return msg::fbuf::ledgermsg::CreateRawInput(
_fbb,
input__);
}
struct RawOutputCollection FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
typedef RawOutputCollectionBuilder Builder;
enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
VT_PUBKEY = 4,
VT_OUTPUTS = 6
};
const flatbuffers::Vector<uint8_t> *pubkey() const {
return GetPointer<const flatbuffers::Vector<uint8_t> *>(VT_PUBKEY);
}
flatbuffers::Vector<uint8_t> *mutable_pubkey() {
return GetPointer<flatbuffers::Vector<uint8_t> *>(VT_PUBKEY);
}
const flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ledgermsg::RawOutput>> *outputs() const {
return GetPointer<const flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ledgermsg::RawOutput>> *>(VT_OUTPUTS);
}
flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ledgermsg::RawOutput>> *mutable_outputs() {
return GetPointer<flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ledgermsg::RawOutput>> *>(VT_OUTPUTS);
}
bool Verify(flatbuffers::Verifier &verifier) const {
return VerifyTableStart(verifier) &&
VerifyOffset(verifier, VT_PUBKEY) &&
verifier.VerifyVector(pubkey()) &&
VerifyOffset(verifier, VT_OUTPUTS) &&
verifier.VerifyVector(outputs()) &&
verifier.VerifyVectorOfTables(outputs()) &&
verifier.EndTable();
}
};
struct RawOutputCollectionBuilder {
typedef RawOutputCollection Table;
flatbuffers::FlatBufferBuilder &fbb_;
flatbuffers::uoffset_t start_;
void add_pubkey(flatbuffers::Offset<flatbuffers::Vector<uint8_t>> pubkey) {
fbb_.AddOffset(RawOutputCollection::VT_PUBKEY, pubkey);
}
void add_outputs(flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ledgermsg::RawOutput>>> outputs) {
fbb_.AddOffset(RawOutputCollection::VT_OUTPUTS, outputs);
}
explicit RawOutputCollectionBuilder(flatbuffers::FlatBufferBuilder &_fbb)
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
flatbuffers::Offset<RawOutputCollection> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<RawOutputCollection>(end);
return o;
}
};
inline flatbuffers::Offset<RawOutputCollection> CreateRawOutputCollection(
flatbuffers::FlatBufferBuilder &_fbb,
flatbuffers::Offset<flatbuffers::Vector<uint8_t>> pubkey = 0,
flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ledgermsg::RawOutput>>> outputs = 0) {
RawOutputCollectionBuilder builder_(_fbb);
builder_.add_outputs(outputs);
builder_.add_pubkey(pubkey);
return builder_.Finish();
}
inline flatbuffers::Offset<RawOutputCollection> CreateRawOutputCollectionDirect(
flatbuffers::FlatBufferBuilder &_fbb,
const std::vector<uint8_t> *pubkey = nullptr,
const std::vector<flatbuffers::Offset<msg::fbuf::ledgermsg::RawOutput>> *outputs = nullptr) {
auto pubkey__ = pubkey ? _fbb.CreateVector<uint8_t>(*pubkey) : 0;
auto outputs__ = outputs ? _fbb.CreateVector<flatbuffers::Offset<msg::fbuf::ledgermsg::RawOutput>>(*outputs) : 0;
return msg::fbuf::ledgermsg::CreateRawOutputCollection(
_fbb,
pubkey__,
outputs__);
}
struct RawOutput FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
typedef RawOutputBuilder Builder;
enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
VT_OUTPUT = 4
};
const flatbuffers::Vector<uint8_t> *output() const {
return GetPointer<const flatbuffers::Vector<uint8_t> *>(VT_OUTPUT);
}
flatbuffers::Vector<uint8_t> *mutable_output() {
return GetPointer<flatbuffers::Vector<uint8_t> *>(VT_OUTPUT);
}
bool Verify(flatbuffers::Verifier &verifier) const {
return VerifyTableStart(verifier) &&
VerifyOffset(verifier, VT_OUTPUT) &&
verifier.VerifyVector(output()) &&
verifier.EndTable();
}
};
struct RawOutputBuilder {
typedef RawOutput Table;
flatbuffers::FlatBufferBuilder &fbb_;
flatbuffers::uoffset_t start_;
void add_output(flatbuffers::Offset<flatbuffers::Vector<uint8_t>> output) {
fbb_.AddOffset(RawOutput::VT_OUTPUT, output);
}
explicit RawOutputBuilder(flatbuffers::FlatBufferBuilder &_fbb)
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
flatbuffers::Offset<RawOutput> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<RawOutput>(end);
return o;
}
};
inline flatbuffers::Offset<RawOutput> CreateRawOutput(
flatbuffers::FlatBufferBuilder &_fbb,
flatbuffers::Offset<flatbuffers::Vector<uint8_t>> output = 0) {
RawOutputBuilder builder_(_fbb);
builder_.add_output(output);
return builder_.Finish();
}
inline flatbuffers::Offset<RawOutput> CreateRawOutputDirect(
flatbuffers::FlatBufferBuilder &_fbb,
const std::vector<uint8_t> *output = nullptr) {
auto output__ = output ? _fbb.CreateVector<uint8_t>(*output) : 0;
return msg::fbuf::ledgermsg::CreateRawOutput(
_fbb,
output__);
}
inline const msg::fbuf::ledgermsg::LedgerBlob *GetLedgerBlob(const void *buf) {
return flatbuffers::GetRoot<msg::fbuf::ledgermsg::LedgerBlob>(buf);
}
inline const msg::fbuf::ledgermsg::LedgerBlob *GetSizePrefixedLedgerBlob(const void *buf) {
return flatbuffers::GetSizePrefixedRoot<msg::fbuf::ledgermsg::LedgerBlob>(buf);
}
inline LedgerBlob *GetMutableLedgerBlob(void *buf) {
return flatbuffers::GetMutableRoot<LedgerBlob>(buf);
}
inline bool VerifyLedgerBlobBuffer(
flatbuffers::Verifier &verifier) {
return verifier.VerifyBuffer<msg::fbuf::ledgermsg::LedgerBlob>(nullptr);
}
inline bool VerifySizePrefixedLedgerBlobBuffer(
flatbuffers::Verifier &verifier) {
return verifier.VerifySizePrefixedBuffer<msg::fbuf::ledgermsg::LedgerBlob>(nullptr);
}
inline void FinishLedgerBlobBuffer(
flatbuffers::FlatBufferBuilder &fbb,
flatbuffers::Offset<msg::fbuf::ledgermsg::LedgerBlob> root) {
fbb.Finish(root);
}
inline void FinishSizePrefixedLedgerBlobBuffer(
flatbuffers::FlatBufferBuilder &fbb,
flatbuffers::Offset<msg::fbuf::ledgermsg::LedgerBlob> root) {
fbb.FinishSizePrefixed(root);
}
} // namespace ledgermsg
} // namespace fbuf
} // namespace msg
#endif // FLATBUFFERS_GENERATED_LEDGERBLOBSCHEMA_MSG_FBUF_LEDGERMSG_H_

80
src/msg/fbuf/ledger_helpers.cpp
View File

@@ -1,11 +1,11 @@
#include "../../pchheader.hpp"
#include "../../p2p/p2p.hpp"
#include "ledger_schema_generated.h"
#include "fullhistory_schema_generated.h"
#include "ledger_blob_schema_generated.h"
#include "common_helpers.hpp"
#include "ledger_helpers.hpp"
namespace msg::fbuf::ledger
namespace msg::fbuf::ledgermsg
{
/**
* Create ledger block from the given proposal struct.
@@ -13,8 +13,8 @@ namespace msg::fbuf::ledger
*/
void create_ledger_block_from_proposal(flatbuffers::FlatBufferBuilder &builder, const p2p::proposal &p, const uint64_t seq_no)
{
flatbuffers::Offset<ledger::LedgerBlock> ledger =
ledger::CreateLedgerBlock(
flatbuffers::Offset<ledgermsg::LedgerBlock> ledger =
ledgermsg::CreateLedgerBlock(
builder,
sv_to_flatbuff_str(builder, conf::cfg.hp_version),
seq_no,
@@ -31,7 +31,7 @@ namespace msg::fbuf::ledger
p2p::proposal create_proposal_from_ledger_block(const std::vector<uint8_t> &ledger_buf)
{
auto ledger = msg::fbuf::ledger::GetLedgerBlock(ledger_buf.data());
auto ledger = msg::fbuf::ledgermsg::GetLedgerBlock(ledger_buf.data());
p2p::proposal p;
p.lcl = flatbuff_bytes_to_sv(ledger->lcl());
p.state_hash = flatbuff_bytes_to_hash(ledger->state_hash());
@@ -47,45 +47,51 @@ namespace msg::fbuf::ledger
}
/**
* Create full history block from the given raw input map.
* @param map The raw input map to be placed in full history.
* Create ledger blob msg from ledger blob struct.
* @param ledger_blob Ledger blob to be placed in file.
*/
void create_full_history_block_from_raw_input_map(flatbuffers::FlatBufferBuilder &builder, const std::unordered_map<std::string, usr::raw_user_input> &map)
void create_ledger_blob_msg_from_ledger_blob(flatbuffers::FlatBufferBuilder &builder, const ledger::ledger_blob &ledger_blob)
{
std::vector<flatbuffers::Offset<msg::fbuf::ledger::RawInput>> fbvec;
fbvec.reserve(map.size());
for (auto const &[key, value] : map)
std::vector<flatbuffers::Offset<msg::fbuf::ledgermsg::RawInputCollection>> raw_inputs;
raw_inputs.reserve(ledger_blob.inputs.size());
std::vector<flatbuffers::Offset<msg::fbuf::ledgermsg::RawOutputCollection>> raw_outputs;
raw_outputs.reserve(ledger_blob.outputs.size());
for (const auto &[key, value] : ledger_blob.inputs)
{
fbvec.push_back(ledger::CreateRawInput(
std::vector<flatbuffers::Offset<msg::fbuf::ledgermsg::RawInput>> inputs;
inputs.reserve(value.size());
for (const auto &input : value)
inputs.push_back(ledgermsg::CreateRawInput(builder, sv_to_flatbuff_bytes(builder, input)));
raw_inputs.push_back(ledgermsg::CreateRawInputCollection(
builder,
sv_to_flatbuff_bytes(builder, key),
sv_to_flatbuff_bytes(builder, value.pubkey),
sv_to_flatbuff_bytes(builder, value.input)));
builder.CreateVector(inputs)));
}
flatbuffers::Offset<ledger::FullHistoryBlock> fullhistory =
ledger::CreateFullHistoryBlock(
builder,
sv_to_flatbuff_str(builder, conf::cfg.hp_version),
builder.CreateVector(fbvec));
builder.Finish(fullhistory); // Finished building message content to get serialised content.
}
const std::unordered_map<std::string, usr::raw_user_input> create_raw_input_map_from_full_history_block(const std::vector<uint8_t> &fullhist_buf)
{
const auto fullhistory = msg::fbuf::ledger::GetFullHistoryBlock(fullhist_buf.data());
const auto fbvec = fullhistory->raw_inputs();
std::unordered_map<std::string, usr::raw_user_input> map;
map.reserve(fbvec->size());
for (auto el : *fbvec)
for (const auto &[key, value] : ledger_blob.outputs)
{
map.emplace(flatbuff_bytes_to_sv(el->hash()),
usr::raw_user_input{
std::string(flatbuff_bytes_to_sv(el->pubkey())),
std::string(flatbuff_bytes_to_sv(el->input()))});
std::vector<flatbuffers::Offset<msg::fbuf::ledgermsg::RawOutput>> outputs;
outputs.reserve(value.size());
for (const auto &output : value)
outputs.push_back(ledgermsg::CreateRawOutput(builder, sv_to_flatbuff_bytes(builder, output)));
raw_outputs.push_back(ledgermsg::CreateRawOutputCollection(
builder,
sv_to_flatbuff_bytes(builder, key),
builder.CreateVector(outputs)));
}
return map;
flatbuffers::Offset<ledgermsg::LedgerBlob> blob =
ledgermsg::CreateLedgerBlob(
builder,
sv_to_flatbuff_bytes(builder, ledger_blob.ledger_hash),
builder.CreateVector(raw_inputs),
builder.CreateVector(raw_outputs));
builder.Finish(blob); // Finished building message content to get serialised content.
}
} // namespace msg::fbuf::ledger
} // namespace msg::fbuf::ledgermsg

8
src/msg/fbuf/ledger_helpers.hpp
View File

@@ -3,9 +3,10 @@
#include "../../pchheader.hpp"
#include "../../p2p/p2p.hpp"
#include "../../ledger/ledger.hpp"
#include "ledger_schema_generated.h"
namespace msg::fbuf::ledger
namespace msg::fbuf::ledgermsg
{
void create_ledger_block_from_proposal(flatbuffers::FlatBufferBuilder &builder, const p2p::proposal &p, const uint64_t seq_no);
@@ -14,9 +15,8 @@ namespace msg::fbuf::ledger
bool verify_ledger_block_buffer(const uint8_t *ledger_buf_ptr, const size_t buf_len);
void create_full_history_block_from_raw_input_map(flatbuffers::FlatBufferBuilder &builder, const std::unordered_map<std::string, usr::raw_user_input> &map);
void create_ledger_blob_msg_from_ledger_blob(flatbuffers::FlatBufferBuilder &builder, const ledger::ledger_blob &ledger_blob);
const std::unordered_map<std::string, usr::raw_user_input> create_raw_input_map_from_full_history_block(const std::vector<uint8_t> &fullhist_buf);
} // namespace msg::fbuf::ledger
} // namespace msg::fbuf::ledgermsg
#endif

2
src/msg/fbuf/ledger_schema.fbs
View File

@@ -1,6 +1,6 @@
include "common_schema.fbs";
namespace msg.fbuf.ledger;
namespace msg.fbuf.ledgermsg;
table LedgerBlock {
version:string;

28
src/msg/fbuf/ledger_schema_generated.h
View File

@@ -1,8 +1,8 @@
// automatically generated by the FlatBuffers compiler, do not modify
#ifndef FLATBUFFERS_GENERATED_LEDGERSCHEMA_MSG_FBUF_LEDGER_H_
#define FLATBUFFERS_GENERATED_LEDGERSCHEMA_MSG_FBUF_LEDGER_H_
#ifndef FLATBUFFERS_GENERATED_LEDGERSCHEMA_MSG_FBUF_LEDGERMSG_H_
#define FLATBUFFERS_GENERATED_LEDGERSCHEMA_MSG_FBUF_LEDGERMSG_H_
#include "flatbuffers/flatbuffers.h"
@@ -10,7 +10,7 @@
namespace msg {
namespace fbuf {
namespace ledger {
namespace ledgermsg {
struct LedgerBlock;
struct LedgerBlockBuilder;
@@ -190,7 +190,7 @@ inline flatbuffers::Offset<LedgerBlock> CreateLedgerBlockDirect(
auto users__ = users ? _fbb.CreateVector<flatbuffers::Offset<msg::fbuf::ByteArray>>(*users) : 0;
auto inputs__ = inputs ? _fbb.CreateVector<flatbuffers::Offset<msg::fbuf::ByteArray>>(*inputs) : 0;
auto output__ = output ? _fbb.CreateVector<uint8_t>(*output) : 0;
return msg::fbuf::ledger::CreateLedgerBlock(
return msg::fbuf::ledgermsg::CreateLedgerBlock(
_fbb,
version__,
seq_no,
@@ -203,12 +203,12 @@ inline flatbuffers::Offset<LedgerBlock> CreateLedgerBlockDirect(
output__);
}
inline const msg::fbuf::ledger::LedgerBlock *GetLedgerBlock(const void *buf) {
return flatbuffers::GetRoot<msg::fbuf::ledger::LedgerBlock>(buf);
inline const msg::fbuf::ledgermsg::LedgerBlock *GetLedgerBlock(const void *buf) {
return flatbuffers::GetRoot<msg::fbuf::ledgermsg::LedgerBlock>(buf);
}
inline const msg::fbuf::ledger::LedgerBlock *GetSizePrefixedLedgerBlock(const void *buf) {
return flatbuffers::GetSizePrefixedRoot<msg::fbuf::ledger::LedgerBlock>(buf);
inline const msg::fbuf::ledgermsg::LedgerBlock *GetSizePrefixedLedgerBlock(const void *buf) {
return flatbuffers::GetSizePrefixedRoot<msg::fbuf::ledgermsg::LedgerBlock>(buf);
}
inline LedgerBlock *GetMutableLedgerBlock(void *buf) {
@@ -217,28 +217,28 @@ inline LedgerBlock *GetMutableLedgerBlock(void *buf) {
inline bool VerifyLedgerBlockBuffer(
flatbuffers::Verifier &verifier) {
return verifier.VerifyBuffer<msg::fbuf::ledger::LedgerBlock>(nullptr);
return verifier.VerifyBuffer<msg::fbuf::ledgermsg::LedgerBlock>(nullptr);
}
inline bool VerifySizePrefixedLedgerBlockBuffer(
flatbuffers::Verifier &verifier) {
return verifier.VerifySizePrefixedBuffer<msg::fbuf::ledger::LedgerBlock>(nullptr);
return verifier.VerifySizePrefixedBuffer<msg::fbuf::ledgermsg::LedgerBlock>(nullptr);
}
inline void FinishLedgerBlockBuffer(
flatbuffers::FlatBufferBuilder &fbb,
flatbuffers::Offset<msg::fbuf::ledger::LedgerBlock> root) {
flatbuffers::Offset<msg::fbuf::ledgermsg::LedgerBlock> root) {
fbb.Finish(root);
}
inline void FinishSizePrefixedLedgerBlockBuffer(
flatbuffers::FlatBufferBuilder &fbb,
flatbuffers::Offset<msg::fbuf::ledger::LedgerBlock> root) {
flatbuffers::Offset<msg::fbuf::ledgermsg::LedgerBlock> root) {
fbb.FinishSizePrefixed(root);
}
} // namespace ledger
} // namespace ledgermsg
} // namespace fbuf
} // namespace msg
#endif // FLATBUFFERS_GENERATED_LEDGERSCHEMA_MSG_FBUF_LEDGER_H_
#endif // FLATBUFFERS_GENERATED_LEDGERSCHEMA_MSG_FBUF_LEDGERMSG_H_

1
src/msg/fbuf/p2pmsg_container.fbs
View File

@@ -10,6 +10,7 @@ table Container { //root type for message
timestamp:uint64;
pubkey:[ubyte];
lcl:[ubyte];
last_primary_shard_id:Sequence_Hash;
signature:[ubyte]; // signature of the message content
content:[ubyte]; // message content: byte array of proposal,npl,etc
}

20
src/msg/fbuf/p2pmsg_container_generated.h
View File

@@ -23,8 +23,9 @@ struct Container FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
VT_TIMESTAMP = 6,
VT_PUBKEY = 8,
VT_LCL = 10,
VT_SIGNATURE = 12,
VT_CONTENT = 14
VT_LAST_PRIMARY_SHARD_ID = 12,
VT_SIGNATURE = 14,
VT_CONTENT = 16
};
uint16_t version() const {
return GetField<uint16_t>(VT_VERSION, 0);
@@ -50,6 +51,12 @@ struct Container FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
flatbuffers::Vector<uint8_t> *mutable_lcl() {
return GetPointer<flatbuffers::Vector<uint8_t> *>(VT_LCL);
}
const msg::fbuf::p2pmsg::Sequence_Hash *last_primary_shard_id() const {
return GetPointer<const msg::fbuf::p2pmsg::Sequence_Hash *>(VT_LAST_PRIMARY_SHARD_ID);
}
msg::fbuf::p2pmsg::Sequence_Hash *mutable_last_primary_shard_id() {
return GetPointer<msg::fbuf::p2pmsg::Sequence_Hash *>(VT_LAST_PRIMARY_SHARD_ID);
}
const flatbuffers::Vector<uint8_t> *signature() const {
return GetPointer<const flatbuffers::Vector<uint8_t> *>(VT_SIGNATURE);
}
@@ -70,6 +77,8 @@ struct Container FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
verifier.VerifyVector(pubkey()) &&
VerifyOffset(verifier, VT_LCL) &&
verifier.VerifyVector(lcl()) &&
VerifyOffset(verifier, VT_LAST_PRIMARY_SHARD_ID) &&
verifier.VerifyTable(last_primary_shard_id()) &&
VerifyOffset(verifier, VT_SIGNATURE) &&
verifier.VerifyVector(signature()) &&
VerifyOffset(verifier, VT_CONTENT) &&
@@ -94,6 +103,9 @@ struct ContainerBuilder {
void add_lcl(flatbuffers::Offset<flatbuffers::Vector<uint8_t>> lcl) {
fbb_.AddOffset(Container::VT_LCL, lcl);
}
void add_last_primary_shard_id(flatbuffers::Offset<msg::fbuf::p2pmsg::Sequence_Hash> last_primary_shard_id) {
fbb_.AddOffset(Container::VT_LAST_PRIMARY_SHARD_ID, last_primary_shard_id);
}
void add_signature(flatbuffers::Offset<flatbuffers::Vector<uint8_t>> signature) {
fbb_.AddOffset(Container::VT_SIGNATURE, signature);
}
@@ -117,12 +129,14 @@ inline flatbuffers::Offset<Container> CreateContainer(
uint64_t timestamp = 0,
flatbuffers::Offset<flatbuffers::Vector<uint8_t>> pubkey = 0,
flatbuffers::Offset<flatbuffers::Vector<uint8_t>> lcl = 0,
flatbuffers::Offset<msg::fbuf::p2pmsg::Sequence_Hash> last_primary_shard_id = 0,
flatbuffers::Offset<flatbuffers::Vector<uint8_t>> signature = 0,
flatbuffers::Offset<flatbuffers::Vector<uint8_t>> content = 0) {
ContainerBuilder builder_(_fbb);
builder_.add_timestamp(timestamp);
builder_.add_content(content);
builder_.add_signature(signature);
builder_.add_last_primary_shard_id(last_primary_shard_id);
builder_.add_lcl(lcl);
builder_.add_pubkey(pubkey);
builder_.add_version(version);
@@ -135,6 +149,7 @@ inline flatbuffers::Offset<Container> CreateContainerDirect(
uint64_t timestamp = 0,
const std::vector<uint8_t> *pubkey = nullptr,
const std::vector<uint8_t> *lcl = nullptr,
flatbuffers::Offset<msg::fbuf::p2pmsg::Sequence_Hash> last_primary_shard_id = 0,
const std::vector<uint8_t> *signature = nullptr,
const std::vector<uint8_t> *content = nullptr) {
auto pubkey__ = pubkey ? _fbb.CreateVector<uint8_t>(*pubkey) : 0;
@@ -147,6 +162,7 @@ inline flatbuffers::Offset<Container> CreateContainerDirect(
timestamp,
pubkey__,
lcl__,
last_primary_shard_id,
signature__,
content__);
}

28
src/msg/fbuf/p2pmsg_content.fbs
View File

@@ -34,8 +34,6 @@ union Message {
Npl_Message,
Hpfs_Request_Message,
Hpfs_Response_Message,
History_Request_Message,
History_Response_Message,
Peer_Requirement_Announcement_Message,
Peer_List_Request_Message,
Peer_List_Response_Message,
@@ -57,18 +55,20 @@ table Proposal_Message { //Proposal type message schema
nonce: [ubyte];
users:[ByteArray];
input_hashes:[ByteArray];
last_blob_shard_id: Sequence_Hash;
output_hash:[ubyte];
output_sig:[ubyte];
state_hash: [ubyte];
patch_hash: [ubyte];
}
table Npl_Message { //NPL type message schema
data:[ubyte];
table Sequence_Hash {
shard_seq_no: uint64;
shard_hash: [ubyte];
}
table History_Request_Message { //Ledger History request type message schema
required_lcl:[ubyte];
table Npl_Message { //NPL type message schema
data:[ubyte];
}
enum Ledger_Response_Error : ubyte
@@ -78,22 +78,6 @@ enum Ledger_Response_Error : ubyte
Req_Ledger_Not_Found = 2
}
table History_Response_Message { //Ledger History request type message schema
requester_lcl:[ubyte];
hist_ledger_blocks:[HistoryLedgerBlockPair];
error: Ledger_Response_Error;
}
table HistoryLedgerBlockPair { //A key, value pair of byte[].
seq_no:uint64;
ledger:HistoryLedgerBlock;
}
table HistoryLedgerBlock {
lcl:[ubyte];
block_buffer:[ubyte];
}
table Hpfs_Request_Message { //Hpfs request message schema
mount_id: uint32;
parent_path:string;

455
src/msg/fbuf/p2pmsg_content_generated.h
View File

@@ -33,21 +33,12 @@ struct NonUnl_Proposal_MessageBuilder;
struct Proposal_Message;
struct Proposal_MessageBuilder;
struct Sequence_Hash;
struct Sequence_HashBuilder;
struct Npl_Message;
struct Npl_MessageBuilder;
struct History_Request_Message;
struct History_Request_MessageBuilder;
struct History_Response_Message;
struct History_Response_MessageBuilder;
struct HistoryLedgerBlockPair;
struct HistoryLedgerBlockPairBuilder;
struct HistoryLedgerBlock;
struct HistoryLedgerBlockBuilder;
struct Hpfs_Request_Message;
struct Hpfs_Request_MessageBuilder;
@@ -81,7 +72,7 @@ struct Peer_List_Response_MessageBuilder;
struct Peer_Properties;
struct Peer_PropertiesBuilder;
enum Message {
enum Message : uint8_t {
Message_NONE = 0,
Message_Peer_Challenge_Response_Message = 1,
Message_Peer_Challenge_Message = 2,
@@ -90,17 +81,15 @@ enum Message {
Message_Npl_Message = 5,
Message_Hpfs_Request_Message = 6,
Message_Hpfs_Response_Message = 7,
Message_History_Request_Message = 8,
Message_History_Response_Message = 9,
Message_Peer_Requirement_Announcement_Message = 10,
Message_Peer_List_Request_Message = 11,
Message_Peer_List_Response_Message = 12,
Message_Available_Capacity_Announcement_Message = 13,
Message_Peer_Requirement_Announcement_Message = 8,
Message_Peer_List_Request_Message = 9,
Message_Peer_List_Response_Message = 10,
Message_Available_Capacity_Announcement_Message = 11,
Message_MIN = Message_NONE,
Message_MAX = Message_Available_Capacity_Announcement_Message
};
inline const Message (&EnumValuesMessage())[14] {
inline const Message (&EnumValuesMessage())[12] {
static const Message values[] = {
Message_NONE,
Message_Peer_Challenge_Response_Message,
@@ -110,8 +99,6 @@ inline const Message (&EnumValuesMessage())[14] {
Message_Npl_Message,
Message_Hpfs_Request_Message,
Message_Hpfs_Response_Message,
Message_History_Request_Message,
Message_History_Response_Message,
Message_Peer_Requirement_Announcement_Message,
Message_Peer_List_Request_Message,
Message_Peer_List_Response_Message,
@@ -121,7 +108,7 @@ inline const Message (&EnumValuesMessage())[14] {
}
inline const char * const *EnumNamesMessage() {
static const char * const names[15] = {
static const char * const names[13] = {
"NONE",
"Peer_Challenge_Response_Message",
"Peer_Challenge_Message",
@@ -130,8 +117,6 @@ inline const char * const *EnumNamesMessage() {
"Npl_Message",
"Hpfs_Request_Message",
"Hpfs_Response_Message",
"History_Request_Message",
"History_Response_Message",
"Peer_Requirement_Announcement_Message",
"Peer_List_Request_Message",
"Peer_List_Response_Message",
@@ -179,14 +164,6 @@ template<> struct MessageTraits<msg::fbuf::p2pmsg::Hpfs_Response_Message> {
static const Message enum_value = Message_Hpfs_Response_Message;
};
template<> struct MessageTraits<msg::fbuf::p2pmsg::History_Request_Message> {
static const Message enum_value = Message_History_Request_Message;
};
template<> struct MessageTraits<msg::fbuf::p2pmsg::History_Response_Message> {
static const Message enum_value = Message_History_Response_Message;
};
template<> struct MessageTraits<msg::fbuf::p2pmsg::Peer_Requirement_Announcement_Message> {
static const Message enum_value = Message_Peer_Requirement_Announcement_Message;
};
@@ -206,7 +183,7 @@ template<> struct MessageTraits<msg::fbuf::p2pmsg::Available_Capacity_Announceme
bool VerifyMessage(flatbuffers::Verifier &verifier, const void *obj, Message type);
bool VerifyMessageVector(flatbuffers::Verifier &verifier, const flatbuffers::Vector<flatbuffers::Offset<void>> *values, const flatbuffers::Vector<uint8_t> *types);
enum Ledger_Response_Error {
enum Ledger_Response_Error : uint8_t {
Ledger_Response_Error_None = 0,
Ledger_Response_Error_Invalid_Min_Ledger = 1,
Ledger_Response_Error_Req_Ledger_Not_Found = 2,
@@ -239,7 +216,7 @@ inline const char *EnumNameLedger_Response_Error(Ledger_Response_Error e) {
return EnumNamesLedger_Response_Error()[index];
}
enum Hpfs_Response {
enum Hpfs_Response : uint8_t {
Hpfs_Response_NONE = 0,
Hpfs_Response_File_HashMap_Response = 1,
Hpfs_Response_Block_Response = 2,
@@ -347,7 +324,6 @@ struct Peer_Challenge_MessageBuilder {
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
Peer_Challenge_MessageBuilder &operator=(const Peer_Challenge_MessageBuilder &);
flatbuffers::Offset<Peer_Challenge_Message> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<Peer_Challenge_Message>(end);
@@ -423,7 +399,6 @@ struct Peer_Challenge_Response_MessageBuilder {
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
Peer_Challenge_Response_MessageBuilder &operator=(const Peer_Challenge_Response_MessageBuilder &);
flatbuffers::Offset<Peer_Challenge_Response_Message> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<Peer_Challenge_Response_Message>(end);
@@ -506,7 +481,6 @@ struct UserInputBuilder {
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
UserInputBuilder &operator=(const UserInputBuilder &);
flatbuffers::Offset<UserInput> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<UserInput>(end);
@@ -583,7 +557,6 @@ struct UserInputGroupBuilder {
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
UserInputGroupBuilder &operator=(const UserInputGroupBuilder &);
flatbuffers::Offset<UserInputGroup> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<UserInputGroup>(end);
@@ -647,12 +620,6 @@ struct Content FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
const msg::fbuf::p2pmsg::Hpfs_Response_Message *message_as_Hpfs_Response_Message() const {
return message_type() == msg::fbuf::p2pmsg::Message_Hpfs_Response_Message ? static_cast<const msg::fbuf::p2pmsg::Hpfs_Response_Message *>(message()) : nullptr;
}
const msg::fbuf::p2pmsg::History_Request_Message *message_as_History_Request_Message() const {
return message_type() == msg::fbuf::p2pmsg::Message_History_Request_Message ? static_cast<const msg::fbuf::p2pmsg::History_Request_Message *>(message()) : nullptr;
}
const msg::fbuf::p2pmsg::History_Response_Message *message_as_History_Response_Message() const {
return message_type() == msg::fbuf::p2pmsg::Message_History_Response_Message ? static_cast<const msg::fbuf::p2pmsg::History_Response_Message *>(message()) : nullptr;
}
const msg::fbuf::p2pmsg::Peer_Requirement_Announcement_Message *message_as_Peer_Requirement_Announcement_Message() const {
return message_type() == msg::fbuf::p2pmsg::Message_Peer_Requirement_Announcement_Message ? static_cast<const msg::fbuf::p2pmsg::Peer_Requirement_Announcement_Message *>(message()) : nullptr;
}
@@ -705,14 +672,6 @@ template<> inline const msg::fbuf::p2pmsg::Hpfs_Response_Message *Content::messa
return message_as_Hpfs_Response_Message();
}
template<> inline const msg::fbuf::p2pmsg::History_Request_Message *Content::message_as<msg::fbuf::p2pmsg::History_Request_Message>() const {
return message_as_History_Request_Message();
}
template<> inline const msg::fbuf::p2pmsg::History_Response_Message *Content::message_as<msg::fbuf::p2pmsg::History_Response_Message>() const {
return message_as_History_Response_Message();
}
template<> inline const msg::fbuf::p2pmsg::Peer_Requirement_Announcement_Message *Content::message_as<msg::fbuf::p2pmsg::Peer_Requirement_Announcement_Message>() const {
return message_as_Peer_Requirement_Announcement_Message();
}
@@ -743,7 +702,6 @@ struct ContentBuilder {
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
ContentBuilder &operator=(const ContentBuilder &);
flatbuffers::Offset<Content> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<Content>(end);
@@ -792,7 +750,6 @@ struct NonUnl_Proposal_MessageBuilder {
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
NonUnl_Proposal_MessageBuilder &operator=(const NonUnl_Proposal_MessageBuilder &);
flatbuffers::Offset<NonUnl_Proposal_Message> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<NonUnl_Proposal_Message>(end);
@@ -826,10 +783,11 @@ struct Proposal_Message FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
VT_NONCE = 10,
VT_USERS = 12,
VT_INPUT_HASHES = 14,
VT_OUTPUT_HASH = 16,
VT_OUTPUT_SIG = 18,
VT_STATE_HASH = 20,
VT_PATCH_HASH = 22
VT_LAST_BLOB_SHARD_ID = 16,
VT_OUTPUT_HASH = 18,
VT_OUTPUT_SIG = 20,
VT_STATE_HASH = 22,
VT_PATCH_HASH = 24
};
uint8_t stage() const {
return GetField<uint8_t>(VT_STAGE, 0);
@@ -867,6 +825,12 @@ struct Proposal_Message FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ByteArray>> *mutable_input_hashes() {
return GetPointer<flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ByteArray>> *>(VT_INPUT_HASHES);
}
const msg::fbuf::p2pmsg::Sequence_Hash *last_blob_shard_id() const {
return GetPointer<const msg::fbuf::p2pmsg::Sequence_Hash *>(VT_LAST_BLOB_SHARD_ID);
}
msg::fbuf::p2pmsg::Sequence_Hash *mutable_last_blob_shard_id() {
return GetPointer<msg::fbuf::p2pmsg::Sequence_Hash *>(VT_LAST_BLOB_SHARD_ID);
}
const flatbuffers::Vector<uint8_t> *output_hash() const {
return GetPointer<const flatbuffers::Vector<uint8_t> *>(VT_OUTPUT_HASH);
}
@@ -904,6 +868,8 @@ struct Proposal_Message FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
VerifyOffset(verifier, VT_INPUT_HASHES) &&
verifier.VerifyVector(input_hashes()) &&
verifier.VerifyVectorOfTables(input_hashes()) &&
VerifyOffset(verifier, VT_LAST_BLOB_SHARD_ID) &&
verifier.VerifyTable(last_blob_shard_id()) &&
VerifyOffset(verifier, VT_OUTPUT_HASH) &&
verifier.VerifyVector(output_hash()) &&
VerifyOffset(verifier, VT_OUTPUT_SIG) &&
@@ -938,6 +904,9 @@ struct Proposal_MessageBuilder {
void add_input_hashes(flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ByteArray>>> input_hashes) {
fbb_.AddOffset(Proposal_Message::VT_INPUT_HASHES, input_hashes);
}
void add_last_blob_shard_id(flatbuffers::Offset<msg::fbuf::p2pmsg::Sequence_Hash> last_blob_shard_id) {
fbb_.AddOffset(Proposal_Message::VT_LAST_BLOB_SHARD_ID, last_blob_shard_id);
}
void add_output_hash(flatbuffers::Offset<flatbuffers::Vector<uint8_t>> output_hash) {
fbb_.AddOffset(Proposal_Message::VT_OUTPUT_HASH, output_hash);
}
@@ -954,7 +923,6 @@ struct Proposal_MessageBuilder {
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
Proposal_MessageBuilder &operator=(const Proposal_MessageBuilder &);
flatbuffers::Offset<Proposal_Message> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<Proposal_Message>(end);
@@ -970,6 +938,7 @@ inline flatbuffers::Offset<Proposal_Message> CreateProposal_Message(
flatbuffers::Offset<flatbuffers::Vector<uint8_t>> nonce = 0,
flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ByteArray>>> users = 0,
flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ByteArray>>> input_hashes = 0,
flatbuffers::Offset<msg::fbuf::p2pmsg::Sequence_Hash> last_blob_shard_id = 0,
flatbuffers::Offset<flatbuffers::Vector<uint8_t>> output_hash = 0,
flatbuffers::Offset<flatbuffers::Vector<uint8_t>> output_sig = 0,
flatbuffers::Offset<flatbuffers::Vector<uint8_t>> state_hash = 0,
@@ -980,6 +949,7 @@ inline flatbuffers::Offset<Proposal_Message> CreateProposal_Message(
builder_.add_state_hash(state_hash);
builder_.add_output_sig(output_sig);
builder_.add_output_hash(output_hash);
builder_.add_last_blob_shard_id(last_blob_shard_id);
builder_.add_input_hashes(input_hashes);
builder_.add_users(users);
builder_.add_nonce(nonce);
@@ -996,6 +966,7 @@ inline flatbuffers::Offset<Proposal_Message> CreateProposal_MessageDirect(
const std::vector<uint8_t> *nonce = nullptr,
const std::vector<flatbuffers::Offset<msg::fbuf::ByteArray>> *users = nullptr,
const std::vector<flatbuffers::Offset<msg::fbuf::ByteArray>> *input_hashes = nullptr,
flatbuffers::Offset<msg::fbuf::p2pmsg::Sequence_Hash> last_blob_shard_id = 0,
const std::vector<uint8_t> *output_hash = nullptr,
const std::vector<uint8_t> *output_sig = nullptr,
const std::vector<uint8_t> *state_hash = nullptr,
@@ -1015,12 +986,82 @@ inline flatbuffers::Offset<Proposal_Message> CreateProposal_MessageDirect(
nonce__,
users__,
input_hashes__,
last_blob_shard_id,
output_hash__,
output_sig__,
state_hash__,
patch_hash__);
}
struct Sequence_Hash FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
typedef Sequence_HashBuilder Builder;
enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
VT_SHARD_SEQ_NO = 4,
VT_SHARD_HASH = 6
};
uint64_t shard_seq_no() const {
return GetField<uint64_t>(VT_SHARD_SEQ_NO, 0);
}
bool mutate_shard_seq_no(uint64_t _shard_seq_no) {
return SetField<uint64_t>(VT_SHARD_SEQ_NO, _shard_seq_no, 0);
}
const flatbuffers::Vector<uint8_t> *shard_hash() const {
return GetPointer<const flatbuffers::Vector<uint8_t> *>(VT_SHARD_HASH);
}
flatbuffers::Vector<uint8_t> *mutable_shard_hash() {
return GetPointer<flatbuffers::Vector<uint8_t> *>(VT_SHARD_HASH);
}
bool Verify(flatbuffers::Verifier &verifier) const {
return VerifyTableStart(verifier) &&
VerifyField<uint64_t>(verifier, VT_SHARD_SEQ_NO) &&
VerifyOffset(verifier, VT_SHARD_HASH) &&
verifier.VerifyVector(shard_hash()) &&
verifier.EndTable();
}
};
struct Sequence_HashBuilder {
typedef Sequence_Hash Table;
flatbuffers::FlatBufferBuilder &fbb_;
flatbuffers::uoffset_t start_;
void add_shard_seq_no(uint64_t shard_seq_no) {
fbb_.AddElement<uint64_t>(Sequence_Hash::VT_SHARD_SEQ_NO, shard_seq_no, 0);
}
void add_shard_hash(flatbuffers::Offset<flatbuffers::Vector<uint8_t>> shard_hash) {
fbb_.AddOffset(Sequence_Hash::VT_SHARD_HASH, shard_hash);
}
explicit Sequence_HashBuilder(flatbuffers::FlatBufferBuilder &_fbb)
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
flatbuffers::Offset<Sequence_Hash> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<Sequence_Hash>(end);
return o;
}
};
inline flatbuffers::Offset<Sequence_Hash> CreateSequence_Hash(
flatbuffers::FlatBufferBuilder &_fbb,
uint64_t shard_seq_no = 0,
flatbuffers::Offset<flatbuffers::Vector<uint8_t>> shard_hash = 0) {
Sequence_HashBuilder builder_(_fbb);
builder_.add_shard_seq_no(shard_seq_no);
builder_.add_shard_hash(shard_hash);
return builder_.Finish();
}
inline flatbuffers::Offset<Sequence_Hash> CreateSequence_HashDirect(
flatbuffers::FlatBufferBuilder &_fbb,
uint64_t shard_seq_no = 0,
const std::vector<uint8_t> *shard_hash = nullptr) {
auto shard_hash__ = shard_hash ? _fbb.CreateVector<uint8_t>(*shard_hash) : 0;
return msg::fbuf::p2pmsg::CreateSequence_Hash(
_fbb,
shard_seq_no,
shard_hash__);
}
struct Npl_Message FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
typedef Npl_MessageBuilder Builder;
enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
@@ -1051,7 +1092,6 @@ struct Npl_MessageBuilder {
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
Npl_MessageBuilder &operator=(const Npl_MessageBuilder &);
flatbuffers::Offset<Npl_Message> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<Npl_Message>(end);
@@ -1076,280 +1116,6 @@ inline flatbuffers::Offset<Npl_Message> CreateNpl_MessageDirect(
data__);
}
struct History_Request_Message FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
typedef History_Request_MessageBuilder Builder;
enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
VT_REQUIRED_LCL = 4
};
const flatbuffers::Vector<uint8_t> *required_lcl() const {
return GetPointer<const flatbuffers::Vector<uint8_t> *>(VT_REQUIRED_LCL);
}
flatbuffers::Vector<uint8_t> *mutable_required_lcl() {
return GetPointer<flatbuffers::Vector<uint8_t> *>(VT_REQUIRED_LCL);
}
bool Verify(flatbuffers::Verifier &verifier) const {
return VerifyTableStart(verifier) &&
VerifyOffset(verifier, VT_REQUIRED_LCL) &&
verifier.VerifyVector(required_lcl()) &&
verifier.EndTable();
}
};
struct History_Request_MessageBuilder {
typedef History_Request_Message Table;
flatbuffers::FlatBufferBuilder &fbb_;
flatbuffers::uoffset_t start_;
void add_required_lcl(flatbuffers::Offset<flatbuffers::Vector<uint8_t>> required_lcl) {
fbb_.AddOffset(History_Request_Message::VT_REQUIRED_LCL, required_lcl);
}
explicit History_Request_MessageBuilder(flatbuffers::FlatBufferBuilder &_fbb)
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
History_Request_MessageBuilder &operator=(const History_Request_MessageBuilder &);
flatbuffers::Offset<History_Request_Message> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<History_Request_Message>(end);
return o;
}
};
inline flatbuffers::Offset<History_Request_Message> CreateHistory_Request_Message(
flatbuffers::FlatBufferBuilder &_fbb,
flatbuffers::Offset<flatbuffers::Vector<uint8_t>> required_lcl = 0) {
History_Request_MessageBuilder builder_(_fbb);
builder_.add_required_lcl(required_lcl);
return builder_.Finish();
}
inline flatbuffers::Offset<History_Request_Message> CreateHistory_Request_MessageDirect(
flatbuffers::FlatBufferBuilder &_fbb,
const std::vector<uint8_t> *required_lcl = nullptr) {
auto required_lcl__ = required_lcl ? _fbb.CreateVector<uint8_t>(*required_lcl) : 0;
return msg::fbuf::p2pmsg::CreateHistory_Request_Message(
_fbb,
required_lcl__);
}
struct History_Response_Message FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
typedef History_Response_MessageBuilder Builder;
enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
VT_REQUESTER_LCL = 4,
VT_HIST_LEDGER_BLOCKS = 6,
VT_ERROR = 8
};
const flatbuffers::Vector<uint8_t> *requester_lcl() const {
return GetPointer<const flatbuffers::Vector<uint8_t> *>(VT_REQUESTER_LCL);
}
flatbuffers::Vector<uint8_t> *mutable_requester_lcl() {
return GetPointer<flatbuffers::Vector<uint8_t> *>(VT_REQUESTER_LCL);
}
const flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::p2pmsg::HistoryLedgerBlockPair>> *hist_ledger_blocks() const {
return GetPointer<const flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::p2pmsg::HistoryLedgerBlockPair>> *>(VT_HIST_LEDGER_BLOCKS);
}
flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::p2pmsg::HistoryLedgerBlockPair>> *mutable_hist_ledger_blocks() {
return GetPointer<flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::p2pmsg::HistoryLedgerBlockPair>> *>(VT_HIST_LEDGER_BLOCKS);
}
msg::fbuf::p2pmsg::Ledger_Response_Error error() const {
return static_cast<msg::fbuf::p2pmsg::Ledger_Response_Error>(GetField<uint8_t>(VT_ERROR, 0));
}
bool mutate_error(msg::fbuf::p2pmsg::Ledger_Response_Error _error) {
return SetField<uint8_t>(VT_ERROR, static_cast<uint8_t>(_error), 0);
}
bool Verify(flatbuffers::Verifier &verifier) const {
return VerifyTableStart(verifier) &&
VerifyOffset(verifier, VT_REQUESTER_LCL) &&
verifier.VerifyVector(requester_lcl()) &&
VerifyOffset(verifier, VT_HIST_LEDGER_BLOCKS) &&
verifier.VerifyVector(hist_ledger_blocks()) &&
verifier.VerifyVectorOfTables(hist_ledger_blocks()) &&
VerifyField<uint8_t>(verifier, VT_ERROR) &&
verifier.EndTable();
}
};
struct History_Response_MessageBuilder {
typedef History_Response_Message Table;
flatbuffers::FlatBufferBuilder &fbb_;
flatbuffers::uoffset_t start_;
void add_requester_lcl(flatbuffers::Offset<flatbuffers::Vector<uint8_t>> requester_lcl) {
fbb_.AddOffset(History_Response_Message::VT_REQUESTER_LCL, requester_lcl);
}
void add_hist_ledger_blocks(flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::p2pmsg::HistoryLedgerBlockPair>>> hist_ledger_blocks) {
fbb_.AddOffset(History_Response_Message::VT_HIST_LEDGER_BLOCKS, hist_ledger_blocks);
}
void add_error(msg::fbuf::p2pmsg::Ledger_Response_Error error) {
fbb_.AddElement<uint8_t>(History_Response_Message::VT_ERROR, static_cast<uint8_t>(error), 0);
}
explicit History_Response_MessageBuilder(flatbuffers::FlatBufferBuilder &_fbb)
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
History_Response_MessageBuilder &operator=(const History_Response_MessageBuilder &);
flatbuffers::Offset<History_Response_Message> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<History_Response_Message>(end);
return o;
}
};
inline flatbuffers::Offset<History_Response_Message> CreateHistory_Response_Message(
flatbuffers::FlatBufferBuilder &_fbb,
flatbuffers::Offset<flatbuffers::Vector<uint8_t>> requester_lcl = 0,
flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::p2pmsg::HistoryLedgerBlockPair>>> hist_ledger_blocks = 0,
msg::fbuf::p2pmsg::Ledger_Response_Error error = msg::fbuf::p2pmsg::Ledger_Response_Error_None) {
History_Response_MessageBuilder builder_(_fbb);
builder_.add_hist_ledger_blocks(hist_ledger_blocks);
builder_.add_requester_lcl(requester_lcl);
builder_.add_error(error);
return builder_.Finish();
}
inline flatbuffers::Offset<History_Response_Message> CreateHistory_Response_MessageDirect(
flatbuffers::FlatBufferBuilder &_fbb,
const std::vector<uint8_t> *requester_lcl = nullptr,
const std::vector<flatbuffers::Offset<msg::fbuf::p2pmsg::HistoryLedgerBlockPair>> *hist_ledger_blocks = nullptr,
msg::fbuf::p2pmsg::Ledger_Response_Error error = msg::fbuf::p2pmsg::Ledger_Response_Error_None) {
auto requester_lcl__ = requester_lcl ? _fbb.CreateVector<uint8_t>(*requester_lcl) : 0;
auto hist_ledger_blocks__ = hist_ledger_blocks ? _fbb.CreateVector<flatbuffers::Offset<msg::fbuf::p2pmsg::HistoryLedgerBlockPair>>(*hist_ledger_blocks) : 0;
return msg::fbuf::p2pmsg::CreateHistory_Response_Message(
_fbb,
requester_lcl__,
hist_ledger_blocks__,
error);
}
struct HistoryLedgerBlockPair FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
typedef HistoryLedgerBlockPairBuilder Builder;
enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
VT_SEQ_NO = 4,
VT_LEDGER = 6
};
uint64_t seq_no() const {
return GetField<uint64_t>(VT_SEQ_NO, 0);
}
bool mutate_seq_no(uint64_t _seq_no) {
return SetField<uint64_t>(VT_SEQ_NO, _seq_no, 0);
}
const msg::fbuf::p2pmsg::HistoryLedgerBlock *ledger() const {
return GetPointer<const msg::fbuf::p2pmsg::HistoryLedgerBlock *>(VT_LEDGER);
}
msg::fbuf::p2pmsg::HistoryLedgerBlock *mutable_ledger() {
return GetPointer<msg::fbuf::p2pmsg::HistoryLedgerBlock *>(VT_LEDGER);
}
bool Verify(flatbuffers::Verifier &verifier) const {
return VerifyTableStart(verifier) &&
VerifyField<uint64_t>(verifier, VT_SEQ_NO) &&
VerifyOffset(verifier, VT_LEDGER) &&
verifier.VerifyTable(ledger()) &&
verifier.EndTable();
}
};
struct HistoryLedgerBlockPairBuilder {
typedef HistoryLedgerBlockPair Table;
flatbuffers::FlatBufferBuilder &fbb_;
flatbuffers::uoffset_t start_;
void add_seq_no(uint64_t seq_no) {
fbb_.AddElement<uint64_t>(HistoryLedgerBlockPair::VT_SEQ_NO, seq_no, 0);
}
void add_ledger(flatbuffers::Offset<msg::fbuf::p2pmsg::HistoryLedgerBlock> ledger) {
fbb_.AddOffset(HistoryLedgerBlockPair::VT_LEDGER, ledger);
}
explicit HistoryLedgerBlockPairBuilder(flatbuffers::FlatBufferBuilder &_fbb)
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
HistoryLedgerBlockPairBuilder &operator=(const HistoryLedgerBlockPairBuilder &);
flatbuffers::Offset<HistoryLedgerBlockPair> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<HistoryLedgerBlockPair>(end);
return o;
}
};
inline flatbuffers::Offset<HistoryLedgerBlockPair> CreateHistoryLedgerBlockPair(
flatbuffers::FlatBufferBuilder &_fbb,
uint64_t seq_no = 0,
flatbuffers::Offset<msg::fbuf::p2pmsg::HistoryLedgerBlock> ledger = 0) {
HistoryLedgerBlockPairBuilder builder_(_fbb);
builder_.add_seq_no(seq_no);
builder_.add_ledger(ledger);
return builder_.Finish();
}
struct HistoryLedgerBlock FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
typedef HistoryLedgerBlockBuilder Builder;
enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
VT_LCL = 4,
VT_BLOCK_BUFFER = 6
};
const flatbuffers::Vector<uint8_t> *lcl() const {
return GetPointer<const flatbuffers::Vector<uint8_t> *>(VT_LCL);
}
flatbuffers::Vector<uint8_t> *mutable_lcl() {
return GetPointer<flatbuffers::Vector<uint8_t> *>(VT_LCL);
}
const flatbuffers::Vector<uint8_t> *block_buffer() const {
return GetPointer<const flatbuffers::Vector<uint8_t> *>(VT_BLOCK_BUFFER);
}
flatbuffers::Vector<uint8_t> *mutable_block_buffer() {
return GetPointer<flatbuffers::Vector<uint8_t> *>(VT_BLOCK_BUFFER);
}
bool Verify(flatbuffers::Verifier &verifier) const {
return VerifyTableStart(verifier) &&
VerifyOffset(verifier, VT_LCL) &&
verifier.VerifyVector(lcl()) &&
VerifyOffset(verifier, VT_BLOCK_BUFFER) &&
verifier.VerifyVector(block_buffer()) &&
verifier.EndTable();
}
};
struct HistoryLedgerBlockBuilder {
typedef HistoryLedgerBlock Table;
flatbuffers::FlatBufferBuilder &fbb_;
flatbuffers::uoffset_t start_;
void add_lcl(flatbuffers::Offset<flatbuffers::Vector<uint8_t>> lcl) {
fbb_.AddOffset(HistoryLedgerBlock::VT_LCL, lcl);
}
void add_block_buffer(flatbuffers::Offset<flatbuffers::Vector<uint8_t>> block_buffer) {
fbb_.AddOffset(HistoryLedgerBlock::VT_BLOCK_BUFFER, block_buffer);
}
explicit HistoryLedgerBlockBuilder(flatbuffers::FlatBufferBuilder &_fbb)
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
HistoryLedgerBlockBuilder &operator=(const HistoryLedgerBlockBuilder &);
flatbuffers::Offset<HistoryLedgerBlock> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<HistoryLedgerBlock>(end);
return o;
}
};
inline flatbuffers::Offset<HistoryLedgerBlock> CreateHistoryLedgerBlock(
flatbuffers::FlatBufferBuilder &_fbb,
flatbuffers::Offset<flatbuffers::Vector<uint8_t>> lcl = 0,
flatbuffers::Offset<flatbuffers::Vector<uint8_t>> block_buffer = 0) {
HistoryLedgerBlockBuilder builder_(_fbb);
builder_.add_block_buffer(block_buffer);
builder_.add_lcl(lcl);
return builder_.Finish();
}
inline flatbuffers::Offset<HistoryLedgerBlock> CreateHistoryLedgerBlockDirect(
flatbuffers::FlatBufferBuilder &_fbb,
const std::vector<uint8_t> *lcl = nullptr,
const std::vector<uint8_t> *block_buffer = nullptr) {
auto lcl__ = lcl ? _fbb.CreateVector<uint8_t>(*lcl) : 0;
auto block_buffer__ = block_buffer ? _fbb.CreateVector<uint8_t>(*block_buffer) : 0;
return msg::fbuf::p2pmsg::CreateHistoryLedgerBlock(
_fbb,
lcl__,
block_buffer__);
}
struct Hpfs_Request_Message FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
typedef Hpfs_Request_MessageBuilder Builder;
enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
@@ -1425,7 +1191,6 @@ struct Hpfs_Request_MessageBuilder {
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
Hpfs_Request_MessageBuilder &operator=(const Hpfs_Request_MessageBuilder &);
flatbuffers::Offset<Hpfs_Request_Message> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<Hpfs_Request_Message>(end);
@@ -1562,7 +1327,6 @@ struct Hpfs_Response_MessageBuilder {
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
Hpfs_Response_MessageBuilder &operator=(const Hpfs_Response_MessageBuilder &);
flatbuffers::Offset<Hpfs_Response_Message> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<Hpfs_Response_Message>(end);
@@ -1635,7 +1399,6 @@ struct Fs_Entry_ResponseBuilder {
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
Fs_Entry_ResponseBuilder &operator=(const Fs_Entry_ResponseBuilder &);
flatbuffers::Offset<Fs_Entry_Response> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<Fs_Entry_Response>(end);
@@ -1701,7 +1464,6 @@ struct File_HashMap_ResponseBuilder {
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
File_HashMap_ResponseBuilder &operator=(const File_HashMap_ResponseBuilder &);
flatbuffers::Offset<File_HashMap_Response> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<File_HashMap_Response>(end);
@@ -1771,7 +1533,6 @@ struct Block_ResponseBuilder {
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
Block_ResponseBuilder &operator=(const Block_ResponseBuilder &);
flatbuffers::Offset<Block_Response> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<Block_Response>(end);
@@ -1853,7 +1614,6 @@ struct Hpfs_FS_Hash_EntryBuilder {
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
Hpfs_FS_Hash_EntryBuilder &operator=(const Hpfs_FS_Hash_EntryBuilder &);
flatbuffers::Offset<Hpfs_FS_Hash_Entry> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<Hpfs_FS_Hash_Entry>(end);
@@ -1916,7 +1676,6 @@ struct Peer_Requirement_Announcement_MessageBuilder {
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
Peer_Requirement_Announcement_MessageBuilder &operator=(const Peer_Requirement_Announcement_MessageBuilder &);
flatbuffers::Offset<Peer_Requirement_Announcement_Message> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<Peer_Requirement_Announcement_Message>(end);
@@ -1972,7 +1731,6 @@ struct Available_Capacity_Announcement_MessageBuilder {
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
Available_Capacity_Announcement_MessageBuilder &operator=(const Available_Capacity_Announcement_MessageBuilder &);
flatbuffers::Offset<Available_Capacity_Announcement_Message> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<Available_Capacity_Announcement_Message>(end);
@@ -2006,7 +1764,6 @@ struct Peer_List_Request_MessageBuilder {
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
Peer_List_Request_MessageBuilder &operator=(const Peer_List_Request_MessageBuilder &);
flatbuffers::Offset<Peer_List_Request_Message> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<Peer_List_Request_Message>(end);
@@ -2051,7 +1808,6 @@ struct Peer_List_Response_MessageBuilder {
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
Peer_List_Response_MessageBuilder &operator=(const Peer_List_Response_MessageBuilder &);
flatbuffers::Offset<Peer_List_Response_Message> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<Peer_List_Response_Message>(end);
@@ -2139,7 +1895,6 @@ struct Peer_PropertiesBuilder {
: fbb_(_fbb) {
start_ = fbb_.StartTable();
}
Peer_PropertiesBuilder &operator=(const Peer_PropertiesBuilder &);
flatbuffers::Offset<Peer_Properties> Finish() {
const auto end = fbb_.EndTable(start_);
auto o = flatbuffers::Offset<Peer_Properties>(end);
@@ -2209,14 +1964,6 @@ inline bool VerifyMessage(flatbuffers::Verifier &verifier, const void *obj, Mess
auto ptr = reinterpret_cast<const msg::fbuf::p2pmsg::Hpfs_Response_Message *>(obj);
return verifier.VerifyTable(ptr);
}
case Message_History_Request_Message: {
auto ptr = reinterpret_cast<const msg::fbuf::p2pmsg::History_Request_Message *>(obj);
return verifier.VerifyTable(ptr);
}
case Message_History_Response_Message: {
auto ptr = reinterpret_cast<const msg::fbuf::p2pmsg::History_Response_Message *>(obj);
return verifier.VerifyTable(ptr);
}
case Message_Peer_Requirement_Announcement_Message: {
auto ptr = reinterpret_cast<const msg::fbuf::p2pmsg::Peer_Requirement_Announcement_Message *>(obj);
return verifier.VerifyTable(ptr);

214
src/msg/fbuf/p2pmsg_helpers.cpp
View File

@@ -195,32 +195,11 @@ namespace msg::fbuf::p2pmsg
}
/**
* Creates a history response stuct from the given histrory response message.
* @param msg Flatbuffer History response message received from the peer.
* @return A History response struct representing the message.
*/
const p2p::history_response create_history_response_from_msg(const History_Response_Message &msg)
{
p2p::history_response hr;
if (msg.requester_lcl())
hr.requester_lcl = flatbuff_bytes_to_sv(msg.requester_lcl());
if (msg.hist_ledger_blocks())
hr.hist_ledger_blocks = flatbuf_historyledgermap_to_historyledgermap(msg.hist_ledger_blocks());
if (msg.error())
hr.error = (p2p::LEDGER_RESPONSE_ERROR)msg.error();
return hr;
}
/**
* Creates a proposal stuct from the given proposal message.
* @param The Flatbuffer poposal received from the peer.
* @return A proposal struct representing the message.
*/
const p2p::proposal create_proposal_from_msg(const Proposal_Message &msg, const flatbuffers::Vector<uint8_t> *pubkey, const uint64_t timestamp, const flatbuffers::Vector<uint8_t> *lcl)
* Creates a proposal stuct from the given proposal message.
* @param msg The Flatbuffer poposal received from the peer.
* @return A proposal struct representing the message.
*/
const p2p::proposal create_proposal_from_msg(const Proposal_Message &msg, const flatbuffers::Vector<uint8_t> *pubkey, const uint64_t timestamp, const flatbuffers::Vector<uint8_t> *lcl, const Sequence_Hash &last_primary_shard_id_msg)
{
p2p::proposal p;
@@ -235,6 +214,17 @@ namespace msg::fbuf::p2pmsg
p.state_hash = flatbuff_bytes_to_sv(msg.state_hash());
p.patch_hash = flatbuff_bytes_to_sv(msg.patch_hash());
p2p::sequence_hash last_primary_shard_id;
last_primary_shard_id.seq_no = last_primary_shard_id_msg.shard_seq_no();
last_primary_shard_id.hash = flatbuff_bytes_to_hash(last_primary_shard_id_msg.shard_hash());
p.last_primary_shard_id = last_primary_shard_id;
p2p::sequence_hash last_blob_shard_id;
const Sequence_Hash &last_blob_shard_id_msg = *msg.last_blob_shard_id();
last_blob_shard_id.seq_no = last_blob_shard_id_msg.shard_seq_no();
last_blob_shard_id.hash = flatbuff_bytes_to_hash(last_blob_shard_id_msg.shard_hash());
p.last_blob_shard_id = last_blob_shard_id;
if (msg.users())
p.users = flatbuf_bytearrayvector_to_stringlist(msg.users());
@@ -250,24 +240,6 @@ namespace msg::fbuf::p2pmsg
return p;
}
/**
* Creates a history request struct from the given history request message.
* @param msg Flatbuffer History request message received from the peer.
* @return A History request struct representing the message.
*/
const p2p::history_request create_history_request_from_msg(const History_Request_Message &msg, const flatbuffers::Vector<uint8_t> *lcl)
{
p2p::history_request hr;
if (lcl)
hr.requester_lcl = flatbuff_bytes_to_sv(lcl);
if (msg.required_lcl())
hr.required_lcl = flatbuff_bytes_to_sv(msg.required_lcl());
return hr;
}
/**
* Creates a hpfs request struct from the given hpfs request message.
* @param msg Flatbuffer State request message received from the peer.
@@ -322,7 +294,7 @@ namespace msg::fbuf::p2pmsg
builder.Finish(message); // Finished building message content to get serialised content.
// Now that we have built the content message
create_containermsg_from_content(container_builder, builder, {}, false);
create_containermsg_from_content(container_builder, builder, {}, {}, false);
}
/**
@@ -345,7 +317,7 @@ namespace msg::fbuf::p2pmsg
// Now that we have built the content message,
// we need to sign it and place it inside a container message.
create_containermsg_from_content(container_builder, builder, {}, true);
create_containermsg_from_content(container_builder, builder, {}, {}, true);
}
void create_msg_from_nonunl_proposal(flatbuffers::FlatBufferBuilder &container_builder, const p2p::nonunl_proposal &nup)
@@ -362,7 +334,7 @@ namespace msg::fbuf::p2pmsg
// Now that we have built the content message,
// we need to sign it and place it inside a container message.
create_containermsg_from_content(container_builder, builder, {}, false);
create_containermsg_from_content(container_builder, builder, {}, {}, false);
}
/**
@@ -375,6 +347,11 @@ namespace msg::fbuf::p2pmsg
// todo:get a average propsal message size and allocate content builder based on that.
flatbuffers::FlatBufferBuilder builder(1024);
const flatbuffers::Offset<Sequence_Hash> last_blob_shard_id_msg = CreateSequence_Hash(
builder,
p.last_blob_shard_id.seq_no,
hash_to_flatbuff_bytes(builder, p.last_blob_shard_id.hash));
const flatbuffers::Offset<Proposal_Message> proposal =
CreateProposal_Message(
builder,
@@ -384,6 +361,7 @@ namespace msg::fbuf::p2pmsg
sv_to_flatbuff_bytes(builder, p.nonce),
stringlist_to_flatbuf_bytearrayvector(builder, p.users),
stringlist_to_flatbuf_bytearrayvector(builder, p.input_hashes),
last_blob_shard_id_msg,
sv_to_flatbuff_bytes(builder, p.output_hash),
sv_to_flatbuff_bytes(builder, p.output_sig),
hash_to_flatbuff_bytes(builder, p.state_hash),
@@ -394,7 +372,7 @@ namespace msg::fbuf::p2pmsg
// Now that we have built the content message,
// we need to sign it and place it inside a container message.
create_containermsg_from_content(container_builder, builder, p.lcl, true);
create_containermsg_from_content(container_builder, builder, p.lcl, p.last_primary_shard_id, true);
}
/**
@@ -402,8 +380,9 @@ namespace msg::fbuf::p2pmsg
* @param container_builder Flatbuffer builder for the container message.
* @param msg The message to be sent as NPL message.
* @param lcl Lcl value to be passed in the container message.
* @param last_primary_shard_id Last primary shard id.
*/
void create_msg_from_npl_output(flatbuffers::FlatBufferBuilder &container_builder, const std::string_view &msg, std::string_view lcl)
void create_msg_from_npl_output(flatbuffers::FlatBufferBuilder &container_builder, const std::string_view &msg, std::string_view lcl, const p2p::sequence_hash &last_primary_shard_id)
{
flatbuffers::FlatBufferBuilder builder(1024);
@@ -417,61 +396,17 @@ namespace msg::fbuf::p2pmsg
// Now that we have built the content message,
// we need to sign it and place it inside a container message.
create_containermsg_from_content(container_builder, builder, lcl, true);
}
/**
* Create history request message from the given history request struct.
* @param container_builder Flatbuffer builder for the container message.
* @param hr The History request struct to be placed in the container message.
*/
void create_msg_from_history_request(flatbuffers::FlatBufferBuilder &container_builder, const p2p::history_request &hr)
{
flatbuffers::FlatBufferBuilder builder(1024);
flatbuffers::Offset<History_Request_Message> hrmsg =
CreateHistory_Request_Message(
builder,
sv_to_flatbuff_bytes(builder, hr.required_lcl));
flatbuffers::Offset<Content> message = CreateContent(builder, Message_History_Request_Message, hrmsg.Union());
builder.Finish(message); // Finished building message content to get serialised content.
// Now that we have built the content message,
// we need to sign it and place it inside a container message.
create_containermsg_from_content(container_builder, builder, hr.requester_lcl, false);
}
/**
* Create history response message from the given history response struct.
* @param container_builder Flatbuffer builder for the container message.
* @param hr The History response struct to be placed in the container message.
*/
void create_msg_from_history_response(flatbuffers::FlatBufferBuilder &container_builder, const p2p::history_response &hr)
{
flatbuffers::FlatBufferBuilder builder(1024);
flatbuffers::Offset<History_Response_Message> hrmsg =
CreateHistory_Response_Message(
builder,
sv_to_flatbuff_bytes(builder, hr.requester_lcl),
historyledgermap_to_flatbuf_historyledgermap(builder, hr.hist_ledger_blocks),
(Ledger_Response_Error)hr.error);
flatbuffers::Offset<Content> message = CreateContent(builder, Message_History_Response_Message, hrmsg.Union());
builder.Finish(message); // Finished building message content to get serialised content.
// Now that we have built the content message,
// we need to sign it and place it inside a container message.
create_containermsg_from_content(container_builder, builder, {}, false);
create_containermsg_from_content(container_builder, builder, lcl, last_primary_shard_id, true);
}
/**
* Create hpfs request message from the given hpfs request struct.
* @param container_builder Flatbuffer builder for the container message.
* @param hr The hpfs request struct to be placed in the container message.
* @param lcl Lcl to be include in the container msg.
* @param last_primary_shard_id Last primary shard id.
*/
void create_msg_from_hpfs_request(flatbuffers::FlatBufferBuilder &container_builder, const p2p::hpfs_request &hr, std::string_view lcl)
void create_msg_from_hpfs_request(flatbuffers::FlatBufferBuilder &container_builder, const p2p::hpfs_request &hr, std::string_view lcl, const p2p::sequence_hash &last_primary_shard_id)
{
flatbuffers::FlatBufferBuilder builder(1024);
@@ -489,7 +424,7 @@ namespace msg::fbuf::p2pmsg
// Now that we have built the content message,
// we need to sign it and place it inside a container message.
create_containermsg_from_content(container_builder, builder, lcl, false);
create_containermsg_from_content(container_builder, builder, lcl, last_primary_shard_id, false);
}
/**
@@ -500,10 +435,11 @@ namespace msg::fbuf::p2pmsg
* @param hash_nodes File or directory entries with hashes in the given parent path.
* @param expected_hash The exptected hash of the requested path.
* @param lcl Lcl to be include in the container msg.
* @param last_primary_shard_id Last primary shard id.
*/
void create_msg_from_fsentry_response(
flatbuffers::FlatBufferBuilder &container_builder, const std::string_view path, const uint32_t mount_id,
std::vector<hpfs::child_hash_node> &hash_nodes, util::h32 expected_hash, std::string_view lcl)
std::vector<hpfs::child_hash_node> &hash_nodes, util::h32 expected_hash, std::string_view lcl, const p2p::sequence_hash &last_primary_shard_id)
{
flatbuffers::FlatBufferBuilder builder(1024);
@@ -523,7 +459,7 @@ namespace msg::fbuf::p2pmsg
// Now that we have built the content message,
// we need to sign it and place it inside a container message.
create_containermsg_from_content(container_builder, builder, lcl, true);
create_containermsg_from_content(container_builder, builder, lcl, last_primary_shard_id, true);
}
/**
@@ -533,10 +469,11 @@ namespace msg::fbuf::p2pmsg
* @param mount_id The mount id of the relavent hpfs mount.
* @param hashmap Hashmap of the file
* @param lcl Lcl to be include in the container msg.
* @param last_primary_shard_id Last primary shard id.
*/
void create_msg_from_filehashmap_response(
flatbuffers::FlatBufferBuilder &container_builder, std::string_view path, const uint32_t mount_id,
std::vector<util::h32> &hashmap, std::size_t file_length, util::h32 expected_hash, std::string_view lcl)
std::vector<util::h32> &hashmap, std::size_t file_length, util::h32 expected_hash, std::string_view lcl, const p2p::sequence_hash &last_primary_shard_id)
{
// todo:get a average propsal message size and allocate content builder based on that.
flatbuffers::FlatBufferBuilder builder(1024);
@@ -561,7 +498,7 @@ namespace msg::fbuf::p2pmsg
// Now that we have built the content message,
// we need to sign it and place it inside a container message.
create_containermsg_from_content(container_builder, builder, lcl, true);
create_containermsg_from_content(container_builder, builder, lcl, last_primary_shard_id, true);
}
/**
@@ -570,8 +507,10 @@ namespace msg::fbuf::p2pmsg
* @param block_resp Block response struct to place in the message.
* @param mount_id The mount id of the relavent hpfs mount.
* @param lcl Lcl to be include in the container message.
* @param last_primary_shard_id Last primary shard id.
*/
void create_msg_from_block_response(flatbuffers::FlatBufferBuilder &container_builder, p2p::block_response &block_resp, const uint32_t mount_id, std::string_view lcl)
void create_msg_from_block_response(flatbuffers::FlatBufferBuilder &container_builder, p2p::block_response &block_resp, const uint32_t mount_id,
std::string_view lcl, const p2p::sequence_hash &last_primary_shard_id)
{
// todo:get a average propsal message size and allocate content builder based on that.
flatbuffers::FlatBufferBuilder builder(1024);
@@ -594,7 +533,7 @@ namespace msg::fbuf::p2pmsg
// Now that we have built the content message,
// we need to sign it and place it inside a container message.
create_containermsg_from_content(container_builder, builder, lcl, true);
create_containermsg_from_content(container_builder, builder, lcl, last_primary_shard_id, true);
}
/**
@@ -603,7 +542,8 @@ namespace msg::fbuf::p2pmsg
* @param need_consensus_msg_forwarding True if number of connections are below threshold and false otherwise.
* @param lcl Lcl value to be passed in the container message.
*/
void create_msg_from_peer_requirement_announcement(flatbuffers::FlatBufferBuilder &container_builder, const bool need_consensus_msg_forwarding, std::string_view lcl)
void create_msg_from_peer_requirement_announcement(flatbuffers::FlatBufferBuilder &container_builder, const bool need_consensus_msg_forwarding,
std::string_view lcl, const p2p::sequence_hash &last_primary_shard_id)
{
flatbuffers::FlatBufferBuilder builder(1024);
@@ -616,7 +556,7 @@ namespace msg::fbuf::p2pmsg
builder.Finish(message); // Finished building message content to get serialised content.
// Now that we have built the content message,
create_containermsg_from_content(container_builder, builder, lcl, false);
create_containermsg_from_content(container_builder, builder, lcl, last_primary_shard_id, false);
}
/**
@@ -626,7 +566,8 @@ namespace msg::fbuf::p2pmsg
* @param timestamp Announced timestamp.
* @param lcl Lcl value to be passed in the container message.
*/
void create_msg_from_available_capacity_announcement(flatbuffers::FlatBufferBuilder &container_builder, const int16_t &available_capacity, const uint64_t &timestamp, std::string_view lcl)
void create_msg_from_available_capacity_announcement(flatbuffers::FlatBufferBuilder &container_builder, const int16_t &available_capacity, const uint64_t &timestamp,
std::string_view lcl, const p2p::sequence_hash &last_primary_shard_id)
{
flatbuffers::FlatBufferBuilder builder(1024);
@@ -640,7 +581,7 @@ namespace msg::fbuf::p2pmsg
builder.Finish(message); // Finished building message content to get serialised content.
// Now that we have built the content message,
create_containermsg_from_content(container_builder, builder, lcl, false);
create_containermsg_from_content(container_builder, builder, lcl, last_primary_shard_id, false);
}
/**
@@ -648,7 +589,7 @@ namespace msg::fbuf::p2pmsg
* @param container_builder Flatbuffer builder for the container message.
* @param lcl Lcl value to be passed in the container message.
*/
void create_msg_from_peer_list_request(flatbuffers::FlatBufferBuilder &container_builder, std::string_view lcl)
void create_msg_from_peer_list_request(flatbuffers::FlatBufferBuilder &container_builder, std::string_view lcl, const p2p::sequence_hash &last_primary_shard_id)
{
flatbuffers::FlatBufferBuilder builder(1024);
@@ -660,7 +601,7 @@ namespace msg::fbuf::p2pmsg
builder.Finish(message); // Finished building message content to get serialised content.
// Now that we have built the content message,
create_containermsg_from_content(container_builder, builder, lcl, false);
create_containermsg_from_content(container_builder, builder, lcl, last_primary_shard_id, false);
}
/**
@@ -670,7 +611,8 @@ namespace msg::fbuf::p2pmsg
* @param skipping_peer Peer that does not need to be sent.
* @param lcl Lcl value to be passed in the container message.
*/
void create_msg_from_peer_list_response(flatbuffers::FlatBufferBuilder &container_builder, const std::vector<conf::peer_properties> &peers, const std::optional<conf::peer_ip_port> &skipping_ip_port, std::string_view lcl)
void create_msg_from_peer_list_response(flatbuffers::FlatBufferBuilder &container_builder, const std::vector<conf::peer_properties> &peers, const std::optional<conf::peer_ip_port> &skipping_ip_port,
std::string_view lcl, const p2p::sequence_hash &last_primary_shard_id)
{
flatbuffers::FlatBufferBuilder builder(1024);
@@ -683,7 +625,7 @@ namespace msg::fbuf::p2pmsg
builder.Finish(message); // Finished building message content to get serialised content.
// Now that we have built the content message,
create_containermsg_from_content(container_builder, builder, lcl, false);
create_containermsg_from_content(container_builder, builder, lcl, last_primary_shard_id, false);
}
/**
@@ -694,7 +636,7 @@ namespace msg::fbuf::p2pmsg
* @param sign Whether to sign the message content.
*/
void create_containermsg_from_content(
flatbuffers::FlatBufferBuilder &container_builder, const flatbuffers::FlatBufferBuilder &content_builder, std::string_view lcl, const bool sign)
flatbuffers::FlatBufferBuilder &container_builder, const flatbuffers::FlatBufferBuilder &content_builder, std::string_view lcl, const p2p::sequence_hash &last_primary_shard_id, const bool sign)
{
const uint8_t *content_buf = content_builder.GetBufferPointer();
const flatbuffers::uoffset_t content_size = content_builder.GetSize();
@@ -719,12 +661,18 @@ namespace msg::fbuf::p2pmsg
if (!lcl.empty())
lcl_offset = sv_to_flatbuff_bytes(container_builder, lcl);
const flatbuffers::Offset<Sequence_Hash> last_primary_shard_id_msg = CreateSequence_Hash(
container_builder,
last_primary_shard_id.seq_no,
hash_to_flatbuff_bytes(container_builder, last_primary_shard_id.hash));
const flatbuffers::Offset<Container> container_message = CreateContainer(
container_builder,
util::PEERMSG_VERSION,
util::get_epoch_milliseconds(),
pubkey_offset,
lcl_offset,
last_primary_shard_id_msg,
sig_offset,
content);
@@ -784,46 +732,6 @@ namespace msg::fbuf::p2pmsg
return builder.CreateVector(fbvec);
}
const std::map<uint64_t, const p2p::history_ledger_block>
flatbuf_historyledgermap_to_historyledgermap(const flatbuffers::Vector<flatbuffers::Offset<HistoryLedgerBlockPair>> *fbvec)
{
std::map<uint64_t, const p2p::history_ledger_block> map;
for (const HistoryLedgerBlockPair *pair : *fbvec)
{
std::list<usr::submitted_user_input> msglist;
p2p::history_ledger_block ledger;
ledger.lcl = flatbuff_bytes_to_sv(pair->ledger()->lcl());
auto raw = pair->ledger()->block_buffer();
ledger.block_buffer = std::vector<uint8_t>(raw->begin(), raw->end());
map.emplace(pair->seq_no(), std::move(ledger));
}
return map;
}
const flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<HistoryLedgerBlockPair>>>
historyledgermap_to_flatbuf_historyledgermap(flatbuffers::FlatBufferBuilder &builder, const std::map<uint64_t, const p2p::history_ledger_block> &map)
{
std::vector<flatbuffers::Offset<HistoryLedgerBlockPair>> fbvec;
fbvec.reserve(map.size());
for (auto const &[seq_no, ledger] : map)
{
flatbuffers::Offset<HistoryLedgerBlock> history_ledger = CreateHistoryLedgerBlock(
builder,
sv_to_flatbuff_bytes(builder, ledger.lcl),
builder.CreateVector(ledger.block_buffer));
fbvec.push_back(CreateHistoryLedgerBlockPair(
builder,
seq_no,
history_ledger));
}
return builder.CreateVector(fbvec);
}
void flatbuf_hpfsfshashentry_to_hpfsfshashentry(std::unordered_map<std::string, p2p::hpfs_fs_hash_entry> &fs_entries, const flatbuffers::Vector<flatbuffers::Offset<Hpfs_FS_Hash_Entry>> *fhashes)
{
for (const Hpfs_FS_Hash_Entry *f_hash : *fhashes)

40
src/msg/fbuf/p2pmsg_helpers.hpp
View File

@@ -30,11 +30,7 @@ namespace msg::fbuf::p2pmsg
const p2p::nonunl_proposal create_nonunl_proposal_from_msg(const NonUnl_Proposal_Message &msg, const uint64_t timestamp);
const p2p::proposal create_proposal_from_msg(const Proposal_Message &msg, const flatbuffers::Vector<uint8_t> *pubkey, const uint64_t timestamp, const flatbuffers::Vector<uint8_t> *lcl);
const p2p::history_request create_history_request_from_msg(const History_Request_Message &msg, const flatbuffers::Vector<uint8_t> *lcl);
const p2p::history_response create_history_response_from_msg(const History_Response_Message &msg);
const p2p::proposal create_proposal_from_msg(const Proposal_Message &msg, const flatbuffers::Vector<uint8_t> *pubkey, const uint64_t timestamp, const flatbuffers::Vector<uint8_t> *lcl, const Sequence_Hash &last_primary_shard_id_msg);
const p2p::hpfs_request create_hpfs_request_from_msg(const Hpfs_Request_Message &msg);
@@ -49,34 +45,34 @@ namespace msg::fbuf::p2pmsg
void create_msg_from_proposal(flatbuffers::FlatBufferBuilder &container_builder, const p2p::proposal &p);
void create_msg_from_history_request(flatbuffers::FlatBufferBuilder &container_builder, const p2p::history_request &hr);
void create_msg_from_npl_output(flatbuffers::FlatBufferBuilder &container_builder, const std::string_view &msg, std::string_view lcl, const p2p::sequence_hash &last_primary_shard_id);
void create_msg_from_history_response(flatbuffers::FlatBufferBuilder &container_builder, const p2p::history_response &hr);
void create_msg_from_npl_output(flatbuffers::FlatBufferBuilder &container_builder, const std::string_view &msg, std::string_view lcl);
void create_msg_from_hpfs_request(flatbuffers::FlatBufferBuilder &container_builder, const p2p::hpfs_request &hr, std::string_view lcl);
void create_msg_from_hpfs_request(flatbuffers::FlatBufferBuilder &container_builder, const p2p::hpfs_request &hr, std::string_view lcl, const p2p::sequence_hash &last_primary_shard_id);
void create_msg_from_fsentry_response(
flatbuffers::FlatBufferBuilder &container_builder, const std::string_view path, const uint32_t mount_id,
std::vector<hpfs::child_hash_node> &hash_nodes, util::h32 expected_hash, std::string_view lcl);
std::vector<hpfs::child_hash_node> &hash_nodes, util::h32 expected_hash, std::string_view lcl, const p2p::sequence_hash &last_primary_shard_id);
void create_msg_from_filehashmap_response(
flatbuffers::FlatBufferBuilder &container_builder, std::string_view path, const uint32_t mount_id,
std::vector<util::h32> &hashmap, std::size_t file_length, util::h32 expected_hash, std::string_view lcl);
std::vector<util::h32> &hashmap, std::size_t file_length, util::h32 expected_hash, std::string_view lcl, const p2p::sequence_hash &last_primary_shard_id);
void create_msg_from_block_response(flatbuffers::FlatBufferBuilder &container_builder, p2p::block_response &block_resp, const uint32_t mount_id, std::string_view lcl);
void create_msg_from_block_response(flatbuffers::FlatBufferBuilder &container_builder, p2p::block_response &block_resp, const uint32_t mount_id,
std::string_view lcl, const p2p::sequence_hash &last_primary_shard_id);
void create_containermsg_from_content(
flatbuffers::FlatBufferBuilder &container_builder, const flatbuffers::FlatBufferBuilder &content_builder, std::string_view lcl, const bool sign);
flatbuffers::FlatBufferBuilder &container_builder, const flatbuffers::FlatBufferBuilder &content_builder, std::string_view lcl, const p2p::sequence_hash &last_primary_shard_id, const bool sign);
void create_msg_from_peer_requirement_announcement(flatbuffers::FlatBufferBuilder &container_builder, const bool need_consensus_msg_forwarding, std::string_view lcl);
void create_msg_from_peer_requirement_announcement(flatbuffers::FlatBufferBuilder &container_builder, const bool need_consensus_msg_forwarding,
std::string_view lcl, const p2p::sequence_hash &last_primary_shard_id);
void create_msg_from_available_capacity_announcement(flatbuffers::FlatBufferBuilder &container_builder, const int16_t &available_capacity, const uint64_t &timestamp, std::string_view lcl);
void create_msg_from_available_capacity_announcement(flatbuffers::FlatBufferBuilder &container_builder, const int16_t &available_capacity, const uint64_t &timestamp,
std::string_view lcl, const p2p::sequence_hash &last_primary_shard_id);
void create_msg_from_peer_list_request(flatbuffers::FlatBufferBuilder &container_builder, std::string_view lcl);
void create_msg_from_peer_list_request(flatbuffers::FlatBufferBuilder &container_builder, std::string_view lcl, const p2p::sequence_hash &last_primary_shard_id);
void create_msg_from_peer_list_response(flatbuffers::FlatBufferBuilder &container_builder, const std::vector<conf::peer_properties> &peers, const std::optional<conf::peer_ip_port> &skipping_ip_port, std::string_view lcl);
void create_msg_from_peer_list_response(flatbuffers::FlatBufferBuilder &container_builder, const std::vector<conf::peer_properties> &peers, const std::optional<conf::peer_ip_port> &skipping_ip_port,
std::string_view lcl, const p2p::sequence_hash &last_primary_shard_id);
//---Conversion helpers from flatbuffers data types to std data types---//
@@ -88,15 +84,9 @@ namespace msg::fbuf::p2pmsg
const flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<UserInputGroup>>>
user_input_map_to_flatbuf_user_input_group(flatbuffers::FlatBufferBuilder &builder, const std::unordered_map<std::string, std::list<usr::submitted_user_input>> &map);
const std::map<uint64_t, const p2p::history_ledger_block>
flatbuf_historyledgermap_to_historyledgermap(const flatbuffers::Vector<flatbuffers::Offset<HistoryLedgerBlockPair>> *fbvec);
const std::vector<conf::peer_properties>
flatbuf_peer_propertieslist_to_peer_propertiesvector(const flatbuffers::Vector<flatbuffers::Offset<Peer_Properties>> *fbvec);
const flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<HistoryLedgerBlockPair>>>
historyledgermap_to_flatbuf_historyledgermap(flatbuffers::FlatBufferBuilder &builder, const std::map<uint64_t, const p2p::history_ledger_block> &map);
const flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<Peer_Properties>>>
peer_propertiesvector_to_flatbuf_peer_propertieslist(flatbuffers::FlatBufferBuilder &builder, const std::vector<conf::peer_properties> &peers, const std::optional<conf::peer_ip_port> &skipping_ip_port);

19
src/p2p/p2p.cpp
View File

@@ -4,7 +4,7 @@
#include "../util/util.hpp"
#include "../hplog.hpp"
#include "../msg/fbuf/p2pmsg_helpers.hpp"
#include "../ledger.hpp"
#include "../ledger/ledger.hpp"
#include "p2p.hpp"
#include "self_node.hpp"
#include "../unl.hpp"
@@ -289,7 +289,7 @@ namespace p2p
void send_peer_requirement_announcement(const bool need_consensus_msg_forwarding, peer_comm_session *session)
{
flatbuffers::FlatBufferBuilder fbuf(1024);
msg::fbuf::p2pmsg::create_msg_from_peer_requirement_announcement(fbuf, need_consensus_msg_forwarding, ledger::ctx.get_lcl());
msg::fbuf::p2pmsg::create_msg_from_peer_requirement_announcement(fbuf, need_consensus_msg_forwarding, ledger::ctx.get_lcl(), ledger::ctx.get_last_primary_shard_id());
if (session)
{
std::string_view msg = std::string_view(
@@ -310,7 +310,7 @@ namespace p2p
{
const uint64_t time_now = util::get_epoch_milliseconds();
flatbuffers::FlatBufferBuilder fbuf(1024);
msg::fbuf::p2pmsg::create_msg_from_available_capacity_announcement(fbuf, available_capacity, time_now, ledger::ctx.get_lcl());
msg::fbuf::p2pmsg::create_msg_from_available_capacity_announcement(fbuf, available_capacity, time_now, ledger::ctx.get_lcl(), ledger::ctx.get_last_primary_shard_id());
broadcast_message(fbuf, false);
}
@@ -321,7 +321,7 @@ namespace p2p
void send_known_peer_list(peer_comm_session *session)
{
flatbuffers::FlatBufferBuilder fbuf(1024);
msg::fbuf::p2pmsg::create_msg_from_peer_list_response(fbuf, ctx.server->req_known_remotes, session->known_ipport, ledger::ctx.get_lcl());
msg::fbuf::p2pmsg::create_msg_from_peer_list_response(fbuf, ctx.server->req_known_remotes, session->known_ipport, ledger::ctx.get_lcl(), ledger::ctx.get_last_primary_shard_id());
std::string_view msg = std::string_view(
reinterpret_cast<const char *>(fbuf.GetBufferPointer()), fbuf.GetSize());
session->send(msg);
@@ -355,7 +355,7 @@ namespace p2p
void send_peer_list_request()
{
flatbuffers::FlatBufferBuilder fbuf(1024);
msg::fbuf::p2pmsg::create_msg_from_peer_list_request(fbuf, ledger::ctx.get_lcl());
msg::fbuf::p2pmsg::create_msg_from_peer_list_request(fbuf, ledger::ctx.get_lcl(), ledger::ctx.get_last_primary_shard_id());
std::string target_pubkey;
send_message_to_random_peer(fbuf, target_pubkey);
LOG_DEBUG << "Peer list request: Requesting from [" << target_pubkey.substr(0, 10) << "]";
@@ -456,4 +456,13 @@ namespace p2p
}
}
/**
* This is a helper method for sequence_hash structure which enables printing it straight away.
*/
std::ostream &operator<<(std::ostream &output, const sequence_hash &seq_hash)
{
output << seq_hash.seq_no << "-" << seq_hash.hash;
return output;
}
} // namespace p2p

44
src/p2p/p2p.hpp
View File

@@ -18,6 +18,29 @@ namespace p2p
constexpr uint16_t HPFS_RES_LIST_CAP = 64; // Maximum state response count.
constexpr uint16_t PEER_LIST_CAP = 64; // Maximum peer count.
struct sequence_hash
{
uint64_t seq_no = 0;
util::h32 hash = util::h32_empty;
bool operator!=(const sequence_hash &seq_hash) const
{
return seq_no != seq_hash.seq_no || hash != seq_hash.hash;
}
bool operator==(const sequence_hash &seq_hash) const
{
return seq_no == seq_hash.seq_no && hash == seq_hash.hash;
}
bool operator<(const sequence_hash &seq_hash) const
{
return seq_no < seq_hash.seq_no || hash < seq_hash.hash;
}
};
// This is a helper method for sequence_hash structure which enables printing it straight away.
std::ostream &operator<<(std::ostream &output, const sequence_hash &seq_hash);
struct proposal
{
std::string pubkey;
@@ -29,6 +52,8 @@ namespace p2p
uint32_t roundtime = 0; // Roundtime of the proposer.
std::string nonce; // Random nonce that is used to reduce lcl predictability.
std::string lcl;
sequence_hash last_primary_shard_id;
sequence_hash last_blob_shard_id;
util::h32 state_hash; // Contract state hash.
util::h32 patch_hash; // Patch file hash.
std::set<std::string> users;
@@ -42,18 +67,6 @@ namespace p2p
std::unordered_map<std::string, std::list<usr::submitted_user_input>> user_inputs;
};
struct history_request
{
std::string requester_lcl;
std::string required_lcl;
};
struct history_ledger_block
{
std::string lcl;
std::vector<uint8_t> block_buffer;
};
struct peer_challenge
{
std::string contract_id;
@@ -75,13 +88,6 @@ namespace p2p
REQ_LEDGER_NOT_FOUND = 2
};
struct history_response
{
std::string requester_lcl;
std::map<uint64_t, const history_ledger_block> hist_ledger_blocks;
LEDGER_RESPONSE_ERROR error = LEDGER_RESPONSE_ERROR::NONE;
};
// Represents an NPL message sent by a peer.
struct npl_message
{

2
src/p2p/peer_comm_server.cpp
View File

@@ -1,7 +1,7 @@
#include "../comm/comm_server.hpp"
#include "../util/util.hpp"
#include "../msg/fbuf/p2pmsg_helpers.hpp"
#include "../ledger.hpp"
#include "../ledger/ledger.hpp"
#include "../unl.hpp"
#include "../conf.hpp"
#include "peer_comm_server.hpp"

44
src/p2p/peer_session_handler.cpp
View File

@@ -9,8 +9,7 @@
#include "../msg/fbuf/p2pmsg_content_generated.h"
#include "../msg/fbuf/p2pmsg_helpers.hpp"
#include "../msg/fbuf/common_helpers.hpp"
#include "../ledger/ledger_sample.hpp"
#include "../ledger.hpp"
#include "../ledger/ledger.hpp"
#include "peer_comm_session.hpp"
#include "p2p.hpp"
#include "../unl.hpp"
@@ -208,7 +207,7 @@ namespace p2p
LOG_DEBUG << "Hpfs contract fs request rejected. Maximum hpfs contract fs request count reached. " << session.display_name();
}
}
else if (hr.mount_id == ledger::ledger_sample::ledger_fs.mount_id)
else if (hr.mount_id == ledger::ledger_fs.mount_id)
{
// Check the cap and insert request with lock.
std::scoped_lock<std::mutex> lock(ctx.collected_msgs.ledger_hpfs_requests_mutex);
@@ -246,7 +245,7 @@ namespace p2p
LOG_DEBUG << "Contract hpfs response rejected. Maximum contract hpfs response count reached. " << session.display_name();
}
}
else if (ledger::ledger_sample::ledger_sync_worker.is_syncing && resp_msg->mount_id() == ledger::ledger_sample::ledger_fs.mount_id)
else if (ledger::ledger_sync_worker.is_syncing && resp_msg->mount_id() == ledger::ledger_fs.mount_id)
{
// Check the cap and insert state_response with lock.
std::scoped_lock<std::mutex> lock(ctx.collected_msgs.ledger_hpfs_responses_mutex);
@@ -263,41 +262,6 @@ namespace p2p
}
}
}
else if (content_message_type == p2pmsg::Message_History_Request_Message) //message is a lcl history request message
{
// Check the cap and insert request with lock.
std::scoped_lock<std::mutex> lock(ledger::sync_ctx.list_mutex);
// If max number of history requests reached skip the rest.
if (ledger::sync_ctx.collected_history_requests.size() < ledger::HISTORY_REQ_LIST_CAP)
{
const p2p::history_request hr = p2pmsg::create_history_request_from_msg(*content->message_as_History_Request_Message(), container->lcl());
ledger::sync_ctx.collected_history_requests.push_back(std::make_pair(session.pubkey, std::move(hr)));
}
else
{
LOG_DEBUG << "History request rejected. Maximum history request count reached. " << session.display_name();
}
}
else if (content_message_type == p2pmsg::Message_History_Response_Message) //message is a lcl history response message
{
if (ledger::sync_ctx.is_syncing) // Only accept history responses if ledger is syncing.
{
// Check the cap and insert response with lock.
std::scoped_lock<std::mutex> lock(ledger::sync_ctx.list_mutex);
// If max number of history respoinses reached skip the rest.
if (ledger::sync_ctx.collected_history_responses.size() < ledger::HISTORY_RES_LIST_CAP)
{
const p2p::history_response hr = p2pmsg::create_history_response_from_msg(*content->message_as_History_Response_Message());
ledger::sync_ctx.collected_history_responses.push_back(std::move(hr));
}
else
{
LOG_DEBUG << "History response rejected. Maximum history response count reached. " << session.display_name();
}
}
}
else
{
session.increment_metric(comm::SESSION_THRESHOLDS::MAX_BADMSGS_PER_MINUTE, 1);
@@ -360,7 +324,7 @@ namespace p2p
return -1;
ctx.collected_msgs.proposals.push_back(
p2pmsg::create_proposal_from_msg(*content->message_as_Proposal_Message(), container->pubkey(), container->timestamp(), container->lcl()));
p2pmsg::create_proposal_from_msg(*content->message_as_Proposal_Message(), container->pubkey(), container->timestamp(), container->lcl(), *container->last_primary_shard_id()));
return 0;
}

1
src/pchheader.hpp
View File

@@ -25,7 +25,6 @@
#include <plog/Log.h>
#include <plog/Appenders/ColorConsoleAppender.h>
#include <poll.h>
#include <queue>
#include <readerwriterqueue/readerwriterqueue.h>
#include <set>
#include <shared_mutex>

8
src/sc/contract_mount.cpp
View File

@@ -13,16 +13,16 @@ namespace sc
if (acquire_rw_session() == -1 ||
conf::populate_patch_config() == -1 ||
get_hash(initial_state_hash, hpfs::RW_SESSION_NAME, hpfs::STATE_DIR_PATH) == -1 ||
get_hash(initial_patch_hash, hpfs::RW_SESSION_NAME, hpfs::PATCH_FILE_PATH) == -1 ||
get_hash(initial_state_hash, hpfs::RW_SESSION_NAME, STATE_DIR_PATH) == -1 ||
get_hash(initial_patch_hash, hpfs::RW_SESSION_NAME, PATCH_FILE_PATH) == -1 ||
release_rw_session() == -1)
{
LOG_ERROR << "Failed to prepare initial fs at mount " << mount_dir << ".";
return -1;
}
set_parent_hash(hpfs::STATE_DIR_PATH, initial_state_hash);
set_parent_hash(hpfs::PATCH_FILE_PATH, initial_patch_hash);
set_parent_hash(STATE_DIR_PATH, initial_state_hash);
set_parent_hash(PATCH_FILE_PATH, initial_patch_hash);
LOG_INFO << "Initial state: " << initial_state_hash << " | patch: " << initial_patch_hash;
return 0;
}

2
src/sc/contract_mount.hpp
View File

@@ -8,6 +8,8 @@
namespace sc
{
constexpr const char *STATE_DIR_PATH = "/state"; // State directory name.
constexpr const char *PATCH_FILE_PATH = "/patch.cfg"; // Config patch filename.
/**
* Represents contract file system mount.
*/

7
src/sc/contract_sync.cpp
View File

@@ -2,13 +2,14 @@
#include "./contract_sync.hpp"
#include "../unl.hpp"
#include "../hpfs/hpfs_mount.hpp"
#include "contract_mount.hpp"
namespace sc
{
void contract_sync::on_current_sync_state_acheived(const util::h32 &acheived_hash)
void contract_sync::on_current_sync_state_acheived(const hpfs::sync_target &synced_target)
{
if (current_target.vpath == hpfs::PATCH_FILE_PATH)
if (synced_target.vpath == PATCH_FILE_PATH)
{
// Appling new patch file changes to hpcore runtime.
if (conf::apply_patch_config(hpfs::RW_SESSION_NAME) == -1)
@@ -20,7 +21,7 @@ namespace sc
unl::update_unl_changes_from_patch();
// Update global hash tracker with the new patch file hash.
fs_mount->set_parent_hash(current_target.vpath, acheived_hash);
fs_mount->set_parent_hash(synced_target.vpath, synced_target.hash);
}
}
}

2
src/sc/contract_sync.hpp
View File

@@ -11,7 +11,7 @@ namespace sc
class contract_sync : public hpfs::hpfs_sync
{
private:
void on_current_sync_state_acheived(const util::h32 &acheived_hash);
void on_current_sync_state_acheived(const hpfs::sync_target &synced_target);
void swap_collected_responses();
};
} // namespace sc

18
src/sc/sc.cpp
View File

@@ -2,7 +2,7 @@
#include "../conf.hpp"
#include "../consensus.hpp"
#include "../hplog.hpp"
#include "../ledger.hpp"
#include "../ledger/ledger.hpp"
#include "../msg/fbuf/p2pmsg_helpers.hpp"
#include "../msg/controlmsg_common.hpp"
#include "../msg/controlmsg_parser.hpp"
@@ -47,9 +47,9 @@ namespace sc
void deinit()
{
contract_fs.deinit();
contract_server.deinit();
contract_sync_worker.deinit();
contract_server.deinit();
contract_fs.deinit();
}
/**
* Executes the contract process and passes the specified context arguments.
@@ -133,7 +133,7 @@ namespace sc
execv_args[j] = conf::cfg.contract.runtime_binexec_args[i].data();
execv_args[len - 1] = NULL;
const std::string current_dir = contract_fs.physical_path(ctx.args.hpfs_session_name, hpfs::STATE_DIR_PATH);
const std::string current_dir = contract_fs.physical_path(ctx.args.hpfs_session_name, STATE_DIR_PATH);
chdir(current_dir.c_str());
if (create_contract_log_files(ctx) == -1)
@@ -249,24 +249,24 @@ namespace sc
else
{
// Read the state hash if not in readonly mode.
if (contract_fs.get_hash(ctx.args.post_execution_state_hash, ctx.args.hpfs_session_name, hpfs::STATE_DIR_PATH) < 1)
if (contract_fs.get_hash(ctx.args.post_execution_state_hash, ctx.args.hpfs_session_name, STATE_DIR_PATH) < 1)
{
contract_fs.release_rw_session();
return -1;
}
util::h32 patch_hash;
const int patch_hash_result = contract_fs.get_hash(patch_hash, ctx.args.hpfs_session_name, hpfs::PATCH_FILE_PATH);
const int patch_hash_result = contract_fs.get_hash(patch_hash, ctx.args.hpfs_session_name, PATCH_FILE_PATH);
if (patch_hash_result == -1)
{
contract_fs.release_rw_session();
return -1;
}
else if (patch_hash_result == 1 && patch_hash != contract_fs.get_parent_hash(hpfs::PATCH_FILE_PATH))
else if (patch_hash_result == 1 && patch_hash != contract_fs.get_parent_hash(PATCH_FILE_PATH))
{
// Update global hash tracker of contract fs with the new patch file hash.
contract_fs.set_parent_hash(hpfs::PATCH_FILE_PATH, patch_hash);
contract_fs.set_parent_hash(PATCH_FILE_PATH, patch_hash);
// Denote that the patch file was updated by the SC.
consensus::is_patch_update_pending = true;
}
@@ -576,7 +576,7 @@ namespace sc
if (!output.empty())
{
flatbuffers::FlatBufferBuilder fbuf(1024);
msg::fbuf::p2pmsg::create_msg_from_npl_output(fbuf, output, ledger::ctx.get_lcl());
msg::fbuf::p2pmsg::create_msg_from_npl_output(fbuf, output, ledger::ctx.get_lcl(), ledger::ctx.get_last_primary_shard_id());
p2p::broadcast_message(fbuf, true, false, !conf::cfg.contract.is_npl_public);
}
}

2
src/usr/input_nonce_map.cpp
View File

@@ -1,6 +1,6 @@
#include "../pchheader.hpp"
#include "../util/util.hpp"
#include "../ledger.hpp"
#include "../ledger/ledger.hpp"
#include "input_nonce_map.hpp"
namespace usr

6
src/usr/user_input.hpp
View File

@@ -32,12 +32,6 @@ namespace usr
}
};
struct raw_user_input
{
const std::string pubkey;
const std::string input;
};
} // namespace usr
#endif

4
src/usr/usr.cpp
View File

@@ -6,7 +6,7 @@
#include "../conf.hpp"
#include "../crypto.hpp"
#include "../hplog.hpp"
#include "../ledger.hpp"
#include "../ledger/ledger.hpp"
#include "../util/buffer_store.hpp"
#include "../hpfs/hpfs_mount.hpp"
#include "usr.hpp"
@@ -464,7 +464,7 @@ namespace usr
util::fork_detach();
// before execution chdir into a valid the latest state data directory that contains an appbill.table
const std::string appbill_dir = sc::contract_fs.rw_dir + hpfs::STATE_DIR_PATH;
const std::string appbill_dir = sc::contract_fs.rw_dir + sc::STATE_DIR_PATH;
chdir(appbill_dir.c_str());
int ret = execv(execv_args[0], execv_args);
std::cerr << errno << ": Appbill process execv failed.\n";

31
src/util/util.cpp
View File

@@ -410,4 +410,35 @@ namespace util
lock.l_type = F_UNLCK;
return fcntl(fd, F_SETLKW, &lock);
}
/**
* Convert given little endian byte stream to uint64_t.
* @param data Byte stream to be converted.
* @return Returns converted uint64_t.
*/
uint64_t uint64_from_bytes(const uint8_t *data)
{
return ((uint64_t)data[0] << 56) +
((uint64_t)data[1] << 48) +
((uint64_t)data[2] << 40) +
((uint64_t)data[3] << 32) +
((uint64_t)data[4] << 24) +
((uint64_t)data[5] << 16) +
((uint64_t)data[6] << 8) +
((uint64_t)data[7]);
}
/**
* Convert given uint64_t to little endian byte stream.
* @param dest Byte stream to be populated.
* @param x uint64_t to be converted.
*/
void uint64_to_bytes(uint8_t *dest, uint64_t x)
{
for (int j = 0; j < 8; j++)
{
*(dest + (7 - j)) = x & 0xff;
x >>= 8;
}
}
} // namespace util

4
src/util/util.hpp
View File

@@ -85,6 +85,10 @@ namespace util
int set_lock(const int fd, struct flock &lock, const bool is_rwlock, const off_t start, const off_t len);
int release_lock(const int fd, struct flock &lock);
uint64_t uint64_from_bytes(const uint8_t *data);
void uint64_to_bytes(uint8_t *dest, const uint64_t x);
} // namespace util
#endif