From 2e2a11b90897fa2269d35f564d17329adf79ac78 Mon Sep 17 00:00:00 2001 From: Savinda Senevirathne Date: Fri, 13 Nov 2020 13:43:01 +0530 Subject: [PATCH] Improved history ledger integrity checks. (#149) * Code cleanup. * Resolving review comments. * Ledger history sync validation improvements and resolving review comments. * Ledger terminology related changes. * Ledger related terminology change updates. * Resolving review comments. --- src/ledger.cpp | 227 ++++++++++++++++++------ src/ledger.hpp | 4 + src/msg/fbuf/ledger_helpers.cpp | 20 +-- src/msg/fbuf/ledger_helpers.hpp | 6 +- src/msg/fbuf/ledger_schema.fbs | 4 +- src/msg/fbuf/ledger_schema_generated.h | 68 +++---- src/msg/fbuf/p2pmsg_content.fbs | 10 +- src/msg/fbuf/p2pmsg_content_generated.h | 124 ++++++------- src/msg/fbuf/p2pmsg_helpers.cpp | 32 ++-- src/msg/fbuf/p2pmsg_helpers.hpp | 8 +- src/p2p/p2p.hpp | 6 +- src/util.cpp | 27 ++- src/util.hpp | 4 +- 13 files changed, 345 insertions(+), 195 deletions(-) diff --git a/src/ledger.cpp b/src/ledger.cpp index c1bb2754..cd9ecd85 100644 --- a/src/ledger.cpp +++ b/src/ledger.cpp @@ -25,42 +25,68 @@ namespace ledger */ int init() { - // Get all records at lcl history direcory and find the last closed ledger. - for (const auto &entry : util::fetch_dir_entries(conf::ctx.hist_dir)) + // Filename list of the history folder. + std::list sorted_folder_entries = util::fetch_dir_entries(conf::ctx.hist_dir); + // Sorting to make filenames in seq_no order. + if (sort_lcl_filenames_and_validate(sorted_folder_entries) == -1) { - const std::string file_path = conf::ctx.hist_dir + "/" + entry.d_name; + return -1; + } + + std::string previous_block_lcl; + uint64_t previous_block_seq_no; + // Get all records at lcl history direcory and find the last closed ledger. + for (const auto &entry : sorted_folder_entries) + { + const std::string file_path = conf::ctx.hist_dir + "/" + entry; if (util::is_dir_exists(file_path)) { - LOG_ERROR << "Found directory " << entry.d_name << " in " << conf::ctx.hist_dir << ". There should be no folders in this directory."; + LOG_ERROR << "Found directory " << entry << " in " << conf::ctx.hist_dir << ". There should be no folders in this directory."; return -1; } else { - const std::string_view extension = util::fetch_file_extension(file_path); - const std::string file_name(util::remove_file_extension(entry.d_name)); - - if (extension != ".lcl") - { - LOG_ERROR << "Found invalid file extension: " << extension << " for lcl file " << entry.d_name << " in " << conf::ctx.hist_dir; - return -1; - } + const std::string file_name(util::remove_file_extension(entry)); const size_t pos = file_name.find("-"); if (pos != std::string::npos) { + uint64_t seq_no; + if (util::stoull(file_name.substr(0, pos), seq_no) == -1) + { + LOG_ERROR << "Found invalid sequence number in lcl file " << entry << " in " << conf::ctx.hist_dir; + return -1; + } std::vector buffer; if (read_ledger(file_path, buffer) == -1) return -1; - if (!msg::fbuf::ledger::verify_ledger_buffer(buffer.data(), buffer.size())) + if (!msg::fbuf::ledger::verify_ledger_block_buffer(buffer.data(), buffer.size())) { LOG_ERROR << "Ledger data verification failed. " << file_name; return -1; } + if (!check_block_integrity(file_name, buffer)) + { + LOG_ERROR << "Ledger block integrity check failed. " << file_name; + return -1; + } + + // Ledger integrity check. + if (!previous_block_lcl.empty()) + { + const p2p::proposal proposal = msg::fbuf::ledger::create_proposal_from_ledger_block(buffer); + if ((seq_no - previous_block_seq_no != 1) && (previous_block_lcl != proposal.lcl)) + { + LOG_ERROR << "Ledger block chain-link verification failed. " << file_name; + return -1; + } + } + previous_block_lcl = file_name; + previous_block_seq_no = seq_no; - const uint64_t seq_no = std::stoull(file_name.substr(0, pos)); ctx.cache.emplace(seq_no, std::move(file_name)); // cache -> [seq_no - hash] } else @@ -254,10 +280,9 @@ namespace ledger const size_t pos = proposal.lcl.find("-"); uint64_t seq_no = 0; - if (pos != std::string::npos) + if (pos != std::string::npos && util::stoull(proposal.lcl.substr(0, pos), seq_no) != -1) { - seq_no = std::stoull(proposal.lcl.substr(0, pos)); // Get lcl sequence number. - seq_no++; // New lcl sequence number. + seq_no++; // New lcl sequence number. } else { @@ -266,9 +291,9 @@ namespace ledger return -1; } - // Serialize lcl using flatbuffer ledger schema. + // Serialize lcl using flatbuffer ledger block schema. flatbuffers::FlatBufferBuilder builder(1024); - msg::fbuf::ledger::create_ledger_from_proposal(builder, proposal, seq_no); + msg::fbuf::ledger::create_ledger_block_from_proposal(builder, proposal, seq_no); // Get binary hash of the serialized lcl. std::string_view ledger_str_buf = msg::fbuf::flatbuff_bytes_to_sv(builder.GetBufferPointer(), builder.GetSize()); @@ -447,7 +472,12 @@ namespace ledger // Get sequence number of required lcl if (pos != std::string::npos) { - req_seq_no = std::stoull(required_lcl.substr(0, pos)); // Get required lcl sequence number + // Get required lcl sequence number + if (util::stoull(required_lcl.substr(0, pos), req_seq_no) == -1) + { + LOG_ERROR << "Retrieving seq_no from required_lcl failed"; + return -1; + } } if (req_seq_no > 0) @@ -494,7 +524,11 @@ namespace ledger // We put the requester's own lcl back in the response so they can validate the liveliness of the response. history_response.requester_lcl = hr.minimum_lcl; - uint64_t min_seq_no = std::stoull(hr.minimum_lcl.substr(0, pos)); // Get required lcl sequence number + uint64_t min_seq_no; + if (util::stoull(hr.minimum_lcl.substr(0, pos), min_seq_no) == -1) // Get required lcl sequence number. + { + LOG_ERROR << "lcl serve: Retrieving minimum ledger sequence no failed. Requested: " << hr.minimum_lcl; + } const auto itr = ctx.cache.find(min_seq_no); if (itr != ctx.cache.end()) // Requested minimum lcl is not in our lcl history cache @@ -533,18 +567,18 @@ namespace ledger //Get raw content of lcls that going to be send. for (const auto &[seq_no, lcl] : led_cache) { - p2p::history_ledger ledger; - ledger.lcl = lcl; + p2p::history_ledger_block ledger_block; + ledger_block.lcl = lcl; // Read lcl file. const std::string file_path = conf::ctx.hist_dir + "/" + lcl + ".lcl"; - if (read_ledger(file_path, ledger.raw_ledger) == -1) + if (read_ledger(file_path, ledger_block.block_buffer) == -1) { LOG_DEBUG << "lcl serve: Error when reading ledger file."; return -1; } - history_response.hist_ledgers.emplace(seq_no, std::move(ledger)); + history_response.hist_ledger_blocks.emplace(seq_no, std::move(ledger_block)); } return 0; @@ -575,7 +609,7 @@ namespace ledger { // Check whether recieved lcl history contains the current lcl node required. bool contains_requested_lcl = false; - for (auto &[seq_no, ledger] : hr.hist_ledgers) + for (auto &[seq_no, ledger] : hr.hist_ledger_blocks) { if (sync_ctx.target_lcl == ledger.lcl) { @@ -593,44 +627,69 @@ namespace ledger // Check integrity of recieved lcl list. // By checking recieved lcl hashes matches lcl content by applying hashing for each raw content. // TODO: Also verify chain hashes. - for (auto &[seq_no, ledger] : hr.hist_ledgers) + std::string previous_history_block_lcl; + uint64_t previous_history_block_seq_no; + for (auto &[seq_no, ledger] : hr.hist_ledger_blocks) { - const size_t pos = ledger.lcl.find("-"); - const std::string rec_lcl_hash = ledger.lcl.substr((pos + 1), (ledger.lcl.size() - 1)); - - // Get binary hash of the serialized lcl. - const std::string lcl = crypto::get_hash(ledger.raw_ledger.data(), ledger.raw_ledger.size()); - - // Get hex from binary hash - std::string lcl_hash; - - util::bin2hex(lcl_hash, - reinterpret_cast(lcl.data()), - lcl.size()); - - // recieved lcl hash and hash generated from recieved lcl content doesn't match -> abandon applying it - if (lcl_hash != rec_lcl_hash) + // Individually check each ledger entry's integrity before the chain check. + if (!check_block_integrity(ledger.lcl, ledger.block_buffer)) { LOG_DEBUG << "lcl sync: Peer sent us a history response but the ledger data does not match the hashes."; // todo: we should penalize peer who sent this. return -1; } + + // Ledger chain integrity check. + if (!previous_history_block_lcl.empty()) + { + const p2p::proposal proposal = msg::fbuf::ledger::create_proposal_from_ledger_block(ledger.block_buffer); + if ((seq_no - previous_history_block_seq_no != 1) && (previous_history_block_lcl != proposal.lcl)) + { + LOG_ERROR << "Ledger block chain-link verification failed. " << ledger.lcl; + return -1; + } + } + previous_history_block_lcl = ledger.lcl; + previous_history_block_seq_no = seq_no; + } + } + + // Performing ledger history joining check. + if (!ctx.cache.empty()) + { + const auto history_itr = hr.hist_ledger_blocks.begin(); + const p2p::proposal history_first_proposal = msg::fbuf::ledger::create_proposal_from_ledger_block(history_itr->second.block_buffer); + + // Removing ledger blocks upto the received histroy response starting point. + const auto reverse_history_ptr = std::make_reverse_iterator(ctx.cache.find(history_itr->first)); + if (reverse_history_ptr != ctx.cache.rend()) + { + // If cache ledger and history ledger are overlapping, remove blocks from end until the + // cache end at the state where history ledger can be straightly joined. + auto it = ctx.cache.rbegin(); + while (it != reverse_history_ptr) + { + remove_ledger(it->second); + // Erase function advance the iteratior. + ctx.cache.erase((--it).base()); + } + + auto &[cache_seq_no, cache_lcl] = get_ledger_cache_top(); + + // Comparing the sequence number and the lcl to validate the joining point. + if ((history_itr->first - cache_seq_no != 1) && (history_first_proposal.lcl != cache_lcl)) + { + LOG_ERROR << "lcl sync: Ledger integrity check at history joining point failed"; + return -1; + } } } // Execution to here means the history data sent checks out. // Save recieved lcl in file system and update lcl history cache. - // TODO: Verify chain hashes at the point of joining with our existing history. - for (auto &[seq_no, ledger] : hr.hist_ledgers) + for (auto &[seq_no, ledger] : hr.hist_ledger_blocks) { - auto prev_dup_itr = ctx.cache.find(seq_no); - if (prev_dup_itr != ctx.cache.end()) - { - remove_ledger(prev_dup_itr->second); - ctx.cache.erase(prev_dup_itr); - } - - write_ledger(ledger.lcl, ledger.raw_ledger.data(), ledger.raw_ledger.size()); + write_ledger(ledger.lcl, ledger.block_buffer.data(), ledger.block_buffer.size()); ctx.cache.emplace(seq_no, ledger.lcl); } @@ -640,5 +699,69 @@ namespace ledger new_lcl = lcl; return 0; } + /** + * Check the integrity of the given ledger. + * @param lcl supplied lcl of the ledger. + * @param raw_ledger ledger. + * @return true if the integrity check passes and false otherwise. + */ + bool check_block_integrity(std::string_view lcl, const std::vector &block_buffer) + { + const size_t pos = lcl.find("-"); + std::string_view supplied_lcl_hash = lcl.substr((pos + 1), (lcl.size() - 1)); + + // Get binary hash of the serialized lcl. + const std::string binary_lcl_hash = crypto::get_hash(block_buffer.data(), block_buffer.size()); + + // Get hex from binary hash. + std::string lcl_hash; + + util::bin2hex(lcl_hash, + reinterpret_cast(binary_lcl_hash.data()), + binary_lcl_hash.size()); + + return lcl_hash == supplied_lcl_hash; + } + + /** + * Sorting given lcl filename list in sequence number order and validate filenames. + * @param list List of lcl filenames. + * @return 0 if success and -1 on error. + */ + int sort_lcl_filenames_and_validate(std::list &list) + { + try + { + list.sort([](std::string &a, std::string &b) { + uint64_t seq_no_a; + uint64_t seq_no_b; + if (util::stoull(a.substr(0, a.find("-")), seq_no_a) == -1) + { + throw "Lcl file parsing error in file " + a + " in " + conf::ctx.hist_dir; + } + if (util::stoull(b.substr(0, b.find("-")), seq_no_b) == -1) + { + throw "Lcl file parsing error in file " + b + " in " + conf::ctx.hist_dir; + } + const std::string_view extension_a = util::fetch_file_extension(conf::ctx.hist_dir + a); + if (extension_a != ".lcl") + { + throw "Found invalid file extension: " + std::string(extension_a) + " for lcl file " + a + " in " + conf::ctx.hist_dir; + } + const std::string_view extension_b = util::fetch_file_extension(conf::ctx.hist_dir + b); + if (extension_b != ".lcl") + { + throw "Found invalid file extension: " + std::string(extension_b) + " for lcl file " + b + " in " + conf::ctx.hist_dir; + } + return seq_no_a < seq_no_b; + }); + return 0; + } + catch (std::string message) + { + LOG_ERROR << message; + return -1; + } + } } // namespace ledger \ No newline at end of file diff --git a/src/ledger.hpp b/src/ledger.hpp index 8b5888c2..3b278293 100644 --- a/src/ledger.hpp +++ b/src/ledger.hpp @@ -93,6 +93,10 @@ namespace ledger int handle_ledger_history_response(const p2p::history_response &hr, std::string &new_lcl); + bool check_block_integrity(std::string_view lcl, const std::vector &block_buffer); + + int sort_lcl_filenames_and_validate(std::list &list); + } // namespace ledger #endif \ No newline at end of file diff --git a/src/msg/fbuf/ledger_helpers.cpp b/src/msg/fbuf/ledger_helpers.cpp index c82bbad6..afba4a80 100644 --- a/src/msg/fbuf/ledger_helpers.cpp +++ b/src/msg/fbuf/ledger_helpers.cpp @@ -8,13 +8,13 @@ namespace msg::fbuf::ledger { /** - * Create ledger from the given proposal struct. - * @param p The proposal struct to be placed in ledger. - */ - void create_ledger_from_proposal(flatbuffers::FlatBufferBuilder &builder, const p2p::proposal &p, const uint64_t seq_no) + * Create ledger block from the given proposal struct. + * @param p The proposal struct to be placed in ledger. + */ + void create_ledger_block_from_proposal(flatbuffers::FlatBufferBuilder &builder, const p2p::proposal &p, const uint64_t seq_no) { - flatbuffers::Offset ledger = - ledger::CreateLedger( + flatbuffers::Offset ledger = + ledger::CreateLedgerBlock( builder, seq_no, p.time, @@ -27,9 +27,9 @@ namespace msg::fbuf::ledger builder.Finish(ledger); // Finished building message content to get serialised content. } - p2p::proposal create_proposal_from_ledger(const std::vector &ledger_buf) + p2p::proposal create_proposal_from_ledger_block(const std::vector &ledger_buf) { - auto ledger = msg::fbuf::ledger::GetLedger(ledger_buf.data()); + auto ledger = msg::fbuf::ledger::GetLedgerBlock(ledger_buf.data()); p2p::proposal p; p.lcl = flatbuff_bytes_to_sv(ledger->lcl()); p.state = flatbuff_bytes_to_hash(ledger->state()); @@ -37,10 +37,10 @@ namespace msg::fbuf::ledger return p; } - bool verify_ledger_buffer(const uint8_t *ledger_buf_ptr, const size_t buf_len) + bool verify_ledger_block_buffer(const uint8_t *ledger_buf_ptr, const size_t buf_len) { flatbuffers::Verifier ledger_verifier(ledger_buf_ptr, buf_len); - return VerifyLedgerBuffer(ledger_verifier); + return VerifyLedgerBlockBuffer(ledger_verifier); } } // namespace msg::fbuf::ledger diff --git a/src/msg/fbuf/ledger_helpers.hpp b/src/msg/fbuf/ledger_helpers.hpp index 3e2a62e2..f1d80b7e 100644 --- a/src/msg/fbuf/ledger_helpers.hpp +++ b/src/msg/fbuf/ledger_helpers.hpp @@ -8,11 +8,11 @@ namespace msg::fbuf::ledger { - void create_ledger_from_proposal(flatbuffers::FlatBufferBuilder &builder, const p2p::proposal &p, const uint64_t seq_no); + void create_ledger_block_from_proposal(flatbuffers::FlatBufferBuilder &builder, const p2p::proposal &p, const uint64_t seq_no); - p2p::proposal create_proposal_from_ledger(const std::vector &ledger_buf); + p2p::proposal create_proposal_from_ledger_block(const std::vector &ledger_buf); - bool verify_ledger_buffer(const uint8_t *ledger_buf_ptr, const size_t buf_len); + bool verify_ledger_block_buffer(const uint8_t *ledger_buf_ptr, const size_t buf_len); } // namespace msg::fbuf::ledger diff --git a/src/msg/fbuf/ledger_schema.fbs b/src/msg/fbuf/ledger_schema.fbs index 9f84e25d..a1c731ee 100644 --- a/src/msg/fbuf/ledger_schema.fbs +++ b/src/msg/fbuf/ledger_schema.fbs @@ -2,7 +2,7 @@ include "common_schema.fbs"; namespace msg.fbuf.ledger; -table Ledger { +table LedgerBlock { seq_no:uint64; time:uint64; lcl:[ubyte]; @@ -17,4 +17,4 @@ table RawInputList { inputs:[ByteArray]; } -root_type Ledger; \ No newline at end of file +root_type LedgerBlock; \ No newline at end of file diff --git a/src/msg/fbuf/ledger_schema_generated.h b/src/msg/fbuf/ledger_schema_generated.h index 96c3bfc2..012df7f7 100644 --- a/src/msg/fbuf/ledger_schema_generated.h +++ b/src/msg/fbuf/ledger_schema_generated.h @@ -12,14 +12,14 @@ namespace msg { namespace fbuf { namespace ledger { -struct Ledger; -struct LedgerBuilder; +struct LedgerBlock; +struct LedgerBlockBuilder; struct RawInputList; struct RawInputListBuilder; -struct Ledger FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { - typedef LedgerBuilder Builder; +struct LedgerBlock FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { + typedef LedgerBlockBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_SEQ_NO = 4, VT_TIME = 6, @@ -92,43 +92,43 @@ struct Ledger FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { } }; -struct LedgerBuilder { - typedef Ledger Table; +struct LedgerBlockBuilder { + typedef LedgerBlock Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_seq_no(uint64_t seq_no) { - fbb_.AddElement(Ledger::VT_SEQ_NO, seq_no, 0); + fbb_.AddElement(LedgerBlock::VT_SEQ_NO, seq_no, 0); } void add_time(uint64_t time) { - fbb_.AddElement(Ledger::VT_TIME, time, 0); + fbb_.AddElement(LedgerBlock::VT_TIME, time, 0); } void add_lcl(flatbuffers::Offset> lcl) { - fbb_.AddOffset(Ledger::VT_LCL, lcl); + fbb_.AddOffset(LedgerBlock::VT_LCL, lcl); } void add_state(flatbuffers::Offset> state) { - fbb_.AddOffset(Ledger::VT_STATE, state); + fbb_.AddOffset(LedgerBlock::VT_STATE, state); } void add_users(flatbuffers::Offset>> users) { - fbb_.AddOffset(Ledger::VT_USERS, users); + fbb_.AddOffset(LedgerBlock::VT_USERS, users); } void add_inputs(flatbuffers::Offset>> inputs) { - fbb_.AddOffset(Ledger::VT_INPUTS, inputs); + fbb_.AddOffset(LedgerBlock::VT_INPUTS, inputs); } void add_outputs(flatbuffers::Offset>> outputs) { - fbb_.AddOffset(Ledger::VT_OUTPUTS, outputs); + fbb_.AddOffset(LedgerBlock::VT_OUTPUTS, outputs); } - explicit LedgerBuilder(flatbuffers::FlatBufferBuilder &_fbb) + explicit LedgerBlockBuilder(flatbuffers::FlatBufferBuilder &_fbb) : fbb_(_fbb) { start_ = fbb_.StartTable(); } - flatbuffers::Offset Finish() { + flatbuffers::Offset Finish() { const auto end = fbb_.EndTable(start_); - auto o = flatbuffers::Offset(end); + auto o = flatbuffers::Offset(end); return o; } }; -inline flatbuffers::Offset CreateLedger( +inline flatbuffers::Offset CreateLedgerBlock( flatbuffers::FlatBufferBuilder &_fbb, uint64_t seq_no = 0, uint64_t time = 0, @@ -137,7 +137,7 @@ inline flatbuffers::Offset CreateLedger( flatbuffers::Offset>> users = 0, flatbuffers::Offset>> inputs = 0, flatbuffers::Offset>> outputs = 0) { - LedgerBuilder builder_(_fbb); + LedgerBlockBuilder builder_(_fbb); builder_.add_time(time); builder_.add_seq_no(seq_no); builder_.add_outputs(outputs); @@ -148,7 +148,7 @@ inline flatbuffers::Offset CreateLedger( return builder_.Finish(); } -inline flatbuffers::Offset CreateLedgerDirect( +inline flatbuffers::Offset CreateLedgerBlockDirect( flatbuffers::FlatBufferBuilder &_fbb, uint64_t seq_no = 0, uint64_t time = 0, @@ -162,7 +162,7 @@ inline flatbuffers::Offset CreateLedgerDirect( auto users__ = users ? _fbb.CreateVector>(*users) : 0; auto inputs__ = inputs ? _fbb.CreateVector>(*inputs) : 0; auto outputs__ = outputs ? _fbb.CreateVector>(*outputs) : 0; - return msg::fbuf::ledger::CreateLedger( + return msg::fbuf::ledger::CreateLedgerBlock( _fbb, seq_no, time, @@ -245,37 +245,37 @@ inline flatbuffers::Offset CreateRawInputListDirect( inputs__); } -inline const msg::fbuf::ledger::Ledger *GetLedger(const void *buf) { - return flatbuffers::GetRoot(buf); +inline const msg::fbuf::ledger::LedgerBlock *GetLedgerBlock(const void *buf) { + return flatbuffers::GetRoot(buf); } -inline const msg::fbuf::ledger::Ledger *GetSizePrefixedLedger(const void *buf) { - return flatbuffers::GetSizePrefixedRoot(buf); +inline const msg::fbuf::ledger::LedgerBlock *GetSizePrefixedLedgerBlock(const void *buf) { + return flatbuffers::GetSizePrefixedRoot(buf); } -inline Ledger *GetMutableLedger(void *buf) { - return flatbuffers::GetMutableRoot(buf); +inline LedgerBlock *GetMutableLedgerBlock(void *buf) { + return flatbuffers::GetMutableRoot(buf); } -inline bool VerifyLedgerBuffer( +inline bool VerifyLedgerBlockBuffer( flatbuffers::Verifier &verifier) { - return verifier.VerifyBuffer(nullptr); + return verifier.VerifyBuffer(nullptr); } -inline bool VerifySizePrefixedLedgerBuffer( +inline bool VerifySizePrefixedLedgerBlockBuffer( flatbuffers::Verifier &verifier) { - return verifier.VerifySizePrefixedBuffer(nullptr); + return verifier.VerifySizePrefixedBuffer(nullptr); } -inline void FinishLedgerBuffer( +inline void FinishLedgerBlockBuffer( flatbuffers::FlatBufferBuilder &fbb, - flatbuffers::Offset root) { + flatbuffers::Offset root) { fbb.Finish(root); } -inline void FinishSizePrefixedLedgerBuffer( +inline void FinishSizePrefixedLedgerBlockBuffer( flatbuffers::FlatBufferBuilder &fbb, - flatbuffers::Offset root) { + flatbuffers::Offset root) { fbb.FinishSizePrefixed(root); } diff --git a/src/msg/fbuf/p2pmsg_content.fbs b/src/msg/fbuf/p2pmsg_content.fbs index 90a37fa7..b0916d52 100644 --- a/src/msg/fbuf/p2pmsg_content.fbs +++ b/src/msg/fbuf/p2pmsg_content.fbs @@ -72,18 +72,18 @@ enum Ledger_Response_Error : ubyte table History_Response_Message { //Ledger History request type message schema requester_lcl:[ubyte]; - hist_ledgers:[HistoryLedgerPair]; + hist_ledger_blocks:[HistoryLedgerBlockPair]; error: Ledger_Response_Error; } -table HistoryLedgerPair { //A key, value pair of byte[]. +table HistoryLedgerBlockPair { //A key, value pair of byte[]. seq_no:uint64; - ledger:HistoryLedger; + ledger:HistoryLedgerBlock; } -table HistoryLedger { +table HistoryLedgerBlock { lcl:[ubyte]; - raw_ledger:[ubyte]; + block_buffer:[ubyte]; } table State_Request_Message { //State request message schema diff --git a/src/msg/fbuf/p2pmsg_content_generated.h b/src/msg/fbuf/p2pmsg_content_generated.h index 2511e1a2..8085380c 100644 --- a/src/msg/fbuf/p2pmsg_content_generated.h +++ b/src/msg/fbuf/p2pmsg_content_generated.h @@ -42,11 +42,11 @@ struct History_Request_MessageBuilder; struct History_Response_Message; struct History_Response_MessageBuilder; -struct HistoryLedgerPair; -struct HistoryLedgerPairBuilder; +struct HistoryLedgerBlockPair; +struct HistoryLedgerBlockPairBuilder; -struct HistoryLedger; -struct HistoryLedgerBuilder; +struct HistoryLedgerBlock; +struct HistoryLedgerBlockBuilder; struct State_Request_Message; struct State_Request_MessageBuilder; @@ -992,7 +992,7 @@ struct History_Response_Message FLATBUFFERS_FINAL_CLASS : private flatbuffers::T typedef History_Response_MessageBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_REQUESTER_LCL = 4, - VT_HIST_LEDGERS = 6, + VT_HIST_LEDGER_BLOCKS = 6, VT_ERROR = 8 }; const flatbuffers::Vector *requester_lcl() const { @@ -1001,11 +1001,11 @@ struct History_Response_Message FLATBUFFERS_FINAL_CLASS : private flatbuffers::T flatbuffers::Vector *mutable_requester_lcl() { return GetPointer *>(VT_REQUESTER_LCL); } - const flatbuffers::Vector> *hist_ledgers() const { - return GetPointer> *>(VT_HIST_LEDGERS); + const flatbuffers::Vector> *hist_ledger_blocks() const { + return GetPointer> *>(VT_HIST_LEDGER_BLOCKS); } - flatbuffers::Vector> *mutable_hist_ledgers() { - return GetPointer> *>(VT_HIST_LEDGERS); + flatbuffers::Vector> *mutable_hist_ledger_blocks() { + return GetPointer> *>(VT_HIST_LEDGER_BLOCKS); } msg::fbuf::p2pmsg::Ledger_Response_Error error() const { return static_cast(GetField(VT_ERROR, 0)); @@ -1017,9 +1017,9 @@ struct History_Response_Message FLATBUFFERS_FINAL_CLASS : private flatbuffers::T return VerifyTableStart(verifier) && VerifyOffset(verifier, VT_REQUESTER_LCL) && verifier.VerifyVector(requester_lcl()) && - VerifyOffset(verifier, VT_HIST_LEDGERS) && - verifier.VerifyVector(hist_ledgers()) && - verifier.VerifyVectorOfTables(hist_ledgers()) && + VerifyOffset(verifier, VT_HIST_LEDGER_BLOCKS) && + verifier.VerifyVector(hist_ledger_blocks()) && + verifier.VerifyVectorOfTables(hist_ledger_blocks()) && VerifyField(verifier, VT_ERROR) && verifier.EndTable(); } @@ -1032,8 +1032,8 @@ struct History_Response_MessageBuilder { void add_requester_lcl(flatbuffers::Offset> requester_lcl) { fbb_.AddOffset(History_Response_Message::VT_REQUESTER_LCL, requester_lcl); } - void add_hist_ledgers(flatbuffers::Offset>> hist_ledgers) { - fbb_.AddOffset(History_Response_Message::VT_HIST_LEDGERS, hist_ledgers); + void add_hist_ledger_blocks(flatbuffers::Offset>> hist_ledger_blocks) { + fbb_.AddOffset(History_Response_Message::VT_HIST_LEDGER_BLOCKS, hist_ledger_blocks); } void add_error(msg::fbuf::p2pmsg::Ledger_Response_Error error) { fbb_.AddElement(History_Response_Message::VT_ERROR, static_cast(error), 0); @@ -1052,10 +1052,10 @@ struct History_Response_MessageBuilder { inline flatbuffers::Offset CreateHistory_Response_Message( flatbuffers::FlatBufferBuilder &_fbb, flatbuffers::Offset> requester_lcl = 0, - flatbuffers::Offset>> hist_ledgers = 0, + flatbuffers::Offset>> hist_ledger_blocks = 0, msg::fbuf::p2pmsg::Ledger_Response_Error error = msg::fbuf::p2pmsg::Ledger_Response_Error_None) { History_Response_MessageBuilder builder_(_fbb); - builder_.add_hist_ledgers(hist_ledgers); + builder_.add_hist_ledger_blocks(hist_ledger_blocks); builder_.add_requester_lcl(requester_lcl); builder_.add_error(error); return builder_.Finish(); @@ -1064,19 +1064,19 @@ inline flatbuffers::Offset CreateHistory_Response_Mess inline flatbuffers::Offset CreateHistory_Response_MessageDirect( flatbuffers::FlatBufferBuilder &_fbb, const std::vector *requester_lcl = nullptr, - const std::vector> *hist_ledgers = nullptr, + const std::vector> *hist_ledger_blocks = nullptr, msg::fbuf::p2pmsg::Ledger_Response_Error error = msg::fbuf::p2pmsg::Ledger_Response_Error_None) { auto requester_lcl__ = requester_lcl ? _fbb.CreateVector(*requester_lcl) : 0; - auto hist_ledgers__ = hist_ledgers ? _fbb.CreateVector>(*hist_ledgers) : 0; + auto hist_ledger_blocks__ = hist_ledger_blocks ? _fbb.CreateVector>(*hist_ledger_blocks) : 0; return msg::fbuf::p2pmsg::CreateHistory_Response_Message( _fbb, requester_lcl__, - hist_ledgers__, + hist_ledger_blocks__, error); } -struct HistoryLedgerPair FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { - typedef HistoryLedgerPairBuilder Builder; +struct HistoryLedgerBlockPair FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { + typedef HistoryLedgerBlockPairBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_SEQ_NO = 4, VT_LEDGER = 6 @@ -1087,11 +1087,11 @@ struct HistoryLedgerPair FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { bool mutate_seq_no(uint64_t _seq_no) { return SetField(VT_SEQ_NO, _seq_no, 0); } - const msg::fbuf::p2pmsg::HistoryLedger *ledger() const { - return GetPointer(VT_LEDGER); + const msg::fbuf::p2pmsg::HistoryLedgerBlock *ledger() const { + return GetPointer(VT_LEDGER); } - msg::fbuf::p2pmsg::HistoryLedger *mutable_ledger() { - return GetPointer(VT_LEDGER); + msg::fbuf::p2pmsg::HistoryLedgerBlock *mutable_ledger() { + return GetPointer(VT_LEDGER); } bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && @@ -1102,42 +1102,42 @@ struct HistoryLedgerPair FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { } }; -struct HistoryLedgerPairBuilder { - typedef HistoryLedgerPair Table; +struct HistoryLedgerBlockPairBuilder { + typedef HistoryLedgerBlockPair Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_seq_no(uint64_t seq_no) { - fbb_.AddElement(HistoryLedgerPair::VT_SEQ_NO, seq_no, 0); + fbb_.AddElement(HistoryLedgerBlockPair::VT_SEQ_NO, seq_no, 0); } - void add_ledger(flatbuffers::Offset ledger) { - fbb_.AddOffset(HistoryLedgerPair::VT_LEDGER, ledger); + void add_ledger(flatbuffers::Offset ledger) { + fbb_.AddOffset(HistoryLedgerBlockPair::VT_LEDGER, ledger); } - explicit HistoryLedgerPairBuilder(flatbuffers::FlatBufferBuilder &_fbb) + explicit HistoryLedgerBlockPairBuilder(flatbuffers::FlatBufferBuilder &_fbb) : fbb_(_fbb) { start_ = fbb_.StartTable(); } - flatbuffers::Offset Finish() { + flatbuffers::Offset Finish() { const auto end = fbb_.EndTable(start_); - auto o = flatbuffers::Offset(end); + auto o = flatbuffers::Offset(end); return o; } }; -inline flatbuffers::Offset CreateHistoryLedgerPair( +inline flatbuffers::Offset CreateHistoryLedgerBlockPair( flatbuffers::FlatBufferBuilder &_fbb, uint64_t seq_no = 0, - flatbuffers::Offset ledger = 0) { - HistoryLedgerPairBuilder builder_(_fbb); + flatbuffers::Offset ledger = 0) { + HistoryLedgerBlockPairBuilder builder_(_fbb); builder_.add_seq_no(seq_no); builder_.add_ledger(ledger); return builder_.Finish(); } -struct HistoryLedger FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { - typedef HistoryLedgerBuilder Builder; +struct HistoryLedgerBlock FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { + typedef HistoryLedgerBlockBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { VT_LCL = 4, - VT_RAW_LEDGER = 6 + VT_BLOCK_BUFFER = 6 }; const flatbuffers::Vector *lcl() const { return GetPointer *>(VT_LCL); @@ -1145,63 +1145,63 @@ struct HistoryLedger FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { flatbuffers::Vector *mutable_lcl() { return GetPointer *>(VT_LCL); } - const flatbuffers::Vector *raw_ledger() const { - return GetPointer *>(VT_RAW_LEDGER); + const flatbuffers::Vector *block_buffer() const { + return GetPointer *>(VT_BLOCK_BUFFER); } - flatbuffers::Vector *mutable_raw_ledger() { - return GetPointer *>(VT_RAW_LEDGER); + flatbuffers::Vector *mutable_block_buffer() { + return GetPointer *>(VT_BLOCK_BUFFER); } bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && VerifyOffset(verifier, VT_LCL) && verifier.VerifyVector(lcl()) && - VerifyOffset(verifier, VT_RAW_LEDGER) && - verifier.VerifyVector(raw_ledger()) && + VerifyOffset(verifier, VT_BLOCK_BUFFER) && + verifier.VerifyVector(block_buffer()) && verifier.EndTable(); } }; -struct HistoryLedgerBuilder { - typedef HistoryLedger Table; +struct HistoryLedgerBlockBuilder { + typedef HistoryLedgerBlock Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; void add_lcl(flatbuffers::Offset> lcl) { - fbb_.AddOffset(HistoryLedger::VT_LCL, lcl); + fbb_.AddOffset(HistoryLedgerBlock::VT_LCL, lcl); } - void add_raw_ledger(flatbuffers::Offset> raw_ledger) { - fbb_.AddOffset(HistoryLedger::VT_RAW_LEDGER, raw_ledger); + void add_block_buffer(flatbuffers::Offset> block_buffer) { + fbb_.AddOffset(HistoryLedgerBlock::VT_BLOCK_BUFFER, block_buffer); } - explicit HistoryLedgerBuilder(flatbuffers::FlatBufferBuilder &_fbb) + explicit HistoryLedgerBlockBuilder(flatbuffers::FlatBufferBuilder &_fbb) : fbb_(_fbb) { start_ = fbb_.StartTable(); } - flatbuffers::Offset Finish() { + flatbuffers::Offset Finish() { const auto end = fbb_.EndTable(start_); - auto o = flatbuffers::Offset(end); + auto o = flatbuffers::Offset(end); return o; } }; -inline flatbuffers::Offset CreateHistoryLedger( +inline flatbuffers::Offset CreateHistoryLedgerBlock( flatbuffers::FlatBufferBuilder &_fbb, flatbuffers::Offset> lcl = 0, - flatbuffers::Offset> raw_ledger = 0) { - HistoryLedgerBuilder builder_(_fbb); - builder_.add_raw_ledger(raw_ledger); + flatbuffers::Offset> block_buffer = 0) { + HistoryLedgerBlockBuilder builder_(_fbb); + builder_.add_block_buffer(block_buffer); builder_.add_lcl(lcl); return builder_.Finish(); } -inline flatbuffers::Offset CreateHistoryLedgerDirect( +inline flatbuffers::Offset CreateHistoryLedgerBlockDirect( flatbuffers::FlatBufferBuilder &_fbb, const std::vector *lcl = nullptr, - const std::vector *raw_ledger = nullptr) { + const std::vector *block_buffer = nullptr) { auto lcl__ = lcl ? _fbb.CreateVector(*lcl) : 0; - auto raw_ledger__ = raw_ledger ? _fbb.CreateVector(*raw_ledger) : 0; - return msg::fbuf::p2pmsg::CreateHistoryLedger( + auto block_buffer__ = block_buffer ? _fbb.CreateVector(*block_buffer) : 0; + return msg::fbuf::p2pmsg::CreateHistoryLedgerBlock( _fbb, lcl__, - raw_ledger__); + block_buffer__); } struct State_Request_Message FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { diff --git a/src/msg/fbuf/p2pmsg_helpers.cpp b/src/msg/fbuf/p2pmsg_helpers.cpp index a2312297..15628e8b 100644 --- a/src/msg/fbuf/p2pmsg_helpers.cpp +++ b/src/msg/fbuf/p2pmsg_helpers.cpp @@ -203,8 +203,8 @@ namespace msg::fbuf::p2pmsg if (msg.requester_lcl()) hr.requester_lcl = flatbuff_bytes_to_sv(msg.requester_lcl()); - if (msg.hist_ledgers()) - hr.hist_ledgers = flatbuf_historyledgermap_to_historyledgermap(msg.hist_ledgers()); + if (msg.hist_ledger_blocks()) + hr.hist_ledger_blocks = flatbuf_historyledgermap_to_historyledgermap(msg.hist_ledger_blocks()); if (msg.error()) hr.error = (p2p::LEDGER_RESPONSE_ERROR)msg.error(); @@ -430,7 +430,7 @@ namespace msg::fbuf::p2pmsg CreateHistory_Response_Message( builder, sv_to_flatbuff_bytes(builder, hr.requester_lcl), - historyledgermap_to_flatbuf_historyledgermap(builder, hr.hist_ledgers), + historyledgermap_to_flatbuf_historyledgermap(builder, hr.hist_ledger_blocks), (Ledger_Response_Error)hr.error); flatbuffers::Offset message = CreateContent(builder, Message_History_Response_Message, hrmsg.Union()); @@ -688,39 +688,39 @@ namespace msg::fbuf::p2pmsg return builder.CreateVector(fbvec); } - const std::map - flatbuf_historyledgermap_to_historyledgermap(const flatbuffers::Vector> *fbvec) + const std::map + flatbuf_historyledgermap_to_historyledgermap(const flatbuffers::Vector> *fbvec) { - std::map map; + std::map map; - for (const HistoryLedgerPair *pair : *fbvec) + for (const HistoryLedgerBlockPair *pair : *fbvec) { std::list msglist; - p2p::history_ledger ledger; + p2p::history_ledger_block ledger; ledger.lcl = flatbuff_bytes_to_sv(pair->ledger()->lcl()); - auto raw = pair->ledger()->raw_ledger(); - ledger.raw_ledger = std::vector(raw->begin(), raw->end()); + auto raw = pair->ledger()->block_buffer(); + ledger.block_buffer = std::vector(raw->begin(), raw->end()); map.emplace(pair->seq_no(), std::move(ledger)); } return map; } - const flatbuffers::Offset>> - historyledgermap_to_flatbuf_historyledgermap(flatbuffers::FlatBufferBuilder &builder, const std::map &map) + const flatbuffers::Offset>> + historyledgermap_to_flatbuf_historyledgermap(flatbuffers::FlatBufferBuilder &builder, const std::map &map) { - std::vector> fbvec; + std::vector> fbvec; fbvec.reserve(map.size()); for (auto const &[seq_no, ledger] : map) { - flatbuffers::Offset history_ledger = CreateHistoryLedger( + flatbuffers::Offset history_ledger = CreateHistoryLedgerBlock( builder, sv_to_flatbuff_bytes(builder, ledger.lcl), - builder.CreateVector(ledger.raw_ledger)); + builder.CreateVector(ledger.block_buffer)); - fbvec.push_back(CreateHistoryLedgerPair( + fbvec.push_back(CreateHistoryLedgerBlockPair( builder, seq_no, history_ledger)); diff --git a/src/msg/fbuf/p2pmsg_helpers.hpp b/src/msg/fbuf/p2pmsg_helpers.hpp index aa439863..34b94ddd 100644 --- a/src/msg/fbuf/p2pmsg_helpers.hpp +++ b/src/msg/fbuf/p2pmsg_helpers.hpp @@ -78,11 +78,11 @@ namespace msg::fbuf::p2pmsg const flatbuffers::Offset>> user_input_map_to_flatbuf_user_input_group(flatbuffers::FlatBufferBuilder &builder, const std::unordered_map> &map); - const std::map - flatbuf_historyledgermap_to_historyledgermap(const flatbuffers::Vector> *fbvec); + const std::map + flatbuf_historyledgermap_to_historyledgermap(const flatbuffers::Vector> *fbvec); - const flatbuffers::Offset>> - historyledgermap_to_flatbuf_historyledgermap(flatbuffers::FlatBufferBuilder &builder, const std::map &map); + const flatbuffers::Offset>> + historyledgermap_to_flatbuf_historyledgermap(flatbuffers::FlatBufferBuilder &builder, const std::map &map); void flatbuf_statefshashentry_to_statefshashentry(std::unordered_map &fs_entries, const flatbuffers::Vector> *fhashes); diff --git a/src/p2p/p2p.hpp b/src/p2p/p2p.hpp index e8632c93..ea818111 100644 --- a/src/p2p/p2p.hpp +++ b/src/p2p/p2p.hpp @@ -40,10 +40,10 @@ namespace p2p std::string required_lcl; }; - struct history_ledger + struct history_ledger_block { std::string lcl; - std::vector raw_ledger; + std::vector block_buffer; }; struct peer_challenge_response @@ -63,7 +63,7 @@ namespace p2p struct history_response { std::string requester_lcl; - std::map hist_ledgers; + std::map hist_ledger_blocks; LEDGER_RESPONSE_ERROR error = LEDGER_RESPONSE_ERROR::NONE; }; diff --git a/src/util.cpp b/src/util.cpp index 207f75b1..309c37e5 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -301,9 +301,9 @@ namespace util * @param path Directory path. * @return Returns the list of entries inside the directory. */ - std::list fetch_dir_entries(std::string_view path) + std::list fetch_dir_entries(std::string_view path) { - std::list entries; + std::list entries; DIR *dr; // Open the directory stream. @@ -317,7 +317,7 @@ namespace util // or previous directory entry. if (std::strcmp(en->d_name, ".") != 0 && std::strcmp(en->d_name, "..") != 0) { - entries.push_back(*en); + entries.push_back(en->d_name); } } // Close directory stream. @@ -411,4 +411,25 @@ namespace util collection.push_back(std::string(str.substr(start))); } + /** + * Converts given string to a uint_64. A wrapper function for std::stoull. + * @param str String variable. + * @param result Variable to store the answer from the conversion. + * @return Returns 0 in a successful conversion and -1 on error. + */ + int stoull(const std::string &str, uint64_t &result) + { + try + { + result = std::stoull(str); + } + catch(const std::exception& e) + { + // Return -1 if any exceptions are captured. + return -1; + } + return 0; + + } + } // namespace util diff --git a/src/util.hpp b/src/util.hpp index 73fadda7..635a0fc6 100644 --- a/src/util.hpp +++ b/src/util.hpp @@ -99,7 +99,7 @@ namespace util int create_dir_tree_recursive(std::string_view path); - std::list fetch_dir_entries(std::string_view path); + std::list fetch_dir_entries(std::string_view path); std::string_view fetch_file_extension(std::string_view path); @@ -111,6 +111,8 @@ namespace util void split_string(std::vector &collection, std::string_view str, std::string_view delimeter); + int stoull(const std::string &str, uint64_t &result); + } // namespace util #endif