diff --git a/src/ledger.cpp b/src/ledger.cpp index cd9ecd85..098ed56c 100644 --- a/src/ledger.cpp +++ b/src/ledger.cpp @@ -13,8 +13,8 @@ namespace p2pmsg = msg::fbuf::p2pmsg; namespace ledger { constexpr int FILE_PERMS = 0644; - constexpr uint64_t MAX_LEDGER_SEQUENCE = 200; // Max ledger count. - constexpr uint16_t SYNCER_IDLE_WAIT = 20; // lcl syncer loop sleep time (milliseconds). + constexpr uint64_t MAX_LEDGER_SEQUENCE = 256; // Max ledger block count to keep. + constexpr uint16_t SYNCER_IDLE_WAIT = 20; // lcl syncer loop sleep time (milliseconds). ledger_context ctx; sync_context sync_ctx; @@ -49,16 +49,10 @@ namespace ledger { const std::string file_name(util::remove_file_extension(entry)); - const size_t pos = file_name.find("-"); - - if (pos != std::string::npos) + uint64_t seq_no; + std::string hash; + if (extract_lcl(file_name, seq_no, hash) != -1) { - uint64_t seq_no; - if (util::stoull(file_name.substr(0, pos), seq_no) == -1) - { - LOG_ERROR << "Found invalid sequence number in lcl file " << entry << " in " << conf::ctx.hist_dir; - return -1; - } std::vector buffer; if (read_ledger(file_path, buffer) == -1) return -1; @@ -68,7 +62,7 @@ namespace ledger LOG_ERROR << "Ledger data verification failed. " << file_name; return -1; } - if (!check_block_integrity(file_name, buffer)) + if (!check_block_integrity(hash, buffer)) { LOG_ERROR << "Ledger block integrity check failed. " << file_name; return -1; @@ -130,28 +124,39 @@ namespace ledger } } - void set_sync_target(std::string_view target_lcl) + void set_sync_target(const std::string &target_lcl) { if (sync_ctx.is_shutting_down) return; - const std::string lcl = ctx.get_lcl(); + // Validate target lcl format. + uint64_t target_seq_no; + std::string target_hash; + if (extract_lcl(target_lcl, target_seq_no, target_hash) == -1) + { + LOG_ERROR << "lcl sync: Invalid target lcl " << target_seq_no; + return; + } + + const std::string current_lcl = ctx.get_lcl(); { std::scoped_lock lock(sync_ctx.target_lcl_mutex); if (sync_ctx.target_lcl == target_lcl) return; sync_ctx.target_lcl = target_lcl; + sync_ctx.target_lcl_seq_no = target_seq_no; sync_ctx.is_syncing = true; - LOG_INFO << "lcl sync: Syncing for target:" << sync_ctx.target_lcl.substr(0, 15) << " (current:" << lcl.substr(0, 15) << ")"; + LOG_INFO << "lcl sync: Syncing for target:" << sync_ctx.target_lcl.substr(0, 15) << " (current:" << current_lcl.substr(0, 15) << ")"; } // Request history from a random peer if needed. - // If target is genesis ledger, we simply clear our ledger history without sending a - // history request. + // We do not send a request if the target is GENESIS block (nothing to request). if (target_lcl != GENESIS_LEDGER) - send_ledger_history_request(lcl, target_lcl); + { + send_ledger_history_request(current_lcl, target_lcl); + } } /** @@ -175,7 +180,7 @@ namespace ledger if (!prev_processed) util::sleep(SYNCER_IDLE_WAIT); - const std::string lcl = ctx.get_lcl(); + const std::string current_lcl = ctx.get_lcl(); // Move over the collected sync items to the local lists. { @@ -194,22 +199,33 @@ namespace ledger { if (sync_ctx.target_lcl == GENESIS_LEDGER) { + LOG_INFO << "lcl sync: Target is GENESIS. Clearing our history."; clear_ledger(); sync_ctx.target_lcl.clear(); + sync_ctx.target_lcl_seq_no = 0; sync_ctx.is_syncing = false; } + // Check the target lcl seq no. to see whether it's too far ahead. That means no one probably has our + // lcl in their ledgers. So we should clear our entire ledger history before requesting from peers. + else if (current_lcl != GENESIS_LEDGER && sync_ctx.target_lcl_seq_no > (ctx.get_seq_no() + MAX_LEDGER_SEQUENCE)) + { + LOG_INFO << "lcl sync: Target " << sync_ctx.target_lcl.substr(0, 15) << " is too far ahead. Clearing our history."; + clear_ledger(); + } else { + // Scan any queued lcl history responses. // Only process the first successful item which matches with our current lcl. for (const p2p::history_response &hr : history_responses) { - if (hr.requester_lcl == lcl) + if (hr.requester_lcl == current_lcl) { std::string new_lcl; if (handle_ledger_history_response(hr, new_lcl) != -1) { LOG_INFO << "lcl sync: Sync complete. New lcl:" << new_lcl.substr(0, 15); sync_ctx.target_lcl.clear(); + sync_ctx.target_lcl_seq_no = 0; sync_ctx.is_syncing = false; break; } @@ -277,20 +293,17 @@ namespace ledger */ int save_ledger(const p2p::proposal &proposal) { - const size_t pos = proposal.lcl.find("-"); uint64_t seq_no = 0; - - if (pos != std::string::npos && util::stoull(proposal.lcl.substr(0, pos), seq_no) != -1) + std::string hash; + if (extract_lcl(proposal.lcl, seq_no, hash) == -1) { - seq_no++; // New lcl sequence number. - } - else - { - // lcl records should follow [ledger sequnce numer]-lcl[lcl hex] format. + // lcl records should follow [ledger sequnce numer]-[lcl hex] format. LOG_ERROR << "Invalid lcl name: " << proposal.lcl << " when saving ledger."; return -1; } + seq_no++; // New lcl sequence number. + // Serialize lcl using flatbuffer ledger block schema. flatbuffers::FlatBufferBuilder builder(1024); msg::fbuf::ledger::create_ledger_block_from_proposal(builder, proposal, seq_no); @@ -441,14 +454,14 @@ namespace ledger /** * Create and send ledger history request to random node from unl list. - * @param minimum_lcl hash of the minimum lcl from which node need lcl history. - * @param required_lcl hash of the required lcl. + * @param current_lcl Current lcl. + * @param required_lcl Required lcl. */ - void send_ledger_history_request(std::string_view minimum_lcl, std::string_view required_lcl) + void send_ledger_history_request(std::string_view current_lcl, std::string_view required_lcl) { p2p::history_request hr; hr.required_lcl = required_lcl; - hr.minimum_lcl = minimum_lcl; + hr.requester_lcl = current_lcl; flatbuffers::FlatBufferBuilder fbuf(1024); p2pmsg::create_msg_from_history_request(fbuf, hr); @@ -466,18 +479,12 @@ namespace ledger */ bool check_required_lcl_availability(const std::string &required_lcl) { - size_t pos = required_lcl.find("-"); uint64_t req_seq_no = 0; - - // Get sequence number of required lcl - if (pos != std::string::npos) + std::string hash; + if (extract_lcl(required_lcl, req_seq_no, hash) == -1) { - // Get required lcl sequence number - if (util::stoull(required_lcl.substr(0, pos), req_seq_no) == -1) - { - LOG_ERROR << "Retrieving seq_no from required_lcl failed"; - return -1; - } + LOG_DEBUG << "Required lcl parse error " << required_lcl; + return -1; } if (req_seq_no > 0) @@ -485,14 +492,14 @@ namespace ledger const auto itr = ctx.cache.find(req_seq_no); if (itr == ctx.cache.end()) { - LOG_DEBUG << "Required lcl peer asked for is not in our lcl cache."; + LOG_DEBUG << "Required lcl seq no peer asked for is not in our lcl cache. " << required_lcl; // Either this node is also not in consesnsus ledger or other node requesting a lcl that is older than node's current // minimum lcl sequence becuase of maximum ledger history range. return false; } else if (itr->second != required_lcl) { - LOG_DEBUG << "Required lcl peer asked for is not in our lcl cache."; + LOG_DEBUG << "Required lcl peer asked for is not in our lcl cache. " << required_lcl; // Either this node or requesting node is in a fork condition. return false; } @@ -513,46 +520,48 @@ namespace ledger */ int retrieve_ledger_history(const p2p::history_request &hr, p2p::history_response &history_response) { - // Get sequence number of minimum lcl required - const size_t pos = hr.minimum_lcl.find("-"); - if (pos == std::string::npos) + uint64_t min_seq_no = 0; + std::string hash; + if (extract_lcl(hr.requester_lcl, min_seq_no, hash) == -1) { - LOG_DEBUG << "lcl serve: Invalid lcl history request. Requested:" << hr.minimum_lcl; + LOG_DEBUG << "lcl serve: Invalid request. Requester lcl invalid:" << hr.requester_lcl; return -1; } // We put the requester's own lcl back in the response so they can validate the liveliness of the response. - history_response.requester_lcl = hr.minimum_lcl; + history_response.requester_lcl = hr.requester_lcl; - uint64_t min_seq_no; - if (util::stoull(hr.minimum_lcl.substr(0, pos), min_seq_no) == -1) // Get required lcl sequence number. + if (min_seq_no > 0) { - LOG_ERROR << "lcl serve: Retrieving minimum ledger sequence no failed. Requested: " << hr.minimum_lcl; - } - - const auto itr = ctx.cache.find(min_seq_no); - if (itr != ctx.cache.end()) // Requested minimum lcl is not in our lcl history cache - { - min_seq_no = itr->first; - - // Check whether minimum lcl requested is same as this node's. - // Evenhough sequence number are same, lcl hash can be changed if one of node is in a fork condition. - if (hr.minimum_lcl != itr->second) + const auto itr = ctx.cache.find(min_seq_no); + if (itr != ctx.cache.end()) // Requested minimum lcl was found in our lcl history cache { - LOG_DEBUG << "lcl serve: Invalid minimum ledger. Requested min lcl:" << hr.minimum_lcl << " Node lcl:" << itr->second; + // Check whether requested minimum lcl hash is same as this node's. + // Evenhough sequence number are same, lcl hash can be changed if one of node is in a fork condition. + if (hr.requester_lcl != itr->second) + { + LOG_DEBUG << "lcl serve: Invalid minimum ledger. Requester lcl:" << hr.requester_lcl << " Node lcl:" << itr->second; + history_response.error = p2p::LEDGER_RESPONSE_ERROR::INVALID_MIN_LEDGER; + return 0; + } + + min_seq_no = itr->first; + } + else if (min_seq_no > ctx.cache.rbegin()->first) //Recieved minimum lcl sequence is ahead of node's lcl sequence. + { + LOG_DEBUG << "lcl serve: Invalid minimum ledger. Requester lcl " << hr.requester_lcl << " is ahead of us."; history_response.error = p2p::LEDGER_RESPONSE_ERROR::INVALID_MIN_LEDGER; return 0; } - } - else if (min_seq_no > ctx.cache.rbegin()->first) //Recieved minimum lcl sequence is ahead of node's lcl sequence. - { - LOG_DEBUG << "lcl serve: Invalid minimum ledger. Recieved minimum seq no is ahead of node current seq no. Requested lcl:" << hr.minimum_lcl; - history_response.error = p2p::LEDGER_RESPONSE_ERROR::INVALID_MIN_LEDGER; - return 0; + else + { + LOG_DEBUG << "lcl serve: Requester lcl is not in our lcl cache. Sending our entire history."; + min_seq_no = ctx.cache.begin()->first; + } } else { - LOG_DEBUG << "lcl serve: Minimum lcl peer asked for is not in our lcl cache. Therefore sending from node minimum lcl."; + LOG_DEBUG << "lcl serve: Requester lcl is GENSIS. Sending our entire history."; min_seq_no = ctx.cache.begin()->first; } @@ -620,21 +629,28 @@ namespace ledger if (!contains_requested_lcl) { - LOG_DEBUG << "lcl sync: Peer sent us a history response but not containing the lcl we asked for."; + LOG_INFO << "lcl sync: Peer sent us a history response but not containing the lcl we asked for."; return -1; } // Check integrity of recieved lcl list. // By checking recieved lcl hashes matches lcl content by applying hashing for each raw content. - // TODO: Also verify chain hashes. std::string previous_history_block_lcl; uint64_t previous_history_block_seq_no; for (auto &[seq_no, ledger] : hr.hist_ledger_blocks) { // Individually check each ledger entry's integrity before the chain check. - if (!check_block_integrity(ledger.lcl, ledger.block_buffer)) + uint64_t lcl_seq_no; + std::string lcl_hash; + if (extract_lcl(ledger.lcl, lcl_seq_no, lcl_hash) == -1) { - LOG_DEBUG << "lcl sync: Peer sent us a history response but the ledger data does not match the hashes."; + LOG_INFO << "lcl sync: Error when parsing lcl " << ledger.lcl; + return -1; + } + + if (!check_block_integrity(lcl_hash, ledger.block_buffer)) + { + LOG_INFO << "lcl sync: Peer sent us a history response but the ledger data does not match the hashes."; // todo: we should penalize peer who sent this. return -1; } @@ -645,7 +661,7 @@ namespace ledger const p2p::proposal proposal = msg::fbuf::ledger::create_proposal_from_ledger_block(ledger.block_buffer); if ((seq_no - previous_history_block_seq_no != 1) && (previous_history_block_lcl != proposal.lcl)) { - LOG_ERROR << "Ledger block chain-link verification failed. " << ledger.lcl; + LOG_INFO << "Ledger block chain-link verification failed. " << ledger.lcl; return -1; } } @@ -670,7 +686,7 @@ namespace ledger while (it != reverse_history_ptr) { remove_ledger(it->second); - // Erase function advance the iteratior. + // Erase function advance the iterator. ctx.cache.erase((--it).base()); } @@ -701,26 +717,22 @@ namespace ledger } /** * Check the integrity of the given ledger. - * @param lcl supplied lcl of the ledger. + * @param supplied_hash supplied hash hex of the ledger block. * @param raw_ledger ledger. * @return true if the integrity check passes and false otherwise. */ - bool check_block_integrity(std::string_view lcl, const std::vector &block_buffer) + bool check_block_integrity(std::string_view supplied_hash, const std::vector &block_buffer) { - const size_t pos = lcl.find("-"); - std::string_view supplied_lcl_hash = lcl.substr((pos + 1), (lcl.size() - 1)); - // Get binary hash of the serialized lcl. - const std::string binary_lcl_hash = crypto::get_hash(block_buffer.data(), block_buffer.size()); + const std::string binary_block_hash = crypto::get_hash(block_buffer.data(), block_buffer.size()); // Get hex from binary hash. - std::string lcl_hash; + std::string block_hash; + util::bin2hex(block_hash, + reinterpret_cast(binary_block_hash.data()), + binary_block_hash.size()); - util::bin2hex(lcl_hash, - reinterpret_cast(binary_lcl_hash.data()), - binary_lcl_hash.size()); - - return lcl_hash == supplied_lcl_hash; + return block_hash == supplied_hash; } /** @@ -733,8 +745,7 @@ namespace ledger try { list.sort([](std::string &a, std::string &b) { - uint64_t seq_no_a; - uint64_t seq_no_b; + uint64_t seq_no_a, seq_no_b; if (util::stoull(a.substr(0, a.find("-")), seq_no_a) == -1) { throw "Lcl file parsing error in file " + a + " in " + conf::ctx.hist_dir; @@ -764,4 +775,27 @@ namespace ledger } } + int extract_lcl(const std::string &lcl, uint64_t &seq_no, std::string &hash) + { + if (lcl == GENESIS_LEDGER) + { + seq_no = 0; + hash = lcl.substr(2); + return 0; + } + + const size_t pos = lcl.find("-"); + if (pos == std::string::npos) + return -1; + + if (util::stoull(lcl.substr(0, pos), seq_no) == -1) + return -1; + + hash = lcl.substr(pos + 1); + if (hash.size() != 64) + return -1; + + return 0; + } + } // namespace ledger \ No newline at end of file diff --git a/src/ledger.hpp b/src/ledger.hpp index 85406d15..adf17b69 100644 --- a/src/ledger.hpp +++ b/src/ledger.hpp @@ -14,6 +14,7 @@ namespace ledger { // The current target lcl that we are syncing towards. std::string target_lcl; + uint64_t target_lcl_seq_no; std::mutex target_lcl_mutex; // Lists holding history requests and responses collected from incoming p2p messages. @@ -71,7 +72,7 @@ namespace ledger void lcl_syncer_loop(); - void set_sync_target(std::string_view target_lcl); + void set_sync_target(const std::string &target_lcl); const std::pair get_ledger_cache_top(); @@ -87,7 +88,7 @@ namespace ledger void remove_ledger(const std::string &file_name); - void send_ledger_history_request(std::string_view minimum_lcl, std::string_view required_lcl); + void send_ledger_history_request(std::string_view current_lcl, std::string_view required_lcl); bool check_required_lcl_availability(const std::string &required_lcl); @@ -95,10 +96,12 @@ namespace ledger int handle_ledger_history_response(const p2p::history_response &hr, std::string &new_lcl); - bool check_block_integrity(std::string_view lcl, const std::vector &block_buffer); + bool check_block_integrity(std::string_view hash, const std::vector &block_buffer); int sort_lcl_filenames_and_validate(std::list &list); + int extract_lcl(const std::string &lcl, uint64_t &seq_no, std::string &hash); + } // namespace ledger #endif \ No newline at end of file diff --git a/src/msg/fbuf/p2pmsg_content.fbs b/src/msg/fbuf/p2pmsg_content.fbs index 90d2b9d4..8d780578 100644 --- a/src/msg/fbuf/p2pmsg_content.fbs +++ b/src/msg/fbuf/p2pmsg_content.fbs @@ -63,7 +63,6 @@ table Npl_Message { //NPL type message schema } table History_Request_Message { //Ledger History request type message schema - minimum_lcl:[ubyte]; required_lcl:[ubyte]; } diff --git a/src/msg/fbuf/p2pmsg_content_generated.h b/src/msg/fbuf/p2pmsg_content_generated.h index 7ee5ea33..a032d94e 100644 --- a/src/msg/fbuf/p2pmsg_content_generated.h +++ b/src/msg/fbuf/p2pmsg_content_generated.h @@ -999,15 +999,8 @@ inline flatbuffers::Offset CreateNpl_MessageDirect( struct History_Request_Message FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef History_Request_MessageBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { - VT_MINIMUM_LCL = 4, - VT_REQUIRED_LCL = 6 + VT_REQUIRED_LCL = 4 }; - const flatbuffers::Vector *minimum_lcl() const { - return GetPointer *>(VT_MINIMUM_LCL); - } - flatbuffers::Vector *mutable_minimum_lcl() { - return GetPointer *>(VT_MINIMUM_LCL); - } const flatbuffers::Vector *required_lcl() const { return GetPointer *>(VT_REQUIRED_LCL); } @@ -1016,8 +1009,6 @@ struct History_Request_Message FLATBUFFERS_FINAL_CLASS : private flatbuffers::Ta } bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && - VerifyOffset(verifier, VT_MINIMUM_LCL) && - verifier.VerifyVector(minimum_lcl()) && VerifyOffset(verifier, VT_REQUIRED_LCL) && verifier.VerifyVector(required_lcl()) && verifier.EndTable(); @@ -1028,9 +1019,6 @@ struct History_Request_MessageBuilder { typedef History_Request_Message Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; - void add_minimum_lcl(flatbuffers::Offset> minimum_lcl) { - fbb_.AddOffset(History_Request_Message::VT_MINIMUM_LCL, minimum_lcl); - } void add_required_lcl(flatbuffers::Offset> required_lcl) { fbb_.AddOffset(History_Request_Message::VT_REQUIRED_LCL, required_lcl); } @@ -1048,23 +1036,18 @@ struct History_Request_MessageBuilder { inline flatbuffers::Offset CreateHistory_Request_Message( flatbuffers::FlatBufferBuilder &_fbb, - flatbuffers::Offset> minimum_lcl = 0, flatbuffers::Offset> required_lcl = 0) { History_Request_MessageBuilder builder_(_fbb); builder_.add_required_lcl(required_lcl); - builder_.add_minimum_lcl(minimum_lcl); return builder_.Finish(); } inline flatbuffers::Offset CreateHistory_Request_MessageDirect( flatbuffers::FlatBufferBuilder &_fbb, - const std::vector *minimum_lcl = nullptr, const std::vector *required_lcl = nullptr) { - auto minimum_lcl__ = minimum_lcl ? _fbb.CreateVector(*minimum_lcl) : 0; auto required_lcl__ = required_lcl ? _fbb.CreateVector(*required_lcl) : 0; return msg::fbuf::p2pmsg::CreateHistory_Request_Message( _fbb, - minimum_lcl__, required_lcl__); } diff --git a/src/msg/fbuf/p2pmsg_helpers.cpp b/src/msg/fbuf/p2pmsg_helpers.cpp index 4c5898f8..db7af8d2 100644 --- a/src/msg/fbuf/p2pmsg_helpers.cpp +++ b/src/msg/fbuf/p2pmsg_helpers.cpp @@ -246,12 +246,12 @@ namespace msg::fbuf::p2pmsg * @param msg Flatbuffer History request message received from the peer. * @return A History request struct representing the message. */ - const p2p::history_request create_history_request_from_msg(const History_Request_Message &msg) + const p2p::history_request create_history_request_from_msg(const History_Request_Message &msg, const flatbuffers::Vector *lcl) { p2p::history_request hr; - if (msg.minimum_lcl()) - hr.minimum_lcl = flatbuff_bytes_to_sv(msg.minimum_lcl()); + if (lcl) + hr.requester_lcl = flatbuff_bytes_to_sv(lcl); if (msg.required_lcl()) hr.required_lcl = flatbuff_bytes_to_sv(msg.required_lcl()); @@ -418,7 +418,6 @@ namespace msg::fbuf::p2pmsg flatbuffers::Offset hrmsg = CreateHistory_Request_Message( builder, - sv_to_flatbuff_bytes(builder, hr.minimum_lcl), sv_to_flatbuff_bytes(builder, hr.required_lcl)); flatbuffers::Offset message = CreateContent(builder, Message_History_Request_Message, hrmsg.Union()); @@ -426,7 +425,7 @@ namespace msg::fbuf::p2pmsg // Now that we have built the content message, // we need to sign it and place it inside a container message. - create_containermsg_from_content(container_builder, builder, {}, false); + create_containermsg_from_content(container_builder, builder, hr.requester_lcl, false); } /** diff --git a/src/msg/fbuf/p2pmsg_helpers.hpp b/src/msg/fbuf/p2pmsg_helpers.hpp index 1bfb99bd..fb749cd5 100644 --- a/src/msg/fbuf/p2pmsg_helpers.hpp +++ b/src/msg/fbuf/p2pmsg_helpers.hpp @@ -32,7 +32,7 @@ namespace msg::fbuf::p2pmsg const p2p::proposal create_proposal_from_msg(const Proposal_Message &msg, const flatbuffers::Vector *pubkey, const uint64_t timestamp, const flatbuffers::Vector *lcl); - const p2p::history_request create_history_request_from_msg(const History_Request_Message &msg); + const p2p::history_request create_history_request_from_msg(const History_Request_Message &msg, const flatbuffers::Vector *lcl); const p2p::history_response create_history_response_from_msg(const History_Response_Message &msg); diff --git a/src/p2p/p2p.hpp b/src/p2p/p2p.hpp index 185be6c3..9c40182b 100644 --- a/src/p2p/p2p.hpp +++ b/src/p2p/p2p.hpp @@ -40,7 +40,7 @@ namespace p2p struct history_request { - std::string minimum_lcl; + std::string requester_lcl; std::string required_lcl; }; diff --git a/src/p2p/peer_session_handler.cpp b/src/p2p/peer_session_handler.cpp index 5b9d51e4..4edab282 100644 --- a/src/p2p/peer_session_handler.cpp +++ b/src/p2p/peer_session_handler.cpp @@ -218,7 +218,7 @@ namespace p2p // If max number of history requests reached skip the rest. if (ledger::sync_ctx.collected_history_requests.size() < ledger::HISTORY_REQ_LIST_CAP) { - const p2p::history_request hr = p2pmsg::create_history_request_from_msg(*content->message_as_History_Request_Message()); + const p2p::history_request hr = p2pmsg::create_history_request_from_msg(*content->message_as_History_Request_Message(), container->lcl()); ledger::sync_ctx.collected_history_requests.push_back(std::make_pair(session.uniqueid, std::move(hr))); } else