P2P Flatbuffers schema refactor. (#255)

CMakeLists.txt

@@ -44,8 +44,8 @@ add_executable(hpcore
     src/sc/sc.cpp
     src/comm/comm_session.cpp
     src/msg/fbuf/common_helpers.cpp
-    src/msg/fbuf/p2pmsg_helpers.cpp
     src/msg/fbuf/ledger_helpers.cpp
+    src/msg/fbuf/p2pmsg_conversion.cpp
     src/msg/json/controlmsg_json.cpp
     src/msg/controlmsg_parser.cpp
     src/msg/json/usrmsg_json.cpp

README.md

@@ -28,9 +28,9 @@ Run the setup script located at the repo root (tested on Ubuntu 18.04).
 ## FlatBuffers message definitions
 If you update flatbuffers message definitions, you need to run the flatbuffers code generator to update the stubs.
 
-Example: When you make a change to `p2pmsg_content.fbc` defnition file, you need to run this:
+Example: When you make a change to `p2pmsg.fbs` defnition file, you need to run this:
 
-`flatc -o src/msg/fbuf/ --gen-mutable --cpp src/msg/fbuf/p2pmsg_content.fbs`
+`flatc -o src/msg/fbuf/ --gen-mutable --cpp src/msg/fbuf/p2pmsg.fbs`
 
 ## Code structure
 Code is divided into subsystems via namespaces.

src/consensus.cpp

@@ -4,7 +4,7 @@
 #include "usr/usr.hpp"
 #include "usr/user_input.hpp"
 #include "p2p/p2p.hpp"
-#include "msg/fbuf/p2pmsg_helpers.hpp"
+#include "msg/fbuf/p2pmsg_conversion.hpp"
 #include "msg/usrmsg_parser.hpp"
 #include "msg/usrmsg_common.hpp"
 #include "p2p/peer_session_handler.hpp"
@@ -400,7 +400,7 @@ namespace consensus
             }
         }
 
-        flatbuffers::FlatBufferBuilder fbuf(1024);
+        flatbuffers::FlatBufferBuilder fbuf;
         p2pmsg::create_msg_from_nonunl_proposal(fbuf, nup);
         p2p::broadcast_message(fbuf, true);
 
@@ -418,7 +418,7 @@ namespace consensus
         if (conf::cfg.node.role == conf::ROLE::OBSERVER || !conf::cfg.node.is_unl) // If we are a non-unl node, do not broadcast proposals.
             return;
 
-        flatbuffers::FlatBufferBuilder fbuf(1024);
+        flatbuffers::FlatBufferBuilder fbuf;
         p2pmsg::create_msg_from_proposal(fbuf, p);
         p2p::broadcast_message(fbuf, true, false, !conf::cfg.contract.is_consensus_public);
 
@@ -436,11 +436,11 @@ namespace consensus
      * @param npl_msg Constructed npl message.
      * @return Returns true if enqueue is success otherwise false.
      */
-    bool push_npl_message(p2p::npl_message &npl_msg)
+    bool push_npl_message(const p2p::npl_message &npl_msg)
     {
         std::scoped_lock lock(ctx.contract_ctx_mutex);
         if (ctx.contract_ctx)
-            return ctx.contract_ctx->args.npl_messages.try_enqueue(npl_msg);
+            return ctx.contract_ctx->args.npl_messages.try_enqueue(std::move(npl_msg));
         return false;
     }
 
@@ -882,9 +882,6 @@ namespace consensus
             // lcl to be passed to the contract.
             args.lcl_id = new_lcl_id;
 
-            // This is currently used for npl message checks.
-            args.lasl_primary_shard_id = new_last_primary_shard_id;
-
             // Populate user bufs.
             if (feed_user_inputs_to_contract_bufmap(args.userbufs, cons_prop) == -1)
                 return -1;

src/consensus.hpp

@@ -121,7 +121,7 @@ namespace consensus
 
     void broadcast_nonunl_proposal();
 
-    bool push_npl_message(p2p::npl_message &npl_message);
+    bool push_npl_message(const p2p::npl_message &npl_message);
 
     int verify_and_populate_candidate_user_inputs(const uint64_t lcl_seq_no);
 

src/hpfs/hpfs_serve.cpp

@@ -2,9 +2,7 @@
 #include "../util/h32.hpp"
 #include "../util/util.hpp"
 #include "../p2p/p2p.hpp"
-#include "../msg/fbuf/p2pmsg_content_generated.h"
-#include "../msg/fbuf/p2pmsg_helpers.hpp"
-#include "../msg/fbuf/common_helpers.hpp"
+#include "../msg/fbuf/p2pmsg_conversion.hpp"
 #include "../ledger/ledger.hpp"
 #include "../hplog.hpp"
 #include "hpfs_serve.hpp"
@@ -94,9 +92,9 @@ namespace hpfs
 
                     // Session id is in binary format. Converting to hex before printing.
                     LOG_DEBUG << "Serving hpfs request from [" << util::to_hex(session_id).substr(2, 10) << "]";
-                    flatbuffers::FlatBufferBuilder fbuf(1024);
+                    flatbuffers::FlatBufferBuilder fbuf;
 
-                    if (hpfs_serve::create_hpfs_response(fbuf, hr, last_primary_shard_id) == 1)
+                    if (hpfs_serve::create_hpfs_response(fbuf, hr) == 1)
                     {
                         // Find the peer that we should send the hpfs response to.
                         std::scoped_lock<std::mutex> lock(p2p::ctx.peer_connections_mutex);
@@ -125,11 +123,10 @@ namespace hpfs
      * Creates the reply message for a given hpfs request.
      * @param fbuf The flatbuffer builder to construct the reply message.
      * @param hr The hpfs request which should be replied to.
-     * @param last_primary_shard_id Last primary shard id.
      * @return 1 if successful hpfs response was generated. 0 if request is invalid
      *         and no response was generated. -1 on error.
      */
-    int hpfs_serve::create_hpfs_response(flatbuffers::FlatBufferBuilder &fbuf, const p2p::hpfs_request &hr, const p2p::sequence_hash &last_primary_shard_id)
+    int hpfs_serve::create_hpfs_response(flatbuffers::FlatBufferBuilder &fbuf, const p2p::hpfs_request &hr)
     {
         LOG_DEBUG << "Serving hpfs req. path:" << hr.parent_path << " block_id:" << hr.block_id;
 
@@ -154,7 +151,7 @@ namespace hpfs
                 resp.hash = hr.expected_hash;
                 resp.data = std::string_view(reinterpret_cast<const char *>(block.data()), block.size());
 
-                msg::fbuf::p2pmsg::create_msg_from_block_response(fbuf, resp, fs_mount->mount_id, last_primary_shard_id);
+                p2pmsg::create_msg_from_block_response(fbuf, resp, fs_mount->mount_id);
                 return 1; // Success.
             }
         }
@@ -174,9 +171,9 @@ namespace hpfs
                 }
                 else if (result == 1)
                 {
-                    msg::fbuf::p2pmsg::create_msg_from_filehashmap_response(
+                    p2pmsg::create_msg_from_filehashmap_response(
                         fbuf, hr.parent_path, fs_mount->mount_id, block_hashes,
-                        file_length, hr.expected_hash, last_primary_shard_id);
+                        file_length, hr.expected_hash);
                     return 1; // Success.
                 }
             }
@@ -194,8 +191,8 @@ namespace hpfs
                 }
                 else if (result == 1)
                 {
-                    msg::fbuf::p2pmsg::create_msg_from_fsentry_response(
-                        fbuf, hr.parent_path, fs_mount->mount_id, child_hash_nodes, hr.expected_hash, last_primary_shard_id);
+                    p2pmsg::create_msg_from_fsentry_response(
+                        fbuf, hr.parent_path, fs_mount->mount_id, child_hash_nodes, hr.expected_hash);
                     return 1; // Success.
                 }
             }

src/hpfs/hpfs_serve.hpp

@@ -4,7 +4,6 @@
 #include "../util/h32.hpp"
 #include "hpfs_mount.hpp"
 #include "../p2p/p2p.hpp"
-#include "../msg/fbuf/p2pmsg_content_generated.h"
 
 namespace hpfs
 {
@@ -28,7 +27,7 @@ namespace hpfs
 
         void deinit();
 
-        int create_hpfs_response(flatbuffers::FlatBufferBuilder &fbuf, const p2p::hpfs_request &hr, const p2p::sequence_hash &last_primary_shard_id);
+        int create_hpfs_response(flatbuffers::FlatBufferBuilder &fbuf, const p2p::hpfs_request &hr);
 
         int get_data_block(std::vector<uint8_t> &block, const std::string_view vpath,
                            const uint32_t block_id, const util::h32 expected_hash);

src/hpfs/hpfs_sync.cpp

@@ -1,15 +1,15 @@
-#include "../msg/fbuf/p2pmsg_helpers.hpp"
-#include "../msg/fbuf/p2pmsg_content_generated.h"
+#include "../pchheader.hpp"
+#include "../msg/fbuf/p2pmsg_conversion.hpp"
+#include "../msg/fbuf/p2pmsg_generated.h"
 #include "../msg/fbuf/common_helpers.hpp"
 #include "../p2p/p2p.hpp"
-#include "../pchheader.hpp"
-#include "../ledger/ledger.hpp"
 #include "../hplog.hpp"
 #include "../util/util.hpp"
 #include "../util/h32.hpp"
+#include "../crypto.hpp"
 #include "hpfs_sync.hpp"
-#include "../sc/sc.hpp"
-#include "../unl.hpp"
+
+namespace p2pmsg = msg::fbuf::p2pmsg;
 
 namespace hpfs
 {
@@ -209,10 +209,8 @@ namespace hpfs
     */
     int hpfs_sync::request_loop(const util::h32 current_target_hash, util::h32 &updated_state)
     {
-        p2p::sequence_hash last_primary_shard_id = ledger::ctx.get_last_primary_shard_id();
-
         // Send the initial root hpfs request of the current target.
-        submit_request(backlog_item{current_target.item_type, current_target.vpath, -1, current_target_hash}, last_primary_shard_id);
+        submit_request(backlog_item{current_target.item_type, current_target.vpath, -1, current_target_hash});
 
         // Indicates whether any responses were processed in the previous loop iteration.
         bool prev_responses_processed = false;
@@ -226,9 +224,6 @@ namespace hpfs
             if (!prev_responses_processed)
                 util::sleep(REQUEST_LOOP_WAIT);
 
-            // Get the current last shard information.
-            last_primary_shard_id = ledger::ctx.get_last_primary_shard_id();
-
             // Move the received hpfs responses to the local response list.
             swap_collected_responses();
 
@@ -245,12 +240,12 @@ namespace hpfs
 
                 LOG_DEBUG << "Hpfs " << name << " sync: Processing hpfs response from [" << response.first.substr(2, 10) << "]";
 
-                const msg::fbuf::p2pmsg::Content *content = msg::fbuf::p2pmsg::GetContent(response.second.data());
-                const msg::fbuf::p2pmsg::Hpfs_Response_Message *resp_msg = content->message_as_Hpfs_Response_Message();
+                const p2pmsg::P2PMsg &msg = *p2pmsg::GetP2PMsg(response.second.data());
+                const p2pmsg::HpfsResponseMsg &resp_msg = *msg.content_as_HpfsResponseMsg();
 
                 // Check whether we are actually waiting for this response. If not, ignore it.
-                std::string_view hash = msg::fbuf::flatbuff_bytes_to_sv(resp_msg->hash());
-                std::string_view vpath = msg::fbuf::flatbuff_str_to_sv(resp_msg->path());
+                std::string_view hash = msg::fbuf::flatbuf_bytes_to_sv(resp_msg.hash());
+                std::string_view vpath = msg::fbuf::flatbuf_str_to_sv(resp_msg.path());
 
                 const std::string key = std::string(vpath).append(hash);
                 const auto pending_resp_itr = submitted_requests.find(key);
@@ -261,15 +256,15 @@ namespace hpfs
                 }
 
                 // Process the message based on response type.
-                const msg::fbuf::p2pmsg::Hpfs_Response msg_type = resp_msg->hpfs_response_type();
+                const p2pmsg::HpfsResponse msg_type = resp_msg.content_type();
 
-                if (msg_type == msg::fbuf::p2pmsg::Hpfs_Response_Fs_Entry_Response)
+                if (msg_type == p2pmsg::HpfsResponse_HpfsFsEntryResponse)
                 {
-                    const msg::fbuf::p2pmsg::Fs_Entry_Response *fs_resp = resp_msg->hpfs_response_as_Fs_Entry_Response();
+                    const p2pmsg::HpfsFsEntryResponse &fs_resp = *resp_msg.content_as_HpfsFsEntryResponse();
 
                     // Get fs entries we have received.
                     std::unordered_map<std::string, p2p::hpfs_fs_hash_entry> peer_fs_entry_map;
-                    msg::fbuf::p2pmsg::flatbuf_hpfsfshashentry_to_hpfsfshashentry(peer_fs_entry_map, fs_resp->entries());
+                    p2pmsg::flatbuf_hpfsfshashentry_to_hpfsfshashentry(peer_fs_entry_map, fs_resp.entries());
 
                     // Commented for now. Need to change the way the hash is calculated once the flatbuffer re-architecture finishes.
                     // Validate received fs data against the hash.
@@ -281,13 +276,13 @@ namespace hpfs
 
                     handle_fs_entry_response(vpath, peer_fs_entry_map);
                 }
-                else if (msg_type == msg::fbuf::p2pmsg::Hpfs_Response_File_HashMap_Response)
+                else if (msg_type == p2pmsg::HpfsResponse_HpfsFileHashMapResponse)
                 {
-                    const msg::fbuf::p2pmsg::File_HashMap_Response *file_resp = resp_msg->hpfs_response_as_File_HashMap_Response();
+                    const p2pmsg::HpfsFileHashMapResponse &file_resp = *resp_msg.content_as_HpfsFileHashMapResponse();
 
                     // File block hashes we received from the peer.
-                    const util::h32 *peer_hashes = reinterpret_cast<const util::h32 *>(file_resp->hash_map()->data());
-                    const size_t peer_hash_count = file_resp->hash_map()->size() / sizeof(util::h32);
+                    const util::h32 *peer_hashes = reinterpret_cast<const util::h32 *>(file_resp.hash_map()->data());
+                    const size_t peer_hash_count = file_resp.hash_map()->size() / sizeof(util::h32);
 
                     // Commented for now. Need to change the way the hash is calculated once the flatbuffer re-architecture finishes.
                     // Validate received hashmap against the hash.
@@ -297,15 +292,15 @@ namespace hpfs
                     //     continue;
                     // }
 
-                    handle_file_hashmap_response(vpath, peer_hashes, peer_hash_count, file_resp->file_length());
+                    handle_file_hashmap_response(vpath, peer_hashes, peer_hash_count, file_resp.file_length());
                 }
-                else if (msg_type == msg::fbuf::p2pmsg::Hpfs_Response_Block_Response)
+                else if (msg_type == p2pmsg::HpfsResponse_HpfsBlockResponse)
                 {
-                    const msg::fbuf::p2pmsg::Block_Response *block_resp = resp_msg->hpfs_response_as_Block_Response();
+                    const p2pmsg::HpfsBlockResponse &block_resp = *resp_msg.content_as_HpfsBlockResponse();
 
                     // Get the file path of the block data we have received.
-                    const uint32_t block_id = block_resp->block_id();
-                    std::string_view buf = msg::fbuf::flatbuff_bytes_to_sv(block_resp->data());
+                    const uint32_t block_id = block_resp.block_id();
+                    std::string_view buf = msg::fbuf::flatbuf_bytes_to_sv(block_resp.data());
 
                     // Commented for now. Need to change the way the hash is calculated once the flatbuffer re-architecture finishes.
                     // Validate received block data against the hash.
@@ -373,7 +368,7 @@ namespace hpfs
                     // Reset the counter and re-submit request.
                     request.waiting_time = 0;
                     LOG_DEBUG << "Hpfs " << name << " sync: Resubmitting request...";
-                    submit_request(request, last_primary_shard_id);
+                    submit_request(request);
                 }
             }
 
@@ -387,7 +382,7 @@ namespace hpfs
                         return 0;
 
                     const backlog_item &request = pending_requests.front();
-                    submit_request(request, last_primary_shard_id);
+                    submit_request(request);
                     pending_requests.pop_front();
                 }
             }
@@ -480,11 +475,10 @@ namespace hpfs
      * @param is_file Whether the requested path if a file or dir.
      * @param block_id The requested block id. Only relevant if requesting a file block. Otherwise -1.
      * @param expected_hash The expected hash of the requested data. The peer will ignore the request if their hash is different.
-     * @param last_primary_shard_id The last primary shard id.
      * @param target_pubkey The peer pubkey the request was submitted to.
      */
     void hpfs_sync::request_state_from_peer(const std::string &path, const bool is_file, const int32_t block_id,
-                                            const util::h32 expected_hash, const p2p::sequence_hash &last_primary_shard_id, std::string &target_pubkey)
+                                            const util::h32 expected_hash, std::string &target_pubkey)
     {
         p2p::hpfs_request hr;
         hr.parent_path = path;
@@ -493,15 +487,15 @@ namespace hpfs
         hr.expected_hash = expected_hash;
         hr.mount_id = fs_mount->mount_id;
 
-        flatbuffers::FlatBufferBuilder fbuf(1024);
-        msg::fbuf::p2pmsg::create_msg_from_hpfs_request(fbuf, hr, last_primary_shard_id);
-        p2p::send_message_to_random_peer(fbuf, target_pubkey); //todo: send to a node that hold the majority hpfs state to improve reliability of retrieving hpfs state.
+        flatbuffers::FlatBufferBuilder fbuf;
+        p2pmsg::create_msg_from_hpfs_request(fbuf, hr);
+        p2p::send_message_to_random_peer(fbuf, target_pubkey); //todo: send to a node that hold the expected hash to improve reliability of retrieving hpfs state.
     }
 
     /**
      * Submits a pending hpfs request to the peer.
      */
-    void hpfs_sync::submit_request(const backlog_item &request, const p2p::sequence_hash &last_primary_shard_id)
+    void hpfs_sync::submit_request(const backlog_item &request)
     {
         const std::string key = std::string(request.path)
                                     .append(reinterpret_cast<const char *>(&request.expected_hash), sizeof(util::h32));
@@ -509,7 +503,7 @@ namespace hpfs
 
         const bool is_file = request.type != BACKLOG_ITEM_TYPE::DIR;
         std::string target_pubkey;
-        request_state_from_peer(request.path, is_file, request.block_id, request.expected_hash, last_primary_shard_id, target_pubkey);
+        request_state_from_peer(request.path, is_file, request.block_id, request.expected_hash, target_pubkey);
 
         if (!target_pubkey.empty())
             LOG_DEBUG << "Hpfs " << name << " sync: Requesting from [" << target_pubkey.substr(2, 10) << "]. type:" << request.type

src/hpfs/hpfs_sync.hpp

@@ -3,9 +3,7 @@
 
 #include "../pchheader.hpp"
 #include "../p2p/p2p.hpp"
-#include "../msg/fbuf/p2pmsg_content_generated.h"
 #include "../util/h32.hpp"
-#include "../crypto.hpp"
 #include "./hpfs_mount.hpp"
 
 namespace hpfs
@@ -109,9 +107,9 @@ namespace hpfs
         bool should_stop_request_loop(const util::h32 &current_target_hash);
 
         void request_state_from_peer(const std::string &path, const bool is_file, const int32_t block_id,
-                                            const util::h32 expected_hash, const p2p::sequence_hash &last_primary_shard_id, std::string &target_pubkey);
+                                            const util::h32 expected_hash, std::string &target_pubkey);
 
-        void submit_request(const backlog_item &request, const p2p::sequence_hash &last_primary_shard_id);
+        void submit_request(const backlog_item &request);
 
         int handle_fs_entry_response(std::string_view vpath, std::unordered_map<std::string, p2p::hpfs_fs_hash_entry> &fs_entry_map);
 

src/msg/fbuf/common_helpers.cpp

@@ -3,123 +3,54 @@
 namespace msg::fbuf
 {
 
-    //---Conversion helpers from flatbuffers data types to std data types---//
+    //---Flatbuf to std---//
 
-    /**
- * Returns string_view from flat buffer data pointer and length.
- */
-    std::string_view flatbuff_bytes_to_sv(const uint8_t *data, const flatbuffers::uoffset_t length)
+    std::string_view flatbuf_bytes_to_sv(const uint8_t *data, const flatbuffers::uoffset_t length)
     {
         const char *signature_content_str = reinterpret_cast<const char *>(data);
         return std::string_view(signature_content_str, length);
     }
 
-    /**
- * Returns string_view from Flat Buffer vector of bytes.
- */
-    std::string_view flatbuff_bytes_to_sv(const flatbuffers::Vector<uint8_t> *buffer)
+    std::string_view flatbuf_bytes_to_sv(const flatbuffers::Vector<uint8_t> *buffer)
     {
-        return flatbuff_bytes_to_sv(buffer->Data(), buffer->size());
+        return flatbuf_bytes_to_sv(buffer->Data(), buffer->size());
     }
 
-    /**
- * Returns return string_view from Flat Buffer string.
- */
-    std::string_view flatbuff_str_to_sv(const flatbuffers::String *buffer)
+    std::string_view flatbuf_str_to_sv(const flatbuffers::String *buffer)
     {
-        return flatbuff_bytes_to_sv(buffer->Data(), buffer->size());
+        return flatbuf_bytes_to_sv(buffer->Data(), buffer->size());
     }
 
-    /**
- * Returns hash from Flat Buffer vector of bytes.
- */
-    util::h32 flatbuff_bytes_to_hash(const flatbuffers::Vector<uint8_t> *buffer)
+    util::h32 flatbuf_bytes_to_hash(const flatbuffers::Vector<uint8_t> *buffer)
     {
         return *reinterpret_cast<const util::h32 *>(buffer->data());
     }
 
-    /**
- * Returns set from Flatbuffer vector of ByteArrays.
- */
-    const std::set<std::string> flatbuf_bytearrayvector_to_stringlist(const flatbuffers::Vector<flatbuffers::Offset<ByteArray>> *fbvec)
+    std::string_view builder_to_string_view(const flatbuffers::FlatBufferBuilder &builder)
     {
-        std::set<std::string> set;
-        for (auto el : *fbvec)
-            set.emplace(std::string(flatbuff_bytes_to_sv(el->array())));
-        return set;
+        std::string_view msg = std::string_view(
+            reinterpret_cast<const char *>(builder.GetBufferPointer()), builder.GetSize());
+        return msg;
     }
 
-    /**
- * Returns a map from Flatbuffer vector of key value pairs.
- */
-    const std::unordered_map<std::string, const std::string>
-    flatbuf_pairvector_to_stringmap(const flatbuffers::Vector<flatbuffers::Offset<BytesKeyValuePair>> *fbvec)
-    {
-        std::unordered_map<std::string, const std::string> map;
-        map.reserve(fbvec->size());
-        for (auto el : *fbvec)
-            map.emplace(flatbuff_bytes_to_sv(el->key()), flatbuff_bytes_to_sv(el->value()));
-        return map;
-    }
+    //---std to Flatbuf---//
 
-    //---Conversion helpers from std data types to flatbuffers data types---//
-    //---These are used in constructing Flatbuffer messages using builders---//
-
-    /**
- * Returns Flatbuffer bytes vector from string_view.
- */
     const flatbuffers::Offset<flatbuffers::Vector<uint8_t>>
-    sv_to_flatbuff_bytes(flatbuffers::FlatBufferBuilder &builder, std::string_view sv)
+    sv_to_flatbuf_bytes(flatbuffers::FlatBufferBuilder &builder, std::string_view sv)
     {
         return builder.CreateVector(reinterpret_cast<const uint8_t *>(sv.data()), sv.size());
     }
 
-    /**
- * Returns Flatbuffer string from string_view.
- */
     const flatbuffers::Offset<flatbuffers::String>
-    sv_to_flatbuff_str(flatbuffers::FlatBufferBuilder &builder, std::string_view sv)
+    sv_to_flatbuf_str(flatbuffers::FlatBufferBuilder &builder, std::string_view sv)
     {
         return builder.CreateString(sv);
     }
 
-    /**
- * Returns Flatbuffer bytes vector from hash.
- */
     const flatbuffers::Offset<flatbuffers::Vector<uint8_t>>
-    hash_to_flatbuff_bytes(flatbuffers::FlatBufferBuilder &builder, const util::h32 hash)
+    hash_to_flatbuf_bytes(flatbuffers::FlatBufferBuilder &builder, const util::h32 hash)
     {
         return builder.CreateVector(reinterpret_cast<const uint8_t *>(&hash), sizeof(util::h32));
     }
 
-    /**
- * Returns Flatbuffer vector of ByteArrays from given set of strings.
- */
-    const flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<ByteArray>>>
-    stringlist_to_flatbuf_bytearrayvector(flatbuffers::FlatBufferBuilder &builder, const std::set<std::string> &set)
-    {
-        std::vector<flatbuffers::Offset<ByteArray>> fbvec;
-        fbvec.reserve(set.size());
-        for (std::string_view str : set)
-            fbvec.push_back(CreateByteArray(builder, sv_to_flatbuff_bytes(builder, str)));
-        return builder.CreateVector(fbvec);
-    }
-
-    /**
- * Returns Flatbuffer vector of key value pairs from given map.
- */
-    const flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<BytesKeyValuePair>>>
-    stringmap_to_flatbuf_bytepairvector(flatbuffers::FlatBufferBuilder &builder, const std::unordered_map<std::string, const std::string> &map)
-    {
-        std::vector<flatbuffers::Offset<BytesKeyValuePair>> fbvec;
-        fbvec.reserve(map.size());
-        for (auto const &[key, value] : map)
-        {
-            fbvec.push_back(CreateBytesKeyValuePair(
-                builder,
-                sv_to_flatbuff_bytes(builder, key),
-                sv_to_flatbuff_bytes(builder, value)));
-        }
-        return builder.CreateVector(fbvec);
-    }
 } // namespace msg::fbuf

src/msg/fbuf/common_helpers.hpp

@@ -3,46 +3,36 @@
 
 #include "../../pchheader.hpp"
 #include "../../util/h32.hpp"
-#include "common_schema_generated.h"
+#include "p2pmsg_generated.h"
 
 namespace msg::fbuf
 {
     /**
- * This section contains common Flatbuffer message reading/writing helpers.
- */
+     * This section contains common Flatbuffer message reading/writing helpers.
+     */
 
-    //---Conversion helpers from flatbuffers data types to std data types---//
+    //---Flatbuf to std---//
 
-    std::string_view flatbuff_bytes_to_sv(const uint8_t *data, const flatbuffers::uoffset_t length);
+    std::string_view flatbuf_bytes_to_sv(const uint8_t *data, const flatbuffers::uoffset_t length);
 
-    std::string_view flatbuff_bytes_to_sv(const flatbuffers::Vector<uint8_t> *buffer);
+    std::string_view flatbuf_bytes_to_sv(const flatbuffers::Vector<uint8_t> *buffer);
 
-    std::string_view flatbuff_str_to_sv(const flatbuffers::String *buffer);
+    std::string_view flatbuf_str_to_sv(const flatbuffers::String *buffer);
 
-    util::h32 flatbuff_bytes_to_hash(const flatbuffers::Vector<uint8_t> *buffer);
+    util::h32 flatbuf_bytes_to_hash(const flatbuffers::Vector<uint8_t> *buffer);
 
-    const std::set<std::string>
-    flatbuf_bytearrayvector_to_stringlist(const flatbuffers::Vector<flatbuffers::Offset<ByteArray>> *fbvec);
+    std::string_view builder_to_string_view(const flatbuffers::FlatBufferBuilder &builder);
 
-    const std::unordered_map<std::string, const std::string>
-    flatbuf_pairvector_to_stringmap(const flatbuffers::Vector<flatbuffers::Offset<BytesKeyValuePair>> *fbvec);
-
-    //---Conversion helpers from std data types to flatbuffers data types---//
+    //---std to Flatbuf---//
 
     const flatbuffers::Offset<flatbuffers::Vector<uint8_t>>
-    sv_to_flatbuff_bytes(flatbuffers::FlatBufferBuilder &builder, std::string_view sv);
+    sv_to_flatbuf_bytes(flatbuffers::FlatBufferBuilder &builder, std::string_view sv);
 
     const flatbuffers::Offset<flatbuffers::String>
-    sv_to_flatbuff_str(flatbuffers::FlatBufferBuilder &builder, std::string_view sv);
+    sv_to_flatbuf_str(flatbuffers::FlatBufferBuilder &builder, std::string_view sv);
 
     const flatbuffers::Offset<flatbuffers::Vector<uint8_t>>
-    hash_to_flatbuff_bytes(flatbuffers::FlatBufferBuilder &builder, util::h32 hash);
-
-    const flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<ByteArray>>>
-    stringlist_to_flatbuf_bytearrayvector(flatbuffers::FlatBufferBuilder &builder, const std::set<std::string> &set);
-
-    const flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<BytesKeyValuePair>>>
-    stringmap_to_flatbuf_bytepairvector(flatbuffers::FlatBufferBuilder &builder, const std::unordered_map<std::string, const std::string> &map);
+    hash_to_flatbuf_bytes(flatbuffers::FlatBufferBuilder &builder, util::h32 hash);
 
 } // namespace msg::fbuf
 

src/msg/fbuf/common_schema.fbs

@@ -1,11 +0,0 @@
-//IDL file for common types.
-namespace msg.fbuf;
-
-table BytesKeyValuePair { //A key, value pair of byte[].
-    key:[ubyte];
-    value:[ubyte];
-}
-
-table ByteArray { //To represent list of byte arrays
-    array:[ubyte];
-}

src/msg/fbuf/common_schema_generated.h

@@ -1,146 +0,0 @@
-// automatically generated by the FlatBuffers compiler, do not modify
-
-
-#ifndef FLATBUFFERS_GENERATED_COMMONSCHEMA_MSG_FBUF_H_
-#define FLATBUFFERS_GENERATED_COMMONSCHEMA_MSG_FBUF_H_
-
-#include "flatbuffers/flatbuffers.h"
-
-namespace msg {
-namespace fbuf {
-
-struct BytesKeyValuePair;
-struct BytesKeyValuePairBuilder;
-
-struct ByteArray;
-struct ByteArrayBuilder;
-
-struct BytesKeyValuePair FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
-  typedef BytesKeyValuePairBuilder Builder;
-  enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
-    VT_KEY = 4,
-    VT_VALUE = 6
-  };
-  const flatbuffers::Vector<uint8_t> *key() const {
-    return GetPointer<const flatbuffers::Vector<uint8_t> *>(VT_KEY);
-  }
-  flatbuffers::Vector<uint8_t> *mutable_key() {
-    return GetPointer<flatbuffers::Vector<uint8_t> *>(VT_KEY);
-  }
-  const flatbuffers::Vector<uint8_t> *value() const {
-    return GetPointer<const flatbuffers::Vector<uint8_t> *>(VT_VALUE);
-  }
-  flatbuffers::Vector<uint8_t> *mutable_value() {
-    return GetPointer<flatbuffers::Vector<uint8_t> *>(VT_VALUE);
-  }
-  bool Verify(flatbuffers::Verifier &verifier) const {
-    return VerifyTableStart(verifier) &&
-           VerifyOffset(verifier, VT_KEY) &&
-           verifier.VerifyVector(key()) &&
-           VerifyOffset(verifier, VT_VALUE) &&
-           verifier.VerifyVector(value()) &&
-           verifier.EndTable();
-  }
-};
-
-struct BytesKeyValuePairBuilder {
-  typedef BytesKeyValuePair Table;
-  flatbuffers::FlatBufferBuilder &fbb_;
-  flatbuffers::uoffset_t start_;
-  void add_key(flatbuffers::Offset<flatbuffers::Vector<uint8_t>> key) {
-    fbb_.AddOffset(BytesKeyValuePair::VT_KEY, key);
-  }
-  void add_value(flatbuffers::Offset<flatbuffers::Vector<uint8_t>> value) {
-    fbb_.AddOffset(BytesKeyValuePair::VT_VALUE, value);
-  }
-  explicit BytesKeyValuePairBuilder(flatbuffers::FlatBufferBuilder &_fbb)
-        : fbb_(_fbb) {
-    start_ = fbb_.StartTable();
-  }
-  flatbuffers::Offset<BytesKeyValuePair> Finish() {
-    const auto end = fbb_.EndTable(start_);
-    auto o = flatbuffers::Offset<BytesKeyValuePair>(end);
-    return o;
-  }
-};
-
-inline flatbuffers::Offset<BytesKeyValuePair> CreateBytesKeyValuePair(
-    flatbuffers::FlatBufferBuilder &_fbb,
-    flatbuffers::Offset<flatbuffers::Vector<uint8_t>> key = 0,
-    flatbuffers::Offset<flatbuffers::Vector<uint8_t>> value = 0) {
-  BytesKeyValuePairBuilder builder_(_fbb);
-  builder_.add_value(value);
-  builder_.add_key(key);
-  return builder_.Finish();
-}
-
-inline flatbuffers::Offset<BytesKeyValuePair> CreateBytesKeyValuePairDirect(
-    flatbuffers::FlatBufferBuilder &_fbb,
-    const std::vector<uint8_t> *key = nullptr,
-    const std::vector<uint8_t> *value = nullptr) {
-  auto key__ = key ? _fbb.CreateVector<uint8_t>(*key) : 0;
-  auto value__ = value ? _fbb.CreateVector<uint8_t>(*value) : 0;
-  return msg::fbuf::CreateBytesKeyValuePair(
-      _fbb,
-      key__,
-      value__);
-}
-
-struct ByteArray FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
-  typedef ByteArrayBuilder Builder;
-  enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
-    VT_ARRAY = 4
-  };
-  const flatbuffers::Vector<uint8_t> *array() const {
-    return GetPointer<const flatbuffers::Vector<uint8_t> *>(VT_ARRAY);
-  }
-  flatbuffers::Vector<uint8_t> *mutable_array() {
-    return GetPointer<flatbuffers::Vector<uint8_t> *>(VT_ARRAY);
-  }
-  bool Verify(flatbuffers::Verifier &verifier) const {
-    return VerifyTableStart(verifier) &&
-           VerifyOffset(verifier, VT_ARRAY) &&
-           verifier.VerifyVector(array()) &&
-           verifier.EndTable();
-  }
-};
-
-struct ByteArrayBuilder {
-  typedef ByteArray Table;
-  flatbuffers::FlatBufferBuilder &fbb_;
-  flatbuffers::uoffset_t start_;
-  void add_array(flatbuffers::Offset<flatbuffers::Vector<uint8_t>> array) {
-    fbb_.AddOffset(ByteArray::VT_ARRAY, array);
-  }
-  explicit ByteArrayBuilder(flatbuffers::FlatBufferBuilder &_fbb)
-        : fbb_(_fbb) {
-    start_ = fbb_.StartTable();
-  }
-  flatbuffers::Offset<ByteArray> Finish() {
-    const auto end = fbb_.EndTable(start_);
-    auto o = flatbuffers::Offset<ByteArray>(end);
-    return o;
-  }
-};
-
-inline flatbuffers::Offset<ByteArray> CreateByteArray(
-    flatbuffers::FlatBufferBuilder &_fbb,
-    flatbuffers::Offset<flatbuffers::Vector<uint8_t>> array = 0) {
-  ByteArrayBuilder builder_(_fbb);
-  builder_.add_array(array);
-  return builder_.Finish();
-}
-
-inline flatbuffers::Offset<ByteArray> CreateByteArrayDirect(
-    flatbuffers::FlatBufferBuilder &_fbb,
-    const std::vector<uint8_t> *array = nullptr) {
-  auto array__ = array ? _fbb.CreateVector<uint8_t>(*array) : 0;
-  return msg::fbuf::CreateByteArray(
-      _fbb,
-      array__);
-}
-
-}  // namespace fbuf
-}  // namespace msg
-
-#endif  // FLATBUFFERS_GENERATED_COMMONSCHEMA_MSG_FBUF_H_

src/msg/fbuf/flatbuf_hasher.hpp

@@ -0,0 +1,89 @@
+#ifndef _HP_MSG_FBUF_FBUF_HASHER
+#define _HP_MSG_FBUF_FBUF_HASHER
+
+#include "../../pchheader.hpp"
+#include "../../util/util.hpp"
+#include "p2pmsg_generated.h"
+
+namespace msg::fbuf::p2pmsg
+{
+    class flatbuf_hasher
+    {
+    private:
+        blake3_hasher hasher;
+
+    public:
+        flatbuf_hasher()
+        {
+            blake3_hasher_init(&hasher);
+        }
+
+        void add(const uint8_t i)
+        {
+            blake3_hasher_update(&hasher, &i, sizeof(uint8_t));
+        }
+
+        void add(const uint32_t i)
+        {
+            uint8_t bytes[4];
+            util::uint32_to_bytes(bytes, i);
+            blake3_hasher_update(&hasher, bytes, sizeof(bytes));
+        }
+
+        void add(const uint64_t i)
+        {
+            uint8_t bytes[8];
+            util::uint64_to_bytes(bytes, i);
+            blake3_hasher_update(&hasher, bytes, sizeof(bytes));
+        }
+
+        void add(std::string_view sv)
+        {
+            blake3_hasher_update(&hasher, sv.data(), sv.size());
+        }
+
+        void add(const std::set<std::string> &sl)
+        {
+            for(const std::string &s : sl)
+                add(s);
+        }
+
+        void add(const flatbuffers::Vector<uint8_t> *v)
+        {
+            blake3_hasher_update(&hasher, v->data(), v->size());
+        }
+
+        void add(const flatbuffers::Vector<flatbuffers::Offset<ByteArray>> *v)
+        {
+            for (const auto el : *v)
+                add(el->array());
+        }
+
+        void add(const util::h32 &h)
+        {
+            add(h.to_string_view());
+        }
+
+        void add(const p2p::sequence_hash &h)
+        {
+            add(h.seq_no);
+            add(h.hash);
+        }
+
+        void add(const SequenceHash *h)
+        {
+            add(h->seq_no());
+            add(h->hash());
+        }
+
+        const std::string hash()
+        {
+            std::string hash;
+            hash.resize(BLAKE3_OUT_LEN);
+            blake3_hasher_finalize(&hasher, reinterpret_cast<uint8_t *>(hash.data()), hash.size());
+            return hash;
+        }
+    };
+}
+
+#endif

src/msg/fbuf/ledger_helpers.cpp

@@ -1,6 +1,5 @@
 #include "../../pchheader.hpp"
 #include "../../p2p/p2p.hpp"
-#include "ledger_schema_generated.h"
 #include "ledger_blob_schema_generated.h"
 #include "common_helpers.hpp"
 #include "ledger_helpers.hpp"
@@ -24,11 +23,11 @@ namespace msg::fbuf::ledgermsg
             inputs.reserve(value.size());
 
             for (const auto &input : value)
-                inputs.push_back(ledgermsg::CreateRawInput(builder, sv_to_flatbuff_bytes(builder, input)));
-            
+                inputs.push_back(ledgermsg::CreateRawInput(builder, sv_to_flatbuf_bytes(builder, input)));
+
             raw_inputs.push_back(ledgermsg::CreateRawInputCollection(
                 builder,
-                sv_to_flatbuff_bytes(builder, key),
+                sv_to_flatbuf_bytes(builder, key),
                 builder.CreateVector(inputs)));
         }
 
@@ -38,18 +37,18 @@ namespace msg::fbuf::ledgermsg
             outputs.reserve(value.size());
 
             for (const auto &output : value)
-                outputs.push_back(ledgermsg::CreateRawOutput(builder, sv_to_flatbuff_bytes(builder, output)));
-            
+                outputs.push_back(ledgermsg::CreateRawOutput(builder, sv_to_flatbuf_bytes(builder, output)));
+
             raw_outputs.push_back(ledgermsg::CreateRawOutputCollection(
                 builder,
-                sv_to_flatbuff_bytes(builder, key),
+                sv_to_flatbuf_bytes(builder, key),
                 builder.CreateVector(outputs)));
         }
 
         flatbuffers::Offset<ledgermsg::LedgerBlob> blob =
             ledgermsg::CreateLedgerBlob(
                 builder,
-                hash_to_flatbuff_bytes(builder, ledger_blob.ledger_hash),
+                hash_to_flatbuf_bytes(builder, ledger_blob.ledger_hash),
                 builder.CreateVector(raw_inputs),
                 builder.CreateVector(raw_outputs));
 

src/msg/fbuf/ledger_helpers.hpp

@@ -4,7 +4,6 @@
 #include "../../pchheader.hpp"
 #include "../../p2p/p2p.hpp"
 #include "../../ledger/ledger.hpp"
-#include "ledger_schema_generated.h"
 
 namespace msg::fbuf::ledgermsg
 {

src/msg/fbuf/ledger_schema.fbs

@@ -1,17 +0,0 @@
-include "common_schema.fbs";
-
-namespace msg.fbuf.ledgermsg;
-
-table LedgerBlock {
-    version:string;
-    seq_no:uint64;
-    time:uint64;
-    lcl:[ubyte];
-    state_hash:[ubyte];
-    patch_hash:[ubyte];
-    users: [ByteArray];
-    inputs: [ByteArray]; 
-    output: [ubyte];
-}
-
-root_type LedgerBlock;

src/msg/fbuf/ledger_schema_generated.h

@@ -1,244 +0,0 @@
-// automatically generated by the FlatBuffers compiler, do not modify
-
-
-#ifndef FLATBUFFERS_GENERATED_LEDGERSCHEMA_MSG_FBUF_LEDGERMSG_H_
-#define FLATBUFFERS_GENERATED_LEDGERSCHEMA_MSG_FBUF_LEDGERMSG_H_
-
-#include "flatbuffers/flatbuffers.h"
-
-#include "common_schema_generated.h"
-
-namespace msg {
-namespace fbuf {
-namespace ledgermsg {
-
-struct LedgerBlock;
-struct LedgerBlockBuilder;
-
-struct LedgerBlock FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
-  typedef LedgerBlockBuilder Builder;
-  enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
-    VT_VERSION = 4,
-    VT_SEQ_NO = 6,
-    VT_TIME = 8,
-    VT_LCL = 10,
-    VT_STATE_HASH = 12,
-    VT_PATCH_HASH = 14,
-    VT_USERS = 16,
-    VT_INPUTS = 18,
-    VT_OUTPUT = 20
-  };
-  const flatbuffers::String *version() const {
-    return GetPointer<const flatbuffers::String *>(VT_VERSION);
-  }
-  flatbuffers::String *mutable_version() {
-    return GetPointer<flatbuffers::String *>(VT_VERSION);
-  }
-  uint64_t seq_no() const {
-    return GetField<uint64_t>(VT_SEQ_NO, 0);
-  }
-  bool mutate_seq_no(uint64_t _seq_no) {
-    return SetField<uint64_t>(VT_SEQ_NO, _seq_no, 0);
-  }
-  uint64_t time() const {
-    return GetField<uint64_t>(VT_TIME, 0);
-  }
-  bool mutate_time(uint64_t _time) {
-    return SetField<uint64_t>(VT_TIME, _time, 0);
-  }
-  const flatbuffers::Vector<uint8_t> *lcl() const {
-    return GetPointer<const flatbuffers::Vector<uint8_t> *>(VT_LCL);
-  }
-  flatbuffers::Vector<uint8_t> *mutable_lcl() {
-    return GetPointer<flatbuffers::Vector<uint8_t> *>(VT_LCL);
-  }
-  const flatbuffers::Vector<uint8_t> *state_hash() const {
-    return GetPointer<const flatbuffers::Vector<uint8_t> *>(VT_STATE_HASH);
-  }
-  flatbuffers::Vector<uint8_t> *mutable_state_hash() {
-    return GetPointer<flatbuffers::Vector<uint8_t> *>(VT_STATE_HASH);
-  }
-  const flatbuffers::Vector<uint8_t> *patch_hash() const {
-    return GetPointer<const flatbuffers::Vector<uint8_t> *>(VT_PATCH_HASH);
-  }
-  flatbuffers::Vector<uint8_t> *mutable_patch_hash() {
-    return GetPointer<flatbuffers::Vector<uint8_t> *>(VT_PATCH_HASH);
-  }
-  const flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ByteArray>> *users() const {
-    return GetPointer<const flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ByteArray>> *>(VT_USERS);
-  }
-  flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ByteArray>> *mutable_users() {
-    return GetPointer<flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ByteArray>> *>(VT_USERS);
-  }
-  const flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ByteArray>> *inputs() const {
-    return GetPointer<const flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ByteArray>> *>(VT_INPUTS);
-  }
-  flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ByteArray>> *mutable_inputs() {
-    return GetPointer<flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ByteArray>> *>(VT_INPUTS);
-  }
-  const flatbuffers::Vector<uint8_t> *output() const {
-    return GetPointer<const flatbuffers::Vector<uint8_t> *>(VT_OUTPUT);
-  }
-  flatbuffers::Vector<uint8_t> *mutable_output() {
-    return GetPointer<flatbuffers::Vector<uint8_t> *>(VT_OUTPUT);
-  }
-  bool Verify(flatbuffers::Verifier &verifier) const {
-    return VerifyTableStart(verifier) &&
-           VerifyOffset(verifier, VT_VERSION) &&
-           verifier.VerifyString(version()) &&
-           VerifyField<uint64_t>(verifier, VT_SEQ_NO) &&
-           VerifyField<uint64_t>(verifier, VT_TIME) &&
-           VerifyOffset(verifier, VT_LCL) &&
-           verifier.VerifyVector(lcl()) &&
-           VerifyOffset(verifier, VT_STATE_HASH) &&
-           verifier.VerifyVector(state_hash()) &&
-           VerifyOffset(verifier, VT_PATCH_HASH) &&
-           verifier.VerifyVector(patch_hash()) &&
-           VerifyOffset(verifier, VT_USERS) &&
-           verifier.VerifyVector(users()) &&
-           verifier.VerifyVectorOfTables(users()) &&
-           VerifyOffset(verifier, VT_INPUTS) &&
-           verifier.VerifyVector(inputs()) &&
-           verifier.VerifyVectorOfTables(inputs()) &&
-           VerifyOffset(verifier, VT_OUTPUT) &&
-           verifier.VerifyVector(output()) &&
-           verifier.EndTable();
-  }
-};
-
-struct LedgerBlockBuilder {
-  typedef LedgerBlock Table;
-  flatbuffers::FlatBufferBuilder &fbb_;
-  flatbuffers::uoffset_t start_;
-  void add_version(flatbuffers::Offset<flatbuffers::String> version) {
-    fbb_.AddOffset(LedgerBlock::VT_VERSION, version);
-  }
-  void add_seq_no(uint64_t seq_no) {
-    fbb_.AddElement<uint64_t>(LedgerBlock::VT_SEQ_NO, seq_no, 0);
-  }
-  void add_time(uint64_t time) {
-    fbb_.AddElement<uint64_t>(LedgerBlock::VT_TIME, time, 0);
-  }
-  void add_lcl(flatbuffers::Offset<flatbuffers::Vector<uint8_t>> lcl) {
-    fbb_.AddOffset(LedgerBlock::VT_LCL, lcl);
-  }
-  void add_state_hash(flatbuffers::Offset<flatbuffers::Vector<uint8_t>> state_hash) {
-    fbb_.AddOffset(LedgerBlock::VT_STATE_HASH, state_hash);
-  }
-  void add_patch_hash(flatbuffers::Offset<flatbuffers::Vector<uint8_t>> patch_hash) {
-    fbb_.AddOffset(LedgerBlock::VT_PATCH_HASH, patch_hash);
-  }
-  void add_users(flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ByteArray>>> users) {
-    fbb_.AddOffset(LedgerBlock::VT_USERS, users);
-  }
-  void add_inputs(flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ByteArray>>> inputs) {
-    fbb_.AddOffset(LedgerBlock::VT_INPUTS, inputs);
-  }
-  void add_output(flatbuffers::Offset<flatbuffers::Vector<uint8_t>> output) {
-    fbb_.AddOffset(LedgerBlock::VT_OUTPUT, output);
-  }
-  explicit LedgerBlockBuilder(flatbuffers::FlatBufferBuilder &_fbb)
-        : fbb_(_fbb) {
-    start_ = fbb_.StartTable();
-  }
-  flatbuffers::Offset<LedgerBlock> Finish() {
-    const auto end = fbb_.EndTable(start_);
-    auto o = flatbuffers::Offset<LedgerBlock>(end);
-    return o;
-  }
-};
-
-inline flatbuffers::Offset<LedgerBlock> CreateLedgerBlock(
-    flatbuffers::FlatBufferBuilder &_fbb,
-    flatbuffers::Offset<flatbuffers::String> version = 0,
-    uint64_t seq_no = 0,
-    uint64_t time = 0,
-    flatbuffers::Offset<flatbuffers::Vector<uint8_t>> lcl = 0,
-    flatbuffers::Offset<flatbuffers::Vector<uint8_t>> state_hash = 0,
-    flatbuffers::Offset<flatbuffers::Vector<uint8_t>> patch_hash = 0,
-    flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ByteArray>>> users = 0,
-    flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<msg::fbuf::ByteArray>>> inputs = 0,
-    flatbuffers::Offset<flatbuffers::Vector<uint8_t>> output = 0) {
-  LedgerBlockBuilder builder_(_fbb);
-  builder_.add_time(time);
-  builder_.add_seq_no(seq_no);
-  builder_.add_output(output);
-  builder_.add_inputs(inputs);
-  builder_.add_users(users);
-  builder_.add_patch_hash(patch_hash);
-  builder_.add_state_hash(state_hash);
-  builder_.add_lcl(lcl);
-  builder_.add_version(version);
-  return builder_.Finish();
-}
-
-inline flatbuffers::Offset<LedgerBlock> CreateLedgerBlockDirect(
-    flatbuffers::FlatBufferBuilder &_fbb,
-    const char *version = nullptr,
-    uint64_t seq_no = 0,
-    uint64_t time = 0,
-    const std::vector<uint8_t> *lcl = nullptr,
-    const std::vector<uint8_t> *state_hash = nullptr,
-    const std::vector<uint8_t> *patch_hash = nullptr,
-    const std::vector<flatbuffers::Offset<msg::fbuf::ByteArray>> *users = nullptr,
-    const std::vector<flatbuffers::Offset<msg::fbuf::ByteArray>> *inputs = nullptr,
-    const std::vector<uint8_t> *output = nullptr) {
-  auto version__ = version ? _fbb.CreateString(version) : 0;
-  auto lcl__ = lcl ? _fbb.CreateVector<uint8_t>(*lcl) : 0;
-  auto state_hash__ = state_hash ? _fbb.CreateVector<uint8_t>(*state_hash) : 0;
-  auto patch_hash__ = patch_hash ? _fbb.CreateVector<uint8_t>(*patch_hash) : 0;
-  auto users__ = users ? _fbb.CreateVector<flatbuffers::Offset<msg::fbuf::ByteArray>>(*users) : 0;
-  auto inputs__ = inputs ? _fbb.CreateVector<flatbuffers::Offset<msg::fbuf::ByteArray>>(*inputs) : 0;
-  auto output__ = output ? _fbb.CreateVector<uint8_t>(*output) : 0;
-  return msg::fbuf::ledgermsg::CreateLedgerBlock(
-      _fbb,
-      version__,
-      seq_no,
-      time,
-      lcl__,
-      state_hash__,
-      patch_hash__,
-      users__,
-      inputs__,
-      output__);
-}
-
-inline const msg::fbuf::ledgermsg::LedgerBlock *GetLedgerBlock(const void *buf) {
-  return flatbuffers::GetRoot<msg::fbuf::ledgermsg::LedgerBlock>(buf);
-}
-
-inline const msg::fbuf::ledgermsg::LedgerBlock *GetSizePrefixedLedgerBlock(const void *buf) {
-  return flatbuffers::GetSizePrefixedRoot<msg::fbuf::ledgermsg::LedgerBlock>(buf);
-}
-
-inline LedgerBlock *GetMutableLedgerBlock(void *buf) {
-  return flatbuffers::GetMutableRoot<LedgerBlock>(buf);
-}
-
-inline bool VerifyLedgerBlockBuffer(
-    flatbuffers::Verifier &verifier) {
-  return verifier.VerifyBuffer<msg::fbuf::ledgermsg::LedgerBlock>(nullptr);
-}
-
-inline bool VerifySizePrefixedLedgerBlockBuffer(
-    flatbuffers::Verifier &verifier) {
-  return verifier.VerifySizePrefixedBuffer<msg::fbuf::ledgermsg::LedgerBlock>(nullptr);
-}
-
-inline void FinishLedgerBlockBuffer(
-    flatbuffers::FlatBufferBuilder &fbb,
-    flatbuffers::Offset<msg::fbuf::ledgermsg::LedgerBlock> root) {
-  fbb.Finish(root);
-}
-
-inline void FinishSizePrefixedLedgerBlockBuffer(
-    flatbuffers::FlatBufferBuilder &fbb,
-    flatbuffers::Offset<msg::fbuf::ledgermsg::LedgerBlock> root) {
-  fbb.FinishSizePrefixed(root);
-}
-
-}  // namespace ledgermsg
-}  // namespace fbuf
-}  // namespace msg
-
-#endif  // FLATBUFFERS_GENERATED_LEDGERSCHEMA_MSG_FBUF_LEDGERMSG_H_

src/msg/fbuf/p2pmsg.fbs

@@ -0,0 +1,154 @@
+// IDL file for p2p message definitions.
+// flatc -o src/msg/fbuf/ --gen-mutable --cpp src/msg/fbuf/p2pmsg.fbs
+
+namespace msg.fbuf.p2pmsg;
+
+union P2PMsgContent {
+    PeerChallengeMsg,
+    PeerChallengeResponseMsg,
+    NonUnlProposalMsg,
+    ProposalMsg,
+    NplMsg,
+    HpfsRequestMsg,
+    HpfsResponseMsg,
+    PeerRequirementAnnouncementMsg,
+    PeerCapacityAnnouncementMsg,
+    PeerListRequestMsg,
+    PeerListResponseMsg
+}
+
+table P2PMsg {
+    hp_version:string;
+    created_on:uint64;
+    content:P2PMsgContent;
+}
+
+table PeerChallengeMsg {
+    contract_id:string;
+    roundtime:uint32;
+    challenge:[ubyte];
+}
+
+table PeerChallengeResponseMsg {
+    challenge:[ubyte];
+    pubkey:[ubyte];
+    sig:[ubyte];
+}
+
+table UserInput {
+    input_container:[ubyte];
+    sig:[ubyte];
+    protocol:uint8;
+}
+
+table UserInputGroup {
+    pubkey:[ubyte];
+    messages:[UserInput];
+}
+
+table NonUnlProposalMsg {
+    user_inputs:[UserInputGroup];
+}
+
+table ProposalMsg {
+    pubkey:[ubyte]; // Sender pubkey.
+    sig:[ubyte]; // Signature of the field data.
+    stage:uint8;
+    time:uint64;
+    roundtime:uint32;
+    nonce: [ubyte];
+    users:[ByteArray];
+    input_hashes:[ByteArray];
+    output_hash:[ubyte];
+    output_sig:[ubyte];
+    state_hash: [ubyte];
+    patch_hash: [ubyte];
+    last_primary_shard_id:SequenceHash;
+    last_blob_shard_id: SequenceHash;
+
+    // Make sure to update signature generation/verification whenever these fields are changed.
+}
+
+table NplMsg {
+    pubkey:[ubyte]; // Sender pubkey.
+    sig:[ubyte]; // Signature of the field data.
+    data:[ubyte];
+    lcl_id:SequenceHash;
+
+    // Make sure to update signature generation/verification whenever these fields are changed.
+}
+
+table HpfsRequestMsg {
+    mount_id: uint32;
+    parent_path:string;
+    is_file:bool;
+    block_id:int32;
+    expected_hash:[ubyte];
+}
+
+union HpfsResponse {
+    HpfsFileHashMapResponse,
+    HpfsBlockResponse,
+    HpfsFsEntryResponse
+}
+
+table HpfsResponseMsg{
+    hash:[ubyte];
+    path: string;
+    mount_id: uint32;
+    content:HpfsResponse;
+}
+
+table HpfsFsEntryResponse{
+    entries: [HpfsFSHashEntry];
+}
+
+table HpfsFileHashMapResponse{
+    file_length:uint64;
+    hash_map:[ubyte];
+}
+
+table HpfsBlockResponse{
+    block_id:uint32;
+    data: [ubyte];
+}
+
+table HpfsFSHashEntry{
+    name: string;
+    is_file: bool;
+    hash: [ubyte];
+}
+
+table PeerRequirementAnnouncementMsg{
+    need_consensus_msg_forwarding: bool;
+}
+
+table PeerCapacityAnnouncementMsg{
+    available_capacity:int16;
+    timestamp:uint64;
+}
+
+table PeerListRequestMsg{
+}
+
+table PeerListResponseMsg{
+    peer_list: [PeerProperties];
+}
+
+table PeerProperties {
+    host_address:string;
+    port:uint16;
+    available_capacity:int16;
+    timestamp:uint64;
+}
+
+table SequenceHash {
+    seq_no: uint64;
+    hash: [ubyte];
+}
+
+table ByteArray { // To help represent list of byte arrays
+    array:[ubyte];
+}
+
+root_type P2PMsg; //root type for all messages

src/msg/fbuf/p2pmsg_container.fbs

@@ -1,18 +0,0 @@
-//IDL file for p2p message container schema.
-/*Message content need to be serialised and parsed into container, we need two root types in flatbuffs to generat
- needed api methods. Since Flatbuff only support one rrot at each IDL files, we had to seperate message schema to 2 files.
-*/
-include "p2pmsg_content.fbs";
-namespace msg.fbuf.p2pmsg;
-
-table Container { //root type for message
-    version:uint16;
-    timestamp:uint64;
-    pubkey:[ubyte];
-    lcl:[ubyte];
-    last_primary_shard_id:Sequence_Hash;
-    signature:[ubyte]; // signature of the message content
-    content:[ubyte]; // message content: byte array of proposal,npl,etc
-}
-
-root_type Container;

src/msg/fbuf/p2pmsg_container_generated.h

@@ -1,208 +0,0 @@
-// automatically generated by the FlatBuffers compiler, do not modify
-
-
-#ifndef FLATBUFFERS_GENERATED_P2PMSGCONTAINER_MSG_FBUF_P2PMSG_H_
-#define FLATBUFFERS_GENERATED_P2PMSGCONTAINER_MSG_FBUF_P2PMSG_H_
-
-#include "flatbuffers/flatbuffers.h"
-
-#include "common_schema_generated.h"
-#include "p2pmsg_content_generated.h"
-
-namespace msg {
-namespace fbuf {
-namespace p2pmsg {
-
-struct Container;
-struct ContainerBuilder;
-
-struct Container FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
-  typedef ContainerBuilder Builder;
-  enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
-    VT_VERSION = 4,
-    VT_TIMESTAMP = 6,
-    VT_PUBKEY = 8,
-    VT_LCL = 10,
-    VT_LAST_PRIMARY_SHARD_ID = 12,
-    VT_SIGNATURE = 14,
-    VT_CONTENT = 16
-  };
-  uint16_t version() const {
-    return GetField<uint16_t>(VT_VERSION, 0);
-  }
-  bool mutate_version(uint16_t _version) {
-    return SetField<uint16_t>(VT_VERSION, _version, 0);
-  }
-  uint64_t timestamp() const {
-    return GetField<uint64_t>(VT_TIMESTAMP, 0);
-  }
-  bool mutate_timestamp(uint64_t _timestamp) {
-    return SetField<uint64_t>(VT_TIMESTAMP, _timestamp, 0);
-  }
-  const flatbuffers::Vector<uint8_t> *pubkey() const {
-    return GetPointer<const flatbuffers::Vector<uint8_t> *>(VT_PUBKEY);
-  }
-  flatbuffers::Vector<uint8_t> *mutable_pubkey() {
-    return GetPointer<flatbuffers::Vector<uint8_t> *>(VT_PUBKEY);
-  }
-  const flatbuffers::Vector<uint8_t> *lcl() const {
-    return GetPointer<const flatbuffers::Vector<uint8_t> *>(VT_LCL);
-  }
-  flatbuffers::Vector<uint8_t> *mutable_lcl() {
-    return GetPointer<flatbuffers::Vector<uint8_t> *>(VT_LCL);
-  }
-  const msg::fbuf::p2pmsg::Sequence_Hash *last_primary_shard_id() const {
-    return GetPointer<const msg::fbuf::p2pmsg::Sequence_Hash *>(VT_LAST_PRIMARY_SHARD_ID);
-  }
-  msg::fbuf::p2pmsg::Sequence_Hash *mutable_last_primary_shard_id() {
-    return GetPointer<msg::fbuf::p2pmsg::Sequence_Hash *>(VT_LAST_PRIMARY_SHARD_ID);
-  }
-  const flatbuffers::Vector<uint8_t> *signature() const {
-    return GetPointer<const flatbuffers::Vector<uint8_t> *>(VT_SIGNATURE);
-  }
-  flatbuffers::Vector<uint8_t> *mutable_signature() {
-    return GetPointer<flatbuffers::Vector<uint8_t> *>(VT_SIGNATURE);
-  }
-  const flatbuffers::Vector<uint8_t> *content() const {
-    return GetPointer<const flatbuffers::Vector<uint8_t> *>(VT_CONTENT);
-  }
-  flatbuffers::Vector<uint8_t> *mutable_content() {
-    return GetPointer<flatbuffers::Vector<uint8_t> *>(VT_CONTENT);
-  }
-  bool Verify(flatbuffers::Verifier &verifier) const {
-    return VerifyTableStart(verifier) &&
-           VerifyField<uint16_t>(verifier, VT_VERSION) &&
-           VerifyField<uint64_t>(verifier, VT_TIMESTAMP) &&
-           VerifyOffset(verifier, VT_PUBKEY) &&
-           verifier.VerifyVector(pubkey()) &&
-           VerifyOffset(verifier, VT_LCL) &&
-           verifier.VerifyVector(lcl()) &&
-           VerifyOffset(verifier, VT_LAST_PRIMARY_SHARD_ID) &&
-           verifier.VerifyTable(last_primary_shard_id()) &&
-           VerifyOffset(verifier, VT_SIGNATURE) &&
-           verifier.VerifyVector(signature()) &&
-           VerifyOffset(verifier, VT_CONTENT) &&
-           verifier.VerifyVector(content()) &&
-           verifier.EndTable();
-  }
-};
-
-struct ContainerBuilder {
-  typedef Container Table;
-  flatbuffers::FlatBufferBuilder &fbb_;
-  flatbuffers::uoffset_t start_;
-  void add_version(uint16_t version) {
-    fbb_.AddElement<uint16_t>(Container::VT_VERSION, version, 0);
-  }
-  void add_timestamp(uint64_t timestamp) {
-    fbb_.AddElement<uint64_t>(Container::VT_TIMESTAMP, timestamp, 0);
-  }
-  void add_pubkey(flatbuffers::Offset<flatbuffers::Vector<uint8_t>> pubkey) {
-    fbb_.AddOffset(Container::VT_PUBKEY, pubkey);
-  }
-  void add_lcl(flatbuffers::Offset<flatbuffers::Vector<uint8_t>> lcl) {
-    fbb_.AddOffset(Container::VT_LCL, lcl);
-  }
-  void add_last_primary_shard_id(flatbuffers::Offset<msg::fbuf::p2pmsg::Sequence_Hash> last_primary_shard_id) {
-    fbb_.AddOffset(Container::VT_LAST_PRIMARY_SHARD_ID, last_primary_shard_id);
-  }
-  void add_signature(flatbuffers::Offset<flatbuffers::Vector<uint8_t>> signature) {
-    fbb_.AddOffset(Container::VT_SIGNATURE, signature);
-  }
-  void add_content(flatbuffers::Offset<flatbuffers::Vector<uint8_t>> content) {
-    fbb_.AddOffset(Container::VT_CONTENT, content);
-  }
-  explicit ContainerBuilder(flatbuffers::FlatBufferBuilder &_fbb)
-        : fbb_(_fbb) {
-    start_ = fbb_.StartTable();
-  }
-  flatbuffers::Offset<Container> Finish() {
-    const auto end = fbb_.EndTable(start_);
-    auto o = flatbuffers::Offset<Container>(end);
-    return o;
-  }
-};
-
-inline flatbuffers::Offset<Container> CreateContainer(
-    flatbuffers::FlatBufferBuilder &_fbb,
-    uint16_t version = 0,
-    uint64_t timestamp = 0,
-    flatbuffers::Offset<flatbuffers::Vector<uint8_t>> pubkey = 0,
-    flatbuffers::Offset<flatbuffers::Vector<uint8_t>> lcl = 0,
-    flatbuffers::Offset<msg::fbuf::p2pmsg::Sequence_Hash> last_primary_shard_id = 0,
-    flatbuffers::Offset<flatbuffers::Vector<uint8_t>> signature = 0,
-    flatbuffers::Offset<flatbuffers::Vector<uint8_t>> content = 0) {
-  ContainerBuilder builder_(_fbb);
-  builder_.add_timestamp(timestamp);
-  builder_.add_content(content);
-  builder_.add_signature(signature);
-  builder_.add_last_primary_shard_id(last_primary_shard_id);
-  builder_.add_lcl(lcl);
-  builder_.add_pubkey(pubkey);
-  builder_.add_version(version);
-  return builder_.Finish();
-}
-
-inline flatbuffers::Offset<Container> CreateContainerDirect(
-    flatbuffers::FlatBufferBuilder &_fbb,
-    uint16_t version = 0,
-    uint64_t timestamp = 0,
-    const std::vector<uint8_t> *pubkey = nullptr,
-    const std::vector<uint8_t> *lcl = nullptr,
-    flatbuffers::Offset<msg::fbuf::p2pmsg::Sequence_Hash> last_primary_shard_id = 0,
-    const std::vector<uint8_t> *signature = nullptr,
-    const std::vector<uint8_t> *content = nullptr) {
-  auto pubkey__ = pubkey ? _fbb.CreateVector<uint8_t>(*pubkey) : 0;
-  auto lcl__ = lcl ? _fbb.CreateVector<uint8_t>(*lcl) : 0;
-  auto signature__ = signature ? _fbb.CreateVector<uint8_t>(*signature) : 0;
-  auto content__ = content ? _fbb.CreateVector<uint8_t>(*content) : 0;
-  return msg::fbuf::p2pmsg::CreateContainer(
-      _fbb,
-      version,
-      timestamp,
-      pubkey__,
-      lcl__,
-      last_primary_shard_id,
-      signature__,
-      content__);
-}
-
-inline const msg::fbuf::p2pmsg::Container *GetContainer(const void *buf) {
-  return flatbuffers::GetRoot<msg::fbuf::p2pmsg::Container>(buf);
-}
-
-inline const msg::fbuf::p2pmsg::Container *GetSizePrefixedContainer(const void *buf) {
-  return flatbuffers::GetSizePrefixedRoot<msg::fbuf::p2pmsg::Container>(buf);
-}
-
-inline Container *GetMutableContainer(void *buf) {
-  return flatbuffers::GetMutableRoot<Container>(buf);
-}
-
-inline bool VerifyContainerBuffer(
-    flatbuffers::Verifier &verifier) {
-  return verifier.VerifyBuffer<msg::fbuf::p2pmsg::Container>(nullptr);
-}
-
-inline bool VerifySizePrefixedContainerBuffer(
-    flatbuffers::Verifier &verifier) {
-  return verifier.VerifySizePrefixedBuffer<msg::fbuf::p2pmsg::Container>(nullptr);
-}
-
-inline void FinishContainerBuffer(
-    flatbuffers::FlatBufferBuilder &fbb,
-    flatbuffers::Offset<msg::fbuf::p2pmsg::Container> root) {
-  fbb.Finish(root);
-}
-
-inline void FinishSizePrefixedContainerBuffer(
-    flatbuffers::FlatBufferBuilder &fbb,
-    flatbuffers::Offset<msg::fbuf::p2pmsg::Container> root) {
-  fbb.FinishSizePrefixed(root);
-}
-
-}  // namespace p2pmsg
-}  // namespace fbuf
-}  // namespace msg
-
-#endif  // FLATBUFFERS_GENERATED_P2PMSGCONTAINER_MSG_FBUF_P2PMSG_H_

src/msg/fbuf/p2pmsg_content.fbs

@@ -1,141 +0,0 @@
-// IDL file for p2p message content schema.
-// flatc -o src/msg/fbuf/ --gen-mutable --cpp src/msg/fbuf/p2pmsg_content.fbs
-
-include "common_schema.fbs";
-namespace msg.fbuf.p2pmsg;
-
-table Peer_Challenge_Message {
-    contract_id:string;
-    roundtime:uint32;
-    challenge:string;
-}
-
-table Peer_Challenge_Response_Message {
-    challenge:string;
-    sig:[ubyte];
-}
-
-table UserInput {
-    input_container:[ubyte];
-    signature:[ubyte];
-    protocol:uint8;
-}
-
-table UserInputGroup {
-    pubkey:[ubyte];
-    messages:[UserInput];
-}
-
-union Message {
-    Peer_Challenge_Response_Message,
-    Peer_Challenge_Message,
-    NonUnl_Proposal_Message,
-    Proposal_Message,
-    Npl_Message,
-    Hpfs_Request_Message,
-    Hpfs_Response_Message,
-    Peer_Requirement_Announcement_Message,
-    Peer_List_Request_Message,
-    Peer_List_Response_Message,
-    Available_Capacity_Announcement_Message
-} //message content type
-
-table Content {
-    message:Message;
-}
-
-table NonUnl_Proposal_Message {
-    user_inputs:[UserInputGroup];
-}
-
-table Proposal_Message { //Proposal type message schema
-    stage:uint8;
-    time:uint64;
-    roundtime:uint32;
-    nonce: [ubyte];
-    users:[ByteArray];
-    input_hashes:[ByteArray];
-    last_blob_shard_id: Sequence_Hash;
-    output_hash:[ubyte];
-    output_sig:[ubyte];
-    state_hash: [ubyte];
-    patch_hash: [ubyte];
-}
-
-table Sequence_Hash {
-    shard_seq_no: uint64;
-    shard_hash: [ubyte];
-}
-
-table Npl_Message { //NPL type message schema
-    data:[ubyte];
-}
-
-enum Ledger_Response_Error : ubyte
-{
-    None = 0,
-    Invalid_Min_Ledger = 1,
-    Req_Ledger_Not_Found = 2
-}
-
-table Hpfs_Request_Message { //Hpfs request message schema
-    mount_id: uint32;
-    parent_path:string;
-    is_file:bool;
-    block_id:int32;
-    expected_hash:[ubyte];
-}
-
-union Hpfs_Response{ File_HashMap_Response, Block_Response, Fs_Entry_Response }
-
-table Hpfs_Response_Message{
-    hpfs_response:Hpfs_Response;
-    hash:[ubyte];
-    path: string;
-    mount_id: uint32;
-}
-
-table Fs_Entry_Response{
-     entries: [Hpfs_FS_Hash_Entry];
-}
-
-table File_HashMap_Response{
-    file_length:uint64;
-    hash_map:[ubyte];
-}
-
-table Block_Response{
-    block_id:uint32;
-    data: [ubyte];
-}
-
-table Hpfs_FS_Hash_Entry{
-    name: string;
-    is_file: bool;
-    hash: [ubyte];
-}
-
-table Peer_Requirement_Announcement_Message{
-    need_consensus_msg_forwarding: bool;
-}
-
-table Available_Capacity_Announcement_Message{
-    available_capacity:int16;
-    timestamp:uint64;
-}
-
-table Peer_List_Request_Message{
-}
-
-table Peer_List_Response_Message{
-    peer_list: [Peer_Properties];
-}
-
-table Peer_Properties {
-    host_address:string;
-    port:uint16;
-    available_capacity:int16;
-    timestamp:uint64;
-}
-
-root_type Content; //root type for message content

src/msg/fbuf/p2pmsg_conversion.cpp

@@ -0,0 +1,554 @@
+#include "../../hpfs/hpfs_mount.hpp"
+#include "../../unl.hpp"
+#include "../../crypto.hpp"
+#include "../../p2p/p2p.hpp"
+#include "common_helpers.hpp"
+#include "flatbuf_hasher.hpp"
+#include "p2pmsg_conversion.hpp"
+
+namespace msg::fbuf::p2pmsg
+{
+    // Length of a peer connection challange.
+    constexpr size_t PEERCHALLENGE_LEN = 16;
+
+    // Max size of messages which are subjected to time (too old) check.
+    constexpr size_t MAX_SIZE_FOR_TIME_CHECK = 1 * 1024 * 1024; // 1 MB
+
+    /**
+     * This section contains Flatbuffer message reading/writing helpers.
+     * These helpers are mainly used by peer_session_handler and other components which sends outgoing p2p messages.
+     * 
+     * A p2p flatbuffer message is a bucket with hp version and the message 'content'.
+     */
+
+    //---Flatbuf to std---//
+
+    bool verify_peer_message(std::string_view message)
+    {
+        // Accessing message buffer
+        const uint8_t *buf = reinterpret_cast<const uint8_t *>(message.data());
+        const size_t buf_size = message.size();
+
+        // Verify container message using flatbuffer verifier
+        flatbuffers::Verifier verifier(buf, buf_size, 16, 100);
+        return VerifyP2PMsgBuffer(verifier);
+    }
+
+    const p2p::peer_message_info get_peer_message_info(std::string_view message)
+    {
+        const auto p2p_msg = p2pmsg::GetP2PMsg(message.data());
+
+        // Check message timestamp (ignore this for large messages).
+        if (message.size() <= MAX_SIZE_FOR_TIME_CHECK)
+        {
+            const uint64_t time_now = util::get_epoch_milliseconds();
+            if (p2p_msg->created_on() < (time_now - (conf::cfg.contract.roundtime * 4)))
+            {
+                LOG_DEBUG << "Peer message is too old.";
+                return p2p::peer_message_info{NULL, P2PMsgContent_NONE, 0};
+            }
+        }
+
+        return p2p::peer_message_info{p2p_msg, p2p_msg->content_type(), p2p_msg->created_on()};
+    }
+
+    bool verify_proposal_msg_signature(const p2p::peer_message_info &mi)
+    {
+        const auto &msg = *mi.p2p_msg->content_as_ProposalMsg();
+
+        // Get hash of proposal data field values and verify the signature against the hash.
+        flatbuf_hasher hasher;
+        hasher.add(msg.stage());
+        hasher.add(msg.time());
+        hasher.add(msg.roundtime());
+        hasher.add(msg.nonce());
+        hasher.add(msg.users());
+        hasher.add(msg.input_hashes());
+        hasher.add(msg.output_hash());
+        hasher.add(msg.output_sig());
+        hasher.add(msg.state_hash());
+        hasher.add(msg.patch_hash());
+        hasher.add(msg.last_primary_shard_id());
+        hasher.add(msg.last_blob_shard_id());
+
+        return crypto::verify(hasher.hash(), flatbuf_bytes_to_sv(msg.sig()), flatbuf_bytes_to_sv(msg.pubkey())) == 0;
+    }
+
+    bool verify_npl_msg_signature(const p2p::peer_message_info &mi)
+    {
+        const auto &msg = *mi.p2p_msg->content_as_NplMsg();
+
+        // Get hash of npl message field values and verify the signature against the hash.
+        flatbuf_hasher hasher;
+        hasher.add(msg.data());
+        hasher.add(msg.lcl_id());
+
+        return crypto::verify(hasher.hash(), flatbuf_bytes_to_sv(msg.sig()), flatbuf_bytes_to_sv(msg.pubkey())) == 0;
+    }
+
+    const p2p::peer_challenge create_peer_challenge_from_msg(const p2p::peer_message_info &mi)
+    {
+        const auto &msg = *mi.p2p_msg->content_as_PeerChallengeMsg();
+        return {
+            std::string(flatbuf_str_to_sv(msg.contract_id())),
+            msg.roundtime(),
+            std::string(flatbuf_bytes_to_sv(msg.challenge()))};
+    }
+
+    const p2p::peer_challenge_response create_peer_challenge_response_from_msg(const p2p::peer_message_info &mi)
+    {
+        const auto &msg = *mi.p2p_msg->content_as_PeerChallengeResponseMsg();
+        return {
+            std::string(flatbuf_bytes_to_sv(msg.challenge())),
+            std::string(flatbuf_bytes_to_sv(msg.sig())),
+            std::string(flatbuf_bytes_to_sv(msg.pubkey()))};
+    }
+
+    const p2p::proposal create_proposal_from_msg(const p2p::peer_message_info &mi)
+    {
+        const auto &msg = *mi.p2p_msg->content_as_ProposalMsg();
+
+        p2p::proposal p;
+        p.pubkey = flatbuf_bytes_to_sv(msg.pubkey());
+        p.sent_timestamp = mi.originated_on;
+        p.recv_timestamp = util::get_epoch_milliseconds();
+        p.time = msg.time();
+        p.roundtime = msg.roundtime();
+        p.nonce = flatbuf_bytes_to_sv(msg.nonce());
+        p.stage = msg.stage();
+        p.state_hash = flatbuf_bytes_to_sv(msg.state_hash());
+        p.patch_hash = flatbuf_bytes_to_sv(msg.patch_hash());
+        p.last_primary_shard_id = flatbuf_seqhash_to_seqhash(msg.last_primary_shard_id());
+        p.last_blob_shard_id = flatbuf_seqhash_to_seqhash(msg.last_blob_shard_id());
+
+        if (msg.users())
+            p.users = flatbuf_bytearrayvector_to_stringlist(msg.users());
+
+        if (msg.input_hashes())
+            p.input_hashes = flatbuf_bytearrayvector_to_stringlist(msg.input_hashes());
+
+        if (msg.output_hash())
+            p.output_hash = flatbuf_bytes_to_sv(msg.output_hash());
+
+        if (msg.output_sig())
+            p.output_sig = flatbuf_bytes_to_sv(msg.output_sig());
+
+        return p;
+    }
+
+    const p2p::npl_message create_npl_from_msg(const p2p::peer_message_info &mi)
+    {
+        const auto &msg = *mi.p2p_msg->content_as_NplMsg();
+        return {
+            std::string(flatbuf_bytes_to_sv(msg.pubkey())),
+            flatbuf_seqhash_to_seqhash(msg.lcl_id()),
+            std::string(flatbuf_bytes_to_sv(msg.data()))};
+    }
+
+    const p2p::nonunl_proposal create_nonunl_proposal_from_msg(const p2p::peer_message_info &mi)
+    {
+        const auto &msg = *mi.p2p_msg->content_as_NonUnlProposalMsg();
+
+        p2p::nonunl_proposal nup;
+        if (msg.user_inputs())
+            nup.user_inputs = flatbuf_user_input_group_to_user_input_map(msg.user_inputs());
+
+        return nup;
+    }
+
+    const std::vector<conf::peer_properties> create_peer_list_response_from_msg(const p2p::peer_message_info &mi)
+    {
+        const auto &msg = *mi.p2p_msg->content_as_PeerListResponseMsg();
+        return flatbuf_peer_propertieslist_to_peer_propertiesvector(msg.peer_list());
+    }
+
+    const p2p::peer_capacity_announcement create_peer_capacity_announcement_from_msg(const p2p::peer_message_info &mi)
+    {
+        const auto &msg = *mi.p2p_msg->content_as_PeerCapacityAnnouncementMsg();
+        return {
+            msg.available_capacity(),
+            msg.timestamp()};
+    }
+
+    const p2p::peer_requirement_announcement create_peer_requirement_announcement_from_msg(const p2p::peer_message_info &mi)
+    {
+        const auto &msg = *mi.p2p_msg->content_as_PeerRequirementAnnouncementMsg();
+        return {
+            msg.need_consensus_msg_forwarding()};
+    }
+
+    const p2p::hpfs_request create_hpfs_request_from_msg(const p2p::peer_message_info &mi)
+    {
+        const auto &msg = *mi.p2p_msg->content_as_HpfsRequestMsg();
+        p2p::hpfs_request hr;
+        hr.mount_id = msg.mount_id();
+        hr.block_id = msg.block_id();
+        hr.is_file = msg.is_file();
+        hr.parent_path = flatbuf_str_to_sv(msg.parent_path());
+        hr.expected_hash = flatbuf_bytes_to_hash(msg.expected_hash());
+        return hr;
+    }
+
+    p2p::sequence_hash flatbuf_seqhash_to_seqhash(const SequenceHash *fbseqhash)
+    {
+        return {
+            fbseqhash->seq_no(),
+            flatbuf_bytes_to_hash(fbseqhash->hash())};
+    }
+
+    const std::set<std::string> flatbuf_bytearrayvector_to_stringlist(const flatbuffers::Vector<flatbuffers::Offset<ByteArray>> *fbvec)
+    {
+        std::set<std::string> set;
+        for (const auto el : *fbvec)
+            set.emplace(std::string(flatbuf_bytes_to_sv(el->array())));
+        return set;
+    }
+
+    const std::unordered_map<std::string, std::list<usr::submitted_user_input>>
+    flatbuf_user_input_group_to_user_input_map(const flatbuffers::Vector<flatbuffers::Offset<UserInputGroup>> *fbvec)
+    {
+        std::unordered_map<std::string, std::list<usr::submitted_user_input>> map;
+        map.reserve(fbvec->size());
+        for (const UserInputGroup *group : *fbvec)
+        {
+            std::list<usr::submitted_user_input> user_inputs_list;
+
+            for (const auto msg : *group->messages())
+            {
+                user_inputs_list.push_back(usr::submitted_user_input{
+                    std::string(flatbuf_bytes_to_sv(msg->input_container())),
+                    std::string(flatbuf_bytes_to_sv(msg->sig())),
+                    static_cast<util::PROTOCOL>(msg->protocol())});
+            }
+
+            map.emplace(flatbuf_bytes_to_sv(group->pubkey()), std::move(user_inputs_list));
+        }
+        return map;
+    }
+
+    void flatbuf_hpfsfshashentry_to_hpfsfshashentry(std::unordered_map<std::string, p2p::hpfs_fs_hash_entry> &fs_entries, const flatbuffers::Vector<flatbuffers::Offset<HpfsFSHashEntry>> *fhashes)
+    {
+        for (const HpfsFSHashEntry *f_hash : *fhashes)
+        {
+            p2p::hpfs_fs_hash_entry entry;
+            entry.name = flatbuf_str_to_sv(f_hash->name());
+            entry.is_file = f_hash->is_file();
+            entry.hash = flatbuf_bytes_to_hash(f_hash->hash());
+
+            fs_entries.emplace(entry.name, std::move(entry));
+        }
+    }
+
+    const std::vector<conf::peer_properties>
+    flatbuf_peer_propertieslist_to_peer_propertiesvector(const flatbuffers::Vector<flatbuffers::Offset<PeerProperties>> *fbvec)
+    {
+        std::vector<conf::peer_properties> peers;
+
+        for (const PeerProperties *peer : *fbvec)
+        {
+            conf::peer_properties properties;
+
+            properties.ip_port.host_address = flatbuf_str_to_sv(peer->host_address());
+            properties.ip_port.port = peer->port();
+            properties.timestamp = peer->timestamp();
+            properties.available_capacity = peer->available_capacity();
+
+            peers.push_back(properties);
+        }
+        return peers;
+    }
+
+    //---std to Flatbuf---//
+
+    const std::string generate_proposal_signature(const p2p::proposal &p)
+    {
+        flatbuf_hasher hasher;
+        hasher.add(p.stage);
+        hasher.add(p.time);
+        hasher.add(p.roundtime);
+        hasher.add(p.nonce);
+        hasher.add(p.users);
+        hasher.add(p.input_hashes);
+        hasher.add(p.output_hash);
+        hasher.add(p.output_sig);
+        hasher.add(p.state_hash);
+        hasher.add(p.patch_hash);
+        hasher.add(p.last_primary_shard_id);
+        hasher.add(p.last_blob_shard_id);
+
+        return crypto::sign(hasher.hash(), conf::cfg.node.private_key);
+    }
+
+    const std::string generate_npl_signature(std::string_view data, const p2p::sequence_hash &lcl_id)
+    {
+        flatbuf_hasher hasher;
+        hasher.add(data);
+        hasher.add(lcl_id);
+
+        return crypto::sign(hasher.hash(), conf::cfg.node.private_key);
+    }
+
+    void create_p2p_msg(flatbuffers::FlatBufferBuilder &builder, const msg::fbuf::p2pmsg::P2PMsgContent content_type, const flatbuffers::Offset<void> content)
+    {
+        const auto p2pmsg = CreateP2PMsg(builder,
+                                         sv_to_flatbuf_str(builder, conf::cfg.hp_version),
+                                         util::get_epoch_milliseconds(),
+                                         content_type,
+                                         content);
+        builder.Finish(p2pmsg);
+    }
+
+    void create_msg_from_peer_challenge(flatbuffers::FlatBufferBuilder &builder, std::string &challenge)
+    {
+        // We calculate the peer challenge to be a random string.
+        crypto::random_bytes(challenge, PEERCHALLENGE_LEN);
+
+        const auto msg = CreatePeerChallengeMsg(
+            builder,
+            sv_to_flatbuf_str(builder, conf::cfg.contract.id),
+            conf::cfg.contract.roundtime,
+            sv_to_flatbuf_bytes(builder, challenge));
+        create_p2p_msg(builder, P2PMsgContent_PeerChallengeMsg, msg.Union());
+    }
+
+    void create_peer_challenge_response_from_challenge(flatbuffers::FlatBufferBuilder &builder, const std::string &challenge)
+    {
+        const std::string sig = crypto::sign(challenge, conf::cfg.node.private_key);
+        const auto msg = CreatePeerChallengeResponseMsg(
+            builder,
+            sv_to_flatbuf_bytes(builder, challenge),
+            sv_to_flatbuf_bytes(builder, conf::cfg.node.public_key),
+            sv_to_flatbuf_bytes(builder, sig));
+
+        create_p2p_msg(builder, P2PMsgContent_PeerChallengeResponseMsg, msg.Union());
+    }
+
+    void create_msg_from_nonunl_proposal(flatbuffers::FlatBufferBuilder &builder, const p2p::nonunl_proposal &nup)
+    {
+        const auto msg = CreateNonUnlProposalMsg(
+            builder,
+            user_input_map_to_flatbuf_user_input_group(builder, nup.user_inputs));
+
+        create_p2p_msg(builder, P2PMsgContent_NonUnlProposalMsg, msg.Union());
+    }
+
+    void create_msg_from_proposal(flatbuffers::FlatBufferBuilder &builder, const p2p::proposal &p)
+    {
+        const auto msg = CreateProposalMsg(
+            builder,
+            sv_to_flatbuf_bytes(builder, conf::cfg.node.public_key),
+            sv_to_flatbuf_bytes(builder, generate_proposal_signature(p)),
+            p.stage,
+            p.time,
+            p.roundtime,
+            sv_to_flatbuf_bytes(builder, p.nonce),
+            stringlist_to_flatbuf_bytearrayvector(builder, p.users),
+            stringlist_to_flatbuf_bytearrayvector(builder, p.input_hashes),
+            sv_to_flatbuf_bytes(builder, p.output_hash),
+            sv_to_flatbuf_bytes(builder, p.output_sig),
+            hash_to_flatbuf_bytes(builder, p.state_hash),
+            hash_to_flatbuf_bytes(builder, p.patch_hash),
+            seqhash_to_flatbuf_seqhash(builder, p.last_primary_shard_id),
+            seqhash_to_flatbuf_seqhash(builder, p.last_blob_shard_id));
+
+        create_p2p_msg(builder, P2PMsgContent_ProposalMsg, msg.Union());
+    }
+
+    void create_msg_from_npl_output(flatbuffers::FlatBufferBuilder &builder, std::string_view data, const p2p::sequence_hash &lcl_id)
+    {
+        const auto msg = CreateNplMsg(
+            builder,
+            sv_to_flatbuf_bytes(builder, conf::cfg.node.public_key),
+            sv_to_flatbuf_bytes(builder, generate_npl_signature(data, lcl_id)),
+            sv_to_flatbuf_bytes(builder, data),
+            seqhash_to_flatbuf_seqhash(builder, lcl_id));
+
+        create_p2p_msg(builder, P2PMsgContent_NplMsg, msg.Union());
+    }
+
+    void create_msg_from_hpfs_request(flatbuffers::FlatBufferBuilder &builder, const p2p::hpfs_request &hr)
+    {
+        const auto msg = CreateHpfsRequestMsg(
+            builder,
+            hr.mount_id,
+            sv_to_flatbuf_str(builder, hr.parent_path),
+            hr.is_file,
+            hr.block_id,
+            hash_to_flatbuf_bytes(builder, hr.expected_hash));
+
+        create_p2p_msg(builder, P2PMsgContent_HpfsRequestMsg, msg.Union());
+    }
+
+    void create_msg_from_fsentry_response(
+        flatbuffers::FlatBufferBuilder &builder, const std::string_view path, const uint32_t mount_id,
+        std::vector<hpfs::child_hash_node> &hash_nodes, util::h32 expected_hash)
+    {
+        const auto child_msg = CreateHpfsFsEntryResponse(
+            builder,
+            hpfsfshashentry_to_flatbuf_hpfsfshashentry(builder, hash_nodes));
+
+        const auto msg = CreateHpfsResponseMsg(
+            builder,
+            hash_to_flatbuf_bytes(builder, expected_hash),
+            sv_to_flatbuf_str(builder, path),
+            mount_id,
+            HpfsResponse_HpfsFsEntryResponse,
+            child_msg.Union());
+
+        create_p2p_msg(builder, P2PMsgContent_HpfsResponseMsg, msg.Union());
+    }
+
+    void create_msg_from_filehashmap_response(
+        flatbuffers::FlatBufferBuilder &builder, std::string_view path, const uint32_t mount_id,
+        std::vector<util::h32> &hashmap, std::size_t file_length, util::h32 expected_hash)
+    {
+        std::string_view hashmap_sv(reinterpret_cast<const char *>(hashmap.data()), hashmap.size() * sizeof(util::h32));
+
+        const auto child_msg = CreateHpfsFileHashMapResponse(
+            builder,
+            file_length,
+            sv_to_flatbuf_bytes(builder, hashmap_sv));
+
+        const auto msg = CreateHpfsResponseMsg(
+            builder,
+            hash_to_flatbuf_bytes(builder, expected_hash),
+            sv_to_flatbuf_str(builder, path),
+            mount_id,
+            HpfsResponse_HpfsFileHashMapResponse,
+            child_msg.Union());
+
+        create_p2p_msg(builder, P2PMsgContent_HpfsResponseMsg, msg.Union());
+    }
+
+    void create_msg_from_block_response(flatbuffers::FlatBufferBuilder &builder, p2p::block_response &block_resp, const uint32_t mount_id)
+    {
+        const auto child_msg = CreateHpfsBlockResponse(
+            builder,
+            block_resp.block_id,
+            sv_to_flatbuf_bytes(builder, block_resp.data));
+
+        const auto msg = CreateHpfsResponseMsg(
+            builder,
+            hash_to_flatbuf_bytes(builder, block_resp.hash),
+            sv_to_flatbuf_str(builder, block_resp.path),
+            mount_id,
+            HpfsResponse_HpfsBlockResponse,
+            child_msg.Union());
+
+        create_p2p_msg(builder, P2PMsgContent_HpfsResponseMsg, msg.Union());
+    }
+
+    void create_msg_from_peer_requirement_announcement(flatbuffers::FlatBufferBuilder &builder, const bool need_consensus_msg_forwarding)
+    {
+        const auto msg = CreatePeerRequirementAnnouncementMsg(
+            builder,
+            need_consensus_msg_forwarding);
+
+        create_p2p_msg(builder, P2PMsgContent_PeerRequirementAnnouncementMsg, msg.Union());
+    }
+
+    void create_msg_from_available_capacity_announcement(flatbuffers::FlatBufferBuilder &builder, const int16_t &available_capacity, const uint64_t &timestamp)
+    {
+        const auto msg = CreatePeerCapacityAnnouncementMsg(
+            builder,
+            available_capacity,
+            timestamp);
+
+        create_p2p_msg(builder, P2PMsgContent_PeerCapacityAnnouncementMsg, msg.Union());
+    }
+
+    void create_msg_from_peer_list_request(flatbuffers::FlatBufferBuilder &builder)
+    {
+        const auto msg = CreatePeerListRequestMsg(builder);
+        create_p2p_msg(builder, P2PMsgContent_PeerListRequestMsg, msg.Union());
+    }
+
+    void create_msg_from_peer_list_response(flatbuffers::FlatBufferBuilder &builder, const std::vector<conf::peer_properties> &peers, const std::optional<conf::peer_ip_port> &skipping_ip_port)
+    {
+        const auto msg = CreatePeerListResponseMsg(
+            builder,
+            peer_propertiesvector_to_flatbuf_peer_propertieslist(builder, peers, skipping_ip_port));
+
+        create_p2p_msg(builder, P2PMsgContent_PeerListResponseMsg, msg.Union());
+    }
+
+    const flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<UserInputGroup>>>
+    user_input_map_to_flatbuf_user_input_group(flatbuffers::FlatBufferBuilder &builder, const std::unordered_map<std::string, std::list<usr::submitted_user_input>> &map)
+    {
+        std::vector<flatbuffers::Offset<UserInputGroup>> fbvec;
+        fbvec.reserve(map.size());
+        for (const auto &[pubkey, msglist] : map)
+        {
+            std::vector<flatbuffers::Offset<UserInput>> fbmsgsvec;
+            for (const usr::submitted_user_input &msg : msglist)
+            {
+                fbmsgsvec.push_back(CreateUserInput(
+                    builder,
+                    sv_to_flatbuf_bytes(builder, msg.input_container),
+                    sv_to_flatbuf_bytes(builder, msg.sig),
+                    static_cast<uint8_t>(msg.protocol)));
+            }
+
+            fbvec.push_back(CreateUserInputGroup(
+                builder,
+                sv_to_flatbuf_bytes(builder, pubkey),
+                builder.CreateVector(fbmsgsvec)));
+        }
+        return builder.CreateVector(fbvec);
+    }
+
+    const flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<HpfsFSHashEntry>>>
+    hpfsfshashentry_to_flatbuf_hpfsfshashentry(
+        flatbuffers::FlatBufferBuilder &builder,
+        std::vector<hpfs::child_hash_node> &hash_nodes)
+    {
+        std::vector<flatbuffers::Offset<HpfsFSHashEntry>> fbvec;
+        fbvec.reserve(hash_nodes.size());
+        for (auto const &hash_node : hash_nodes)
+        {
+            flatbuffers::Offset<HpfsFSHashEntry> hpfs_fs_entry = CreateHpfsFSHashEntry(
+                builder,
+                sv_to_flatbuf_str(builder, hash_node.name),
+                hash_node.is_file,
+                hash_to_flatbuf_bytes(builder, hash_node.hash));
+
+            fbvec.push_back(hpfs_fs_entry);
+        }
+        return builder.CreateVector(fbvec);
+    }
+
+    const flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<PeerProperties>>>
+    peer_propertiesvector_to_flatbuf_peer_propertieslist(flatbuffers::FlatBufferBuilder &builder, const std::vector<conf::peer_properties> &peers, const std::optional<conf::peer_ip_port> &skipping_ip_port)
+    {
+        std::vector<flatbuffers::Offset<PeerProperties>> fbvec;
+        fbvec.reserve(peers.size());
+        for (auto peer : peers)
+        {
+            // Skipping the requestedc peer from the peer list response.
+            if (!skipping_ip_port.has_value() || peer.ip_port != skipping_ip_port.value())
+                fbvec.push_back(CreatePeerProperties(
+                    builder,
+                    sv_to_flatbuf_str(builder, peer.ip_port.host_address),
+                    peer.ip_port.port,
+                    peer.available_capacity,
+                    peer.timestamp));
+        }
+        return builder.CreateVector(fbvec);
+    }
+
+    const flatbuffers::Offset<msg::fbuf::p2pmsg::SequenceHash>
+    seqhash_to_flatbuf_seqhash(flatbuffers::FlatBufferBuilder &builder, const p2p::sequence_hash &seqhash)
+    {
+        return CreateSequenceHash(builder, seqhash.seq_no, hash_to_flatbuf_bytes(builder, seqhash.hash));
+    }
+
+    const flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<ByteArray>>>
+    stringlist_to_flatbuf_bytearrayvector(flatbuffers::FlatBufferBuilder &builder, const std::set<std::string> &set)
+    {
+        std::vector<flatbuffers::Offset<ByteArray>> fbvec;
+        fbvec.reserve(set.size());
+        for (std::string_view str : set)
+            fbvec.push_back(CreateByteArray(builder, sv_to_flatbuf_bytes(builder, str)));
+        return builder.CreateVector(fbvec);
+    }
+}

src/msg/fbuf/p2pmsg_conversion.hpp

@@ -0,0 +1,108 @@
+#ifndef _HP_MSG_FBUF_P2PMSG_CONVERSION_
+#define _HP_MSG_FBUF_P2PMSG_CONVERSION_
+
+#include "../../pchheader.hpp"
+#include "../../p2p/p2p.hpp"
+#include "../../hpfs/hpfs_mount.hpp"
+#include "p2pmsg_generated.h"
+
+namespace msg::fbuf::p2pmsg
+{
+
+    //---Flatbuf to std---//
+
+    bool verify_peer_message(std::string_view message);
+
+    const p2p::peer_message_info get_peer_message_info(std::string_view message);
+
+    bool verify_proposal_msg_signature(const p2p::peer_message_info &mi);
+
+    bool verify_npl_msg_signature(const p2p::peer_message_info &mi);
+
+    const p2p::peer_challenge create_peer_challenge_from_msg(const p2p::peer_message_info &mi);
+
+    const p2p::peer_challenge_response create_peer_challenge_response_from_msg(const p2p::peer_message_info &mi);
+
+    const p2p::proposal create_proposal_from_msg(const p2p::peer_message_info &mi);
+
+    const p2p::npl_message create_npl_from_msg(const p2p::peer_message_info &mi);
+
+    const p2p::nonunl_proposal create_nonunl_proposal_from_msg(const p2p::peer_message_info &mi);
+
+    const std::vector<conf::peer_properties> create_peer_list_response_from_msg(const p2p::peer_message_info &mi);
+
+    const p2p::peer_capacity_announcement create_peer_capacity_announcement_from_msg(const p2p::peer_message_info &mi);
+
+    const p2p::peer_requirement_announcement create_peer_requirement_announcement_from_msg(const p2p::peer_message_info &mi);
+
+    const p2p::hpfs_request create_hpfs_request_from_msg(const p2p::peer_message_info &mi);
+
+    p2p::sequence_hash flatbuf_seqhash_to_seqhash(const msg::fbuf::p2pmsg::SequenceHash *fbseqhash);
+
+    const std::set<std::string> flatbuf_bytearrayvector_to_stringlist(const flatbuffers::Vector<flatbuffers::Offset<ByteArray>> *fbvec);
+
+    const std::unordered_map<std::string, std::list<usr::submitted_user_input>>
+    flatbuf_user_input_group_to_user_input_map(const flatbuffers::Vector<flatbuffers::Offset<UserInputGroup>> *fbvec);
+
+    void flatbuf_hpfsfshashentry_to_hpfsfshashentry(std::unordered_map<std::string, p2p::hpfs_fs_hash_entry> &fs_entries, const flatbuffers::Vector<flatbuffers::Offset<HpfsFSHashEntry>> *fhashes);
+
+    const std::vector<conf::peer_properties>
+    flatbuf_peer_propertieslist_to_peer_propertiesvector(const flatbuffers::Vector<flatbuffers::Offset<PeerProperties>> *fbvec);
+
+    //---std to Flatbuf---//
+
+    const std::string generate_proposal_signature(const p2p::proposal &p);
+
+    const std::string generate_npl_signature(std::string_view data, const p2p::sequence_hash &lcl_id);
+
+    void create_p2p_msg(flatbuffers::FlatBufferBuilder &builder, const msg::fbuf::p2pmsg::P2PMsgContent content_type, const flatbuffers::Offset<void> content);
+
+    void create_msg_from_peer_challenge(flatbuffers::FlatBufferBuilder &builder, std::string &challenge);
+
+    void create_peer_challenge_response_from_challenge(flatbuffers::FlatBufferBuilder &builder, const std::string &challenge);
+
+    void create_msg_from_nonunl_proposal(flatbuffers::FlatBufferBuilder &builder, const p2p::nonunl_proposal &nup);
+
+    void create_msg_from_proposal(flatbuffers::FlatBufferBuilder &builder, const p2p::proposal &p);
+
+    void create_msg_from_npl_output(flatbuffers::FlatBufferBuilder &builder, std::string_view data, const p2p::sequence_hash &lcl_id);
+
+    void create_msg_from_hpfs_request(flatbuffers::FlatBufferBuilder &builder, const p2p::hpfs_request &hr);
+
+    void create_msg_from_fsentry_response(
+        flatbuffers::FlatBufferBuilder &builder, const std::string_view path, const uint32_t mount_id,
+        std::vector<hpfs::child_hash_node> &hash_nodes, util::h32 expected_hash);
+
+    void create_msg_from_filehashmap_response(
+        flatbuffers::FlatBufferBuilder &builder, std::string_view path, const uint32_t mount_id,
+        std::vector<util::h32> &hashmap, std::size_t file_length, util::h32 expected_hash);
+
+    void create_msg_from_block_response(flatbuffers::FlatBufferBuilder &builder, p2p::block_response &block_resp, const uint32_t mount_id);
+
+    void create_msg_from_peer_requirement_announcement(flatbuffers::FlatBufferBuilder &builder, const bool need_consensus_msg_forwarding);
+
+    void create_msg_from_available_capacity_announcement(flatbuffers::FlatBufferBuilder &builder, const int16_t &available_capacity, const uint64_t &timestamp);
+
+    void create_msg_from_peer_list_request(flatbuffers::FlatBufferBuilder &builder);
+
+    void create_msg_from_peer_list_response(flatbuffers::FlatBufferBuilder &builder, const std::vector<conf::peer_properties> &peers, const std::optional<conf::peer_ip_port> &skipping_ip_port);
+
+    const flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<UserInputGroup>>>
+    user_input_map_to_flatbuf_user_input_group(flatbuffers::FlatBufferBuilder &builder, const std::unordered_map<std::string, std::list<usr::submitted_user_input>> &map);
+
+    const flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<HpfsFSHashEntry>>>
+    hpfsfshashentry_to_flatbuf_hpfsfshashentry(
+        flatbuffers::FlatBufferBuilder &builder,
+        std::vector<hpfs::child_hash_node> &hash_nodes);
+
+    const flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<PeerProperties>>>
+    peer_propertiesvector_to_flatbuf_peer_propertieslist(flatbuffers::FlatBufferBuilder &builder, const std::vector<conf::peer_properties> &peers, const std::optional<conf::peer_ip_port> &skipping_ip_port);
+
+    const flatbuffers::Offset<msg::fbuf::p2pmsg::SequenceHash>
+    seqhash_to_flatbuf_seqhash(flatbuffers::FlatBufferBuilder &builder, const p2p::sequence_hash &seqhash);
+
+    const flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<ByteArray>>>
+    stringlist_to_flatbuf_bytearrayvector(flatbuffers::FlatBufferBuilder &builder, const std::set<std::string> &set);
+}
+
+#endif

src/msg/fbuf/p2pmsg_helpers.cpp

@@ -1,800 +0,0 @@
-#include "../../pchheader.hpp"
-#include "../../conf.hpp"
-#include "../../crypto.hpp"
-#include "../../util/util.hpp"
-#include "../../hplog.hpp"
-#include "../../util/h32.hpp"
-#include "../../unl.hpp"
-#include "p2pmsg_container_generated.h"
-#include "p2pmsg_content_generated.h"
-#include "common_helpers.hpp"
-#include "p2pmsg_helpers.hpp"
-
-namespace msg::fbuf::p2pmsg
-{
-
-    // Length of a peer connection challange.
-    constexpr size_t PEERCHALLENGE_LEN = 16;
-
-    // Max size of messages which are subjected to time (too old) check.
-    constexpr size_t MAX_SIZE_FOR_TIME_CHECK = 1 * 1024 * 1024; // 1 MB
-
-    /**
-     * This section contains Flatbuffer message reading/writing helpers.
-     * These helpers are mainly used by peer_session_handler.
-     * 
-     * All Flatbuffer peer messages are 'Container' messages. 'Container' message is a bucket
-     * which some common headers (version, singature etc..) and the message 'Content' (Proposal, NPL etc..).
-     * 
-     * Therefore, when constructing peer messages, we have to first construct 'Content' message and then
-     * place the 'Content' inside a 'Conatiner. 'Content' and 'Container' messages are constructed using
-     * Flatbuffer builders.
-     * 
-     * Reading is also 2 steps because of this. We have first interprit the 'Container' message from the
-     * received data and then interprit the 'Content' portion of it separately to read the actual content.
-     */
-
-    //---Message validation helpers---/
-
-    /**
-     * Verifies Conatiner message structure and outputs faltbuffer Container pointer to access the given buffer.
-     * 
-     * @param container_ref A pointer reference to assign the pointer to the Container object.
-     * @param container_buf The buffer containing the data that should be validated and interpreted
-     *                      via the container pointer.
-     * @return 0 on successful verification. -1 for failure.
-     */
-    int validate_and_extract_container(const Container **container_ref, std::string_view container_buf)
-    {
-        //Accessing message buffer
-        const uint8_t *container_buf_ptr = reinterpret_cast<const uint8_t *>(container_buf.data());
-        const size_t container_buf_size = container_buf.length();
-
-        //Defining Flatbuffer verifier (default max depth = 64, max_tables = 1000000,)
-        flatbuffers::Verifier container_verifier(container_buf_ptr, container_buf_size);
-
-        //Verify container message using flatbuffer verifier
-        if (!VerifyContainerBuffer(container_verifier))
-        {
-            LOG_DEBUG << "Flatbuffer verify: Bad peer message container.";
-            return -1;
-        }
-
-        //Get message container
-        const Container *container = GetContainer(container_buf_ptr);
-
-        //check protocol version of message whether it is greater than minimum supported protocol version.
-        const uint16_t version = container->version();
-        if (version < util::MIN_PEERMSG_VERSION)
-        {
-            LOG_DEBUG << "Peer message is from unsupported protocol version (" << version << ").";
-            return -1;
-        }
-
-        //check message timestamp (ignore this for large messages).
-        if (container_buf_size <= MAX_SIZE_FOR_TIME_CHECK)
-        {
-            const uint64_t time_now = util::get_epoch_milliseconds();
-            if (container->timestamp() < (time_now - conf::cfg.contract.roundtime * 4))
-            {
-                LOG_DEBUG << "Peer message is too old.";
-                return -1;
-            }
-        }
-
-        //Assign container and content out params.
-        *container_ref = container;
-        return 0;
-    }
-
-    /**
-     * Validates the container message signing keys to see if the message is from a trusted source (UNL).
-     * @return 0 on successful verification. -1 for failure.
-     */
-    int validate_container_trust(const Container *container)
-    {
-        std::string_view msg_pubkey = flatbuff_bytes_to_sv(container->pubkey());
-        std::string_view msg_sig = flatbuff_bytes_to_sv(container->signature());
-
-        if (msg_pubkey.empty() || msg_sig.empty())
-        {
-            LOG_DEBUG << "Peer message key pair incomplete. Trust verification failed.";
-            return -1;
-        }
-
-        //validate if the message is not from a unl node.
-        if (!unl::exists(std::string(msg_pubkey)))
-        {
-            LOG_DEBUG << "Peer message pubkey verification failed. Not a UNL node.";
-            return -1;
-        }
-
-        //verify message signature.
-        //this is performed towards end since this is bit expensive
-        std::string_view msg_content = flatbuff_bytes_to_sv(container->content());
-
-        if (crypto::verify(msg_content, msg_sig, msg_pubkey) != 0)
-        {
-            LOG_DEBUG << "Peer message signature verification failed.";
-            return -1;
-        }
-
-        return 0;
-    }
-
-    /**
-     * Verifies the Content message structure and outputs faltbuffer Content pointer to access the given buffer.
-     * 
-     * @param content_ref A pointer reference to assign the pointer to the Content object.
-     * @param content_ptr Pointer to the buffer containing the data that should validated and interpreted
-     *                      via the container pointer.
-     * @param content_size Data buffer size.
-     * @return 0 on successful verification. -1 for failure.
-     */
-    int validate_and_extract_content(const Content **content_ref, const uint8_t *content_ptr, const flatbuffers::uoffset_t content_size)
-    {
-        //Defining Flatbuffer verifier for message content verification.
-        //Since content is also serialised by using Flatbuffer we can verify it using Flatbuffer.
-        flatbuffers::Verifier content_verifier(content_ptr, content_size);
-
-        //verify content message using flatbuffer verifier.
-        if (!VerifyContainerBuffer(content_verifier))
-        {
-            LOG_DEBUG << "Flatbuffer verify: Bad content.";
-            return -1;
-        }
-
-        *content_ref = GetContent(content_ptr);
-        return 0;
-    }
-
-    //---Message reading helpers---/
-
-    /**
-     * Returns challenge from the peer challenge message.
-     * @param The Flatbuffer peer challenge message received from the peer.
-     * @return Peer challenge struct.
-     */
-    const p2p::peer_challenge get_peer_challenge_from_msg(const Peer_Challenge_Message &msg)
-    {
-        return {
-            std::string(flatbuff_str_to_sv(msg.contract_id())),
-            msg.roundtime(),
-            std::string(flatbuff_str_to_sv(msg.challenge()))};
-    }
-
-    /**
-     * Creates a peer challenge response struct from the given peer challenge response message.
-     * @param The Flatbuffer peer challenge response message received from the peer.
-     * @return A peer challenge response struct representing the message.
-     */
-    const p2p::peer_challenge_response create_peer_challenge_response_from_msg(const Peer_Challenge_Response_Message &msg, const flatbuffers::Vector<uint8_t> *pubkey)
-    {
-        p2p::peer_challenge_response pchalresp;
-
-        pchalresp.challenge = flatbuff_str_to_sv(msg.challenge());
-        pchalresp.signature = flatbuff_bytes_to_sv(msg.sig());
-        pchalresp.pubkey = flatbuff_bytes_to_sv(pubkey);
-
-        return pchalresp;
-    }
-
-    /**
-     * Creates a non-unl proposal stuct from the given non-unl proposal message.
-     * @param The Flatbuffer non-unl poporal received from the peer.
-     * @return A non-unl proposal struct representing the message.
-     */
-    const p2p::nonunl_proposal create_nonunl_proposal_from_msg(const NonUnl_Proposal_Message &msg, const uint64_t timestamp)
-    {
-        p2p::nonunl_proposal nup;
-
-        if (msg.user_inputs())
-            nup.user_inputs = flatbuf_user_input_group_to_user_input_map(msg.user_inputs());
-
-        return nup;
-    }
-
-    /**
- * Creates a proposal stuct from the given proposal message.
- * @param msg The Flatbuffer poposal received from the peer.
- * @return A proposal struct representing the message.
- */
-    const p2p::proposal create_proposal_from_msg(const Proposal_Message &msg, const flatbuffers::Vector<uint8_t> *pubkey, const uint64_t timestamp, const Sequence_Hash &last_primary_shard_id_msg)
-    {
-        p2p::proposal p;
-
-        p.pubkey = flatbuff_bytes_to_sv(pubkey);
-        p.sent_timestamp = timestamp;
-        p.recv_timestamp = util::get_epoch_milliseconds();
-        p.time = msg.time();
-        p.roundtime = msg.roundtime();
-        p.nonce = flatbuff_bytes_to_sv(msg.nonce());
-        p.stage = msg.stage();
-        p.state_hash = flatbuff_bytes_to_sv(msg.state_hash());
-        p.patch_hash = flatbuff_bytes_to_sv(msg.patch_hash());
-
-        p2p::sequence_hash last_primary_shard_id;
-        last_primary_shard_id.seq_no = last_primary_shard_id_msg.shard_seq_no();
-        last_primary_shard_id.hash = flatbuff_bytes_to_hash(last_primary_shard_id_msg.shard_hash());
-        p.last_primary_shard_id = last_primary_shard_id;
-
-        p2p::sequence_hash last_blob_shard_id;
-        const Sequence_Hash &last_blob_shard_id_msg = *msg.last_blob_shard_id();
-        last_blob_shard_id.seq_no = last_blob_shard_id_msg.shard_seq_no();
-        last_blob_shard_id.hash = flatbuff_bytes_to_hash(last_blob_shard_id_msg.shard_hash());
-        p.last_blob_shard_id = last_blob_shard_id;
-
-        if (msg.users())
-            p.users = flatbuf_bytearrayvector_to_stringlist(msg.users());
-
-        if (msg.input_hashes())
-            p.input_hashes = flatbuf_bytearrayvector_to_stringlist(msg.input_hashes());
-
-        if (msg.output_hash())
-            p.output_hash = flatbuff_bytes_to_sv(msg.output_hash());
-
-        if (msg.output_sig())
-            p.output_sig = flatbuff_bytes_to_sv(msg.output_sig());
-
-        return p;
-    }
-
-    /**
-     * Creates a hpfs request struct from the given hpfs request message.
-     * @param msg Flatbuffer State request message received from the peer.
-     * @return A hpfs request struct representing the message.
-     */
-    const p2p::hpfs_request create_hpfs_request_from_msg(const Hpfs_Request_Message &msg)
-    {
-        p2p::hpfs_request hr;
-        hr.mount_id = msg.mount_id();
-        hr.block_id = msg.block_id();
-        hr.is_file = msg.is_file();
-        hr.parent_path = flatbuff_str_to_sv(msg.parent_path());
-        hr.expected_hash = flatbuff_bytes_to_hash(msg.expected_hash());
-
-        return hr;
-    }
-
-    /**
-     * Creates a peer property list from the given peer list response message.
-     * @param msg Flatbuffer Peer List response message received from the peer.
-     * @return A Peer list representing the message.
-     */
-    const std::vector<conf::peer_properties> create_peer_list_response_from_msg(const Peer_List_Response_Message &msg)
-    {
-        return flatbuf_peer_propertieslist_to_peer_propertiesvector(msg.peer_list());
-    }
-
-    //---Message creation helpers---//
-
-    /**
-     * Create peer challenge message from the given challenge.
-     * @param container_builder Flatbuffer builder for the container message.
-     * @param challenge Challenge message needed to convert to flatbuffer message.
-     */
-    void create_msg_from_peer_challenge(flatbuffers::FlatBufferBuilder &container_builder, std::string &challenge)
-    {
-        flatbuffers::FlatBufferBuilder builder(1024);
-
-        // We calculate the peer challenge to be a random string.
-        // Use libsodium to generate the random challenge bytes.
-        challenge.resize(PEERCHALLENGE_LEN);
-        randombytes_buf(challenge.data(), PEERCHALLENGE_LEN);
-
-        const flatbuffers::Offset<Peer_Challenge_Message> peer_challenge_msg =
-            CreatePeer_Challenge_Message(
-                builder,
-                sv_to_flatbuff_str(builder, conf::cfg.contract.id),
-                conf::cfg.contract.roundtime,
-                sv_to_flatbuff_str(builder, challenge));
-
-        const flatbuffers::Offset<Content> message = CreateContent(builder, Message_Peer_Challenge_Message, peer_challenge_msg.Union());
-        builder.Finish(message); // Finished building message content to get serialised content.
-
-        // Now that we have built the content message
-        create_containermsg_from_content(container_builder, builder, {}, false);
-    }
-
-    /**
-     * Create peer challenge response message from the given challenge.
-     * @param container_builder Flatbuffer builder for the container message.
-     * @param challenge Message which need to be signed and placed in the container message.
-     */
-    void create_peer_challenge_response_from_challenge(flatbuffers::FlatBufferBuilder &container_builder, const std::string &challenge)
-    {
-        flatbuffers::FlatBufferBuilder builder(1024);
-
-        const flatbuffers::Offset<Peer_Challenge_Response_Message> challenge_resp_msg =
-            CreatePeer_Challenge_Response_Message(
-                builder,
-                sv_to_flatbuff_str(builder, challenge),
-                sv_to_flatbuff_bytes(builder, crypto::sign(challenge, conf::cfg.node.private_key)));
-
-        const flatbuffers::Offset<Content> message = CreateContent(builder, Message_Peer_Challenge_Response_Message, challenge_resp_msg.Union());
-        builder.Finish(message); // Finished building message content to get serialised content.
-
-        // Now that we have built the content message,
-        // we need to sign it and place it inside a container message.
-        create_containermsg_from_content(container_builder, builder, {}, true);
-    }
-
-    void create_msg_from_nonunl_proposal(flatbuffers::FlatBufferBuilder &container_builder, const p2p::nonunl_proposal &nup)
-    {
-        flatbuffers::FlatBufferBuilder builder(1024);
-
-        const flatbuffers::Offset<NonUnl_Proposal_Message> nupmsg =
-            CreateNonUnl_Proposal_Message(
-                builder,
-                user_input_map_to_flatbuf_user_input_group(builder, nup.user_inputs));
-
-        const flatbuffers::Offset<Content> message = CreateContent(builder, Message_NonUnl_Proposal_Message, nupmsg.Union());
-        builder.Finish(message); // Finished building message content to get serialised content.
-
-        // Now that we have built the content message,
-        // we need to sign it and place it inside a container message.
-        create_containermsg_from_content(container_builder, builder, {}, false);
-    }
-
-    /**
-     * Create proposal peer message from the given proposal struct.
-     * @param container_builder Flatbuffer builder for the container message.
-     * @param p The proposal struct to be placed in the container message.
-     */
-    void create_msg_from_proposal(flatbuffers::FlatBufferBuilder &container_builder, const p2p::proposal &p)
-    {
-        // todo:get a average propsal message size and allocate content builder based on that.
-        flatbuffers::FlatBufferBuilder builder(1024);
-
-        const flatbuffers::Offset<Sequence_Hash> last_blob_shard_id_msg = CreateSequence_Hash(
-            builder,
-            p.last_blob_shard_id.seq_no,
-            hash_to_flatbuff_bytes(builder, p.last_blob_shard_id.hash));
-
-        const flatbuffers::Offset<Proposal_Message> proposal =
-            CreateProposal_Message(
-                builder,
-                p.stage,
-                p.time,
-                p.roundtime,
-                sv_to_flatbuff_bytes(builder, p.nonce),
-                stringlist_to_flatbuf_bytearrayvector(builder, p.users),
-                stringlist_to_flatbuf_bytearrayvector(builder, p.input_hashes),
-                last_blob_shard_id_msg,
-                sv_to_flatbuff_bytes(builder, p.output_hash),
-                sv_to_flatbuff_bytes(builder, p.output_sig),
-                hash_to_flatbuff_bytes(builder, p.state_hash),
-                hash_to_flatbuff_bytes(builder, p.patch_hash));
-
-        const flatbuffers::Offset<Content> message = CreateContent(builder, Message_Proposal_Message, proposal.Union());
-        builder.Finish(message); // Finished building message content to get serialised content.
-
-        // Now that we have built the content message,
-        // we need to sign it and place it inside a container message.
-        create_containermsg_from_content(container_builder, builder, p.last_primary_shard_id, true);
-    }
-
-    /**
-     * Ctreat npl message from the given npl output srtuct.
-     * @param container_builder Flatbuffer builder for the container message.
-     * @param msg The message to be sent as NPL message.
-     * @param last_primary_shard_id Last primary shard id.
-     */
-    void create_msg_from_npl_output(flatbuffers::FlatBufferBuilder &container_builder, const std::string_view &msg, const p2p::sequence_hash &last_primary_shard_id)
-    {
-        flatbuffers::FlatBufferBuilder builder(1024);
-
-        const flatbuffers::Offset<Npl_Message> npl =
-            CreateNpl_Message(
-                builder,
-                sv_to_flatbuff_bytes(builder, msg));
-
-        const flatbuffers::Offset<Content> message = CreateContent(builder, Message_Npl_Message, npl.Union());
-        builder.Finish(message); // Finished building message content to get serialised content.
-
-        // Now that we have built the content message,
-        // we need to sign it and place it inside a container message.
-        create_containermsg_from_content(container_builder, builder, last_primary_shard_id, true);
-    }
-
-    /**
-     * Create hpfs request message from the given hpfs request struct.
-     * @param container_builder Flatbuffer builder for the container message.
-     * @param hr The hpfs request struct to be placed in the container message.
-     * @param last_primary_shard_id Last primary shard id.
-     */
-    void create_msg_from_hpfs_request(flatbuffers::FlatBufferBuilder &container_builder, const p2p::hpfs_request &hr, const p2p::sequence_hash &last_primary_shard_id)
-    {
-        flatbuffers::FlatBufferBuilder builder(1024);
-
-        flatbuffers::Offset<Hpfs_Request_Message> srmsg =
-            CreateHpfs_Request_Message(
-                builder,
-                hr.mount_id,
-                sv_to_flatbuff_str(builder, hr.parent_path),
-                hr.is_file,
-                hr.block_id,
-                hash_to_flatbuff_bytes(builder, hr.expected_hash));
-
-        flatbuffers::Offset<Content> message = CreateContent(builder, Message_Hpfs_Request_Message, srmsg.Union());
-        builder.Finish(message); // Finished building message content to get serialised content.
-
-        // Now that we have built the content message,
-        // we need to sign it and place it inside a container message.
-        create_containermsg_from_content(container_builder, builder, last_primary_shard_id, false);
-    }
-
-    /**
-     * Create content response message from the given content response.
-     * @param container_builder Flatbuffer builder for the container message.
-     * @param path The path of the directory.
-     * @param mount_id The mount id of the relavent hpfs mount.
-     * @param hash_nodes File or directory entries with hashes in the given parent path.
-     * @param expected_hash The exptected hash of the requested path.
-     * @param last_primary_shard_id Last primary shard id.
-     */
-    void create_msg_from_fsentry_response(
-        flatbuffers::FlatBufferBuilder &container_builder, const std::string_view path, const uint32_t mount_id,
-        std::vector<hpfs::child_hash_node> &hash_nodes, util::h32 expected_hash, const p2p::sequence_hash &last_primary_shard_id)
-    {
-        flatbuffers::FlatBufferBuilder builder(1024);
-
-        const flatbuffers::Offset<Fs_Entry_Response> resp =
-            CreateFs_Entry_Response(
-                builder,
-                hpfsfshashentry_to_flatbuff_hpfsfshashentry(builder, hash_nodes));
-
-        const flatbuffers::Offset<Hpfs_Response_Message> st_resp = CreateHpfs_Response_Message(
-            builder, Hpfs_Response_Fs_Entry_Response,
-            resp.Union(),
-            hash_to_flatbuff_bytes(builder, expected_hash),
-            sv_to_flatbuff_str(builder, path), mount_id);
-
-        flatbuffers::Offset<Content> message = CreateContent(builder, Message_Hpfs_Response_Message, st_resp.Union());
-        builder.Finish(message); // Finished building message content to get serialised content.
-
-        // Now that we have built the content message,
-        // we need to sign it and place it inside a container message.
-        create_containermsg_from_content(container_builder, builder, last_primary_shard_id, true);
-    }
-
-    /**
-     * Create content response message from the given content response.
-     * @param container_builder Flatbuffer builder for the container message.
-     * @param path The path of the directory.
-     * @param mount_id The mount id of the relavent hpfs mount.
-     * @param hashmap Hashmap of the file
-     * @param last_primary_shard_id Last primary shard id.
-     */
-    void create_msg_from_filehashmap_response(
-        flatbuffers::FlatBufferBuilder &container_builder, std::string_view path, const uint32_t mount_id,
-        std::vector<util::h32> &hashmap, std::size_t file_length, util::h32 expected_hash, const p2p::sequence_hash &last_primary_shard_id)
-    {
-        // todo:get a average propsal message size and allocate content builder based on that.
-        flatbuffers::FlatBufferBuilder builder(1024);
-
-        std::string_view hashmap_sv(reinterpret_cast<const char *>(hashmap.data()), hashmap.size() * sizeof(util::h32));
-
-        const flatbuffers::Offset<File_HashMap_Response> resp =
-            CreateFile_HashMap_Response(
-                builder,
-                file_length,
-                sv_to_flatbuff_bytes(builder, hashmap_sv));
-
-        const flatbuffers::Offset<Hpfs_Response_Message> st_resp = CreateHpfs_Response_Message(
-            builder,
-            Hpfs_Response_File_HashMap_Response,
-            resp.Union(),
-            hash_to_flatbuff_bytes(builder, expected_hash),
-            sv_to_flatbuff_str(builder, path), mount_id);
-
-        flatbuffers::Offset<Content> message = CreateContent(builder, Message_Hpfs_Response_Message, st_resp.Union());
-        builder.Finish(message); // Finished building message content to get serialised content.
-
-        // Now that we have built the content message,
-        // we need to sign it and place it inside a container message.
-        create_containermsg_from_content(container_builder, builder, last_primary_shard_id, true);
-    }
-
-    /**
-     * Create content response message from the given content response.
-     * @param container_builder Flatbuffer builder for the container message.
-     * @param block_resp Block response struct to place in the message.
-     * @param mount_id The mount id of the relavent hpfs mount.
-     * @param last_primary_shard_id Last primary shard id.
-     */
-    void create_msg_from_block_response(flatbuffers::FlatBufferBuilder &container_builder, p2p::block_response &block_resp, const uint32_t mount_id,
-                                        const p2p::sequence_hash &last_primary_shard_id)
-    {
-        // todo:get a average propsal message size and allocate content builder based on that.
-        flatbuffers::FlatBufferBuilder builder(1024);
-
-        const flatbuffers::Offset<Block_Response> resp =
-            CreateBlock_Response(
-                builder,
-                block_resp.block_id,
-                sv_to_flatbuff_bytes(builder, block_resp.data));
-
-        const flatbuffers::Offset<Hpfs_Response_Message> st_resp = CreateHpfs_Response_Message(
-            builder,
-            Hpfs_Response_Block_Response,
-            resp.Union(),
-            hash_to_flatbuff_bytes(builder, block_resp.hash),
-            sv_to_flatbuff_str(builder, block_resp.path), mount_id);
-
-        flatbuffers::Offset<Content> message = CreateContent(builder, Message_Hpfs_Response_Message, st_resp.Union());
-        builder.Finish(message); // Finished building message content to get serialised content.
-
-        // Now that we have built the content message,
-        // we need to sign it and place it inside a container message.
-        create_containermsg_from_content(container_builder, builder, last_primary_shard_id, true);
-    }
-
-    /**
-     * Create connected status announcement message.
-     * @param container_builder Flatbuffer builder for the container message.
-     * @param need_consensus_msg_forwarding True if number of connections are below threshold and false otherwise.
-     */
-    void create_msg_from_peer_requirement_announcement(flatbuffers::FlatBufferBuilder &container_builder, const bool need_consensus_msg_forwarding,
-                                                       const p2p::sequence_hash &last_primary_shard_id)
-    {
-        flatbuffers::FlatBufferBuilder builder(1024);
-
-        const flatbuffers::Offset<Peer_Requirement_Announcement_Message> announcement =
-            CreatePeer_Requirement_Announcement_Message(
-                builder,
-                need_consensus_msg_forwarding);
-
-        const flatbuffers::Offset<Content> message = CreateContent(builder, Message_Peer_Requirement_Announcement_Message, announcement.Union());
-        builder.Finish(message); // Finished building message content to get serialised content.
-
-        // Now that we have built the content message,
-        create_containermsg_from_content(container_builder, builder, last_primary_shard_id, false);
-    }
-
-    /**
-     * Create available capacity announcement message.
-     * @param container_builder Flatbuffer builder for the container message.
-     * @param available_capacity Number of incoming connection slots available, -1 means there's no limitation for connections.
-     * @param timestamp Announced timestamp.
-     */
-    void create_msg_from_available_capacity_announcement(flatbuffers::FlatBufferBuilder &container_builder, const int16_t &available_capacity, const uint64_t &timestamp,
-                                                         const p2p::sequence_hash &last_primary_shard_id)
-    {
-        flatbuffers::FlatBufferBuilder builder(1024);
-
-        const flatbuffers::Offset<Available_Capacity_Announcement_Message> announcement =
-            CreateAvailable_Capacity_Announcement_Message(
-                builder,
-                available_capacity,
-                timestamp);
-
-        const flatbuffers::Offset<Content> message = CreateContent(builder, Message_Available_Capacity_Announcement_Message, announcement.Union());
-        builder.Finish(message); // Finished building message content to get serialised content.
-
-        // Now that we have built the content message,
-        create_containermsg_from_content(container_builder, builder, last_primary_shard_id, false);
-    }
-
-    /**
-     * Create peer list request message.
-     * @param container_builder Flatbuffer builder for the container message.
-     */
-    void create_msg_from_peer_list_request(flatbuffers::FlatBufferBuilder &container_builder, const p2p::sequence_hash &last_primary_shard_id)
-    {
-        flatbuffers::FlatBufferBuilder builder(1024);
-
-        const flatbuffers::Offset<Peer_List_Request_Message> request =
-            CreatePeer_List_Request_Message(
-                builder);
-
-        const flatbuffers::Offset<Content> message = CreateContent(builder, Message_Peer_List_Request_Message, request.Union());
-        builder.Finish(message); // Finished building message content to get serialised content.
-
-        // Now that we have built the content message,
-        create_containermsg_from_content(container_builder, builder, last_primary_shard_id, false);
-    }
-
-    /**
-     * Create peer list response message.
-     * @param container_builder Flatbuffer builder for the container message.
-     * @param peers Peer list to be sent to another peer.
-     * @param skipping_peer Peer that does not need to be sent.
-     */
-    void create_msg_from_peer_list_response(flatbuffers::FlatBufferBuilder &container_builder, const std::vector<conf::peer_properties> &peers, const std::optional<conf::peer_ip_port> &skipping_ip_port,
-                                            const p2p::sequence_hash &last_primary_shard_id)
-    {
-        flatbuffers::FlatBufferBuilder builder(1024);
-
-        const flatbuffers::Offset<Peer_List_Response_Message> response =
-            CreatePeer_List_Response_Message(
-                builder,
-                peer_propertiesvector_to_flatbuf_peer_propertieslist(builder, peers, skipping_ip_port));
-
-        const flatbuffers::Offset<Content> message = CreateContent(builder, Message_Peer_List_Response_Message, response.Union());
-        builder.Finish(message); // Finished building message content to get serialised content.
-
-        // Now that we have built the content message,
-        create_containermsg_from_content(container_builder, builder, last_primary_shard_id, false);
-    }
-
-    /**
-     * Creates a Flatbuffer container message from the given Content message.
-     * @param container_builder The Flatbuffer builder to which the final container message should be written to.
-     * @param content_builder The Flatbuffer builder containing the content message that should be placed
-     *                        inside the container message.
-     * @param sign Whether to sign the message content.
-     */
-    void create_containermsg_from_content(
-        flatbuffers::FlatBufferBuilder &container_builder, const flatbuffers::FlatBufferBuilder &content_builder, const p2p::sequence_hash &last_primary_shard_id, const bool sign)
-    {
-        const uint8_t *content_buf = content_builder.GetBufferPointer();
-        const flatbuffers::uoffset_t content_size = content_builder.GetSize();
-
-        // Create container message content from serialised content from previous step.
-        const flatbuffers::Offset<flatbuffers::Vector<uint8_t>> content = container_builder.CreateVector(content_buf, content_size);
-
-        flatbuffers::Offset<flatbuffers::Vector<uint8_t>> pubkey_offset = 0;
-        flatbuffers::Offset<flatbuffers::Vector<uint8_t>> sig_offset = 0;
-
-        if (sign)
-        {
-            // Sign message content with this node's private key.
-            std::string_view content_to_sign(reinterpret_cast<const char *>(content_buf), content_size);
-
-            sig_offset = sv_to_flatbuff_bytes(container_builder, crypto::sign(content_to_sign, conf::cfg.node.private_key));
-            pubkey_offset = sv_to_flatbuff_bytes(container_builder, conf::cfg.node.public_key);
-        }
-
-        const flatbuffers::Offset<Sequence_Hash> last_primary_shard_id_msg = CreateSequence_Hash(
-            container_builder,
-            last_primary_shard_id.seq_no,
-            hash_to_flatbuff_bytes(container_builder, last_primary_shard_id.hash));
-
-        const flatbuffers::Offset<Container> container_message = CreateContainer(
-            container_builder,
-            util::PEERMSG_VERSION,
-            util::get_epoch_milliseconds(),
-            pubkey_offset,
-            0,
-            last_primary_shard_id_msg,
-            sig_offset,
-            content);
-
-        // Finish building message container to get serialised message.
-        container_builder.Finish(container_message);
-    }
-
-    //---Conversion helpers from flatbuffers data types to std data types---//
-
-    const std::unordered_map<std::string, std::list<usr::submitted_user_input>>
-    flatbuf_user_input_group_to_user_input_map(const flatbuffers::Vector<flatbuffers::Offset<UserInputGroup>> *fbvec)
-    {
-        std::unordered_map<std::string, std::list<usr::submitted_user_input>> map;
-        map.reserve(fbvec->size());
-        for (const UserInputGroup *group : *fbvec)
-        {
-            std::list<usr::submitted_user_input> user_inputs_list;
-
-            for (const auto msg : *group->messages())
-            {
-                user_inputs_list.push_back(usr::submitted_user_input{
-                    std::string(flatbuff_bytes_to_sv(msg->input_container())),
-                    std::string(flatbuff_bytes_to_sv(msg->signature())),
-                    static_cast<util::PROTOCOL>(msg->protocol())});
-            }
-
-            map.emplace(flatbuff_bytes_to_sv(group->pubkey()), std::move(user_inputs_list));
-        }
-        return map;
-    }
-
-    //---Conversion helpers from std data types to flatbuffers data types---//
-    //---These are used in constructing Flatbuffer messages using builders---//
-
-    const flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<UserInputGroup>>>
-    user_input_map_to_flatbuf_user_input_group(flatbuffers::FlatBufferBuilder &builder, const std::unordered_map<std::string, std::list<usr::submitted_user_input>> &map)
-    {
-        std::vector<flatbuffers::Offset<UserInputGroup>> fbvec;
-        fbvec.reserve(map.size());
-        for (const auto &[pubkey, msglist] : map)
-        {
-            std::vector<flatbuffers::Offset<UserInput>> fbmsgsvec;
-            for (const usr::submitted_user_input &msg : msglist)
-            {
-                fbmsgsvec.push_back(CreateUserInput(
-                    builder,
-                    sv_to_flatbuff_bytes(builder, msg.input_container),
-                    sv_to_flatbuff_bytes(builder, msg.sig),
-                    static_cast<uint8_t>(msg.protocol)));
-            }
-
-            fbvec.push_back(CreateUserInputGroup(
-                builder,
-                sv_to_flatbuff_bytes(builder, pubkey),
-                builder.CreateVector(fbmsgsvec)));
-        }
-        return builder.CreateVector(fbvec);
-    }
-
-    void flatbuf_hpfsfshashentry_to_hpfsfshashentry(std::unordered_map<std::string, p2p::hpfs_fs_hash_entry> &fs_entries, const flatbuffers::Vector<flatbuffers::Offset<Hpfs_FS_Hash_Entry>> *fhashes)
-    {
-        for (const Hpfs_FS_Hash_Entry *f_hash : *fhashes)
-        {
-            p2p::hpfs_fs_hash_entry entry;
-            entry.name = flatbuff_str_to_sv(f_hash->name());
-            entry.is_file = f_hash->is_file();
-            entry.hash = flatbuff_bytes_to_hash(f_hash->hash());
-
-            fs_entries.emplace(entry.name, std::move(entry));
-        }
-    }
-
-    flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<Hpfs_FS_Hash_Entry>>>
-    hpfsfshashentry_to_flatbuff_hpfsfshashentry(
-        flatbuffers::FlatBufferBuilder &builder,
-        std::vector<hpfs::child_hash_node> &hash_nodes)
-    {
-        std::vector<flatbuffers::Offset<Hpfs_FS_Hash_Entry>> fbvec;
-        fbvec.reserve(hash_nodes.size());
-        for (auto const &hash_node : hash_nodes)
-        {
-            flatbuffers::Offset<Hpfs_FS_Hash_Entry> hpfs_fs_entry = CreateHpfs_FS_Hash_Entry(
-                builder,
-                sv_to_flatbuff_str(builder, hash_node.name),
-                hash_node.is_file,
-                hash_to_flatbuff_bytes(builder, hash_node.hash));
-
-            fbvec.push_back(hpfs_fs_entry);
-        }
-        return builder.CreateVector(fbvec);
-    }
-
-    /**
-     * Create peer list message from the given vector of peer properties structs.
-     * @param container_builder Flatbuffer builder for the container message.
-     * @param peers The Vector of peer properties to be placed in the container message.
-     * @param skipping_peer Peer that does not need to be sent.
-     */
-    const flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<Peer_Properties>>>
-    peer_propertiesvector_to_flatbuf_peer_propertieslist(flatbuffers::FlatBufferBuilder &builder, const std::vector<conf::peer_properties> &peers, const std::optional<conf::peer_ip_port> &skipping_ip_port)
-    {
-        std::vector<flatbuffers::Offset<Peer_Properties>> fbvec;
-        fbvec.reserve(peers.size());
-        for (auto peer : peers)
-        {
-            // Skipping the requestedc peer from the peer list response.
-            if (!skipping_ip_port.has_value() || peer.ip_port != skipping_ip_port.value())
-                fbvec.push_back(CreatePeer_Properties(
-                    builder,
-                    sv_to_flatbuff_str(builder, peer.ip_port.host_address),
-                    peer.ip_port.port,
-                    peer.available_capacity,
-                    peer.timestamp));
-        }
-        return builder.CreateVector(fbvec);
-    }
-
-    /**
-     * Create vector of peer properties structs from the given peer list message.
-     * @param fbvec The peer list message to be convert to a list of peer properties structs.
-     */
-    const std::vector<conf::peer_properties>
-    flatbuf_peer_propertieslist_to_peer_propertiesvector(const flatbuffers::Vector<flatbuffers::Offset<Peer_Properties>> *fbvec)
-    {
-        std::vector<conf::peer_properties> peers;
-
-        for (const Peer_Properties *peer : *fbvec)
-        {
-            conf::peer_properties properties;
-
-            properties.ip_port.host_address = flatbuff_str_to_sv(peer->host_address());
-            properties.ip_port.port = peer->port();
-            properties.timestamp = peer->timestamp();
-            properties.available_capacity = peer->available_capacity();
-
-            peers.push_back(properties);
-        }
-        return peers;
-    }
-} // namespace msg::fbuf::p2pmsg

src/msg/fbuf/p2pmsg_helpers.hpp

@@ -1,106 +0,0 @@
-#ifndef _HP_MSG_FBUF_P2PMSG_HELPERS_
-#define _HP_MSG_FBUF_P2PMSG_HELPERS_
-
-#include "../../pchheader.hpp"
-#include "../../p2p/p2p.hpp"
-#include "../../util/h32.hpp"
-#include "../../hpfs/hpfs_mount.hpp"
-#include "p2pmsg_container_generated.h"
-#include "p2pmsg_content_generated.h"
-
-namespace msg::fbuf::p2pmsg
-{
-    /**
- * This section contains Flatbuffer p2p message reading/writing helpers.
- */
-
-    //---Message validation helpers---/
-
-    int validate_and_extract_container(const Container **container_ref, std::string_view container_buf);
-
-    int validate_container_trust(const Container *container);
-
-    int validate_and_extract_content(const Content **content_ref, const uint8_t *content_ptr, const flatbuffers::uoffset_t content_size);
-
-    //---Message reading helpers---/
-
-    const p2p::peer_challenge get_peer_challenge_from_msg(const Peer_Challenge_Message &msg);
-
-    const p2p::peer_challenge_response create_peer_challenge_response_from_msg(const Peer_Challenge_Response_Message &msg, const flatbuffers::Vector<uint8_t> *pubkey);
-
-    const p2p::nonunl_proposal create_nonunl_proposal_from_msg(const NonUnl_Proposal_Message &msg, const uint64_t timestamp);
-
-    const p2p::proposal create_proposal_from_msg(const Proposal_Message &msg, const flatbuffers::Vector<uint8_t> *pubkey, const uint64_t timestamp, const Sequence_Hash &last_primary_shard_id_msg);
-
-    const p2p::hpfs_request create_hpfs_request_from_msg(const Hpfs_Request_Message &msg);
-
-    const std::vector<conf::peer_properties> create_peer_list_response_from_msg(const Peer_List_Response_Message &msg);
-
-    //---Message creation helpers---//
-    void create_peer_challenge_response_from_challenge(flatbuffers::FlatBufferBuilder &container_builder, const std::string &challenge);
-
-    void create_msg_from_peer_challenge(flatbuffers::FlatBufferBuilder &container_builder, std::string &challenge);
-
-    void create_msg_from_nonunl_proposal(flatbuffers::FlatBufferBuilder &container_builder, const p2p::nonunl_proposal &nup);
-
-    void create_msg_from_proposal(flatbuffers::FlatBufferBuilder &container_builder, const p2p::proposal &p);
-
-    void create_msg_from_npl_output(flatbuffers::FlatBufferBuilder &container_builder, const std::string_view &msg, const p2p::sequence_hash &last_primary_shard_id);
-
-    void create_msg_from_hpfs_request(flatbuffers::FlatBufferBuilder &container_builder, const p2p::hpfs_request &hr, const p2p::sequence_hash &last_primary_shard_id);
-
-    void create_msg_from_fsentry_response(
-        flatbuffers::FlatBufferBuilder &container_builder, const std::string_view path, const uint32_t mount_id,
-        std::vector<hpfs::child_hash_node> &hash_nodes, util::h32 expected_hash, const p2p::sequence_hash &last_primary_shard_id);
-
-    void create_msg_from_filehashmap_response(
-        flatbuffers::FlatBufferBuilder &container_builder, std::string_view path, const uint32_t mount_id,
-        std::vector<util::h32> &hashmap, std::size_t file_length, util::h32 expected_hash, const p2p::sequence_hash &last_primary_shard_id);
-
-    void create_msg_from_block_response(flatbuffers::FlatBufferBuilder &container_builder, p2p::block_response &block_resp, const uint32_t mount_id,
-                                        const p2p::sequence_hash &last_primary_shard_id);
-
-    void create_containermsg_from_content(
-        flatbuffers::FlatBufferBuilder &container_builder, const flatbuffers::FlatBufferBuilder &content_builder, const p2p::sequence_hash &last_primary_shard_id, const bool sign);
-
-    void create_msg_from_peer_requirement_announcement(flatbuffers::FlatBufferBuilder &container_builder, const bool need_consensus_msg_forwarding,
-                                                       const p2p::sequence_hash &last_primary_shard_id);
-
-    void create_msg_from_available_capacity_announcement(flatbuffers::FlatBufferBuilder &container_builder, const int16_t &available_capacity, const uint64_t &timestamp,
-                                                         const p2p::sequence_hash &last_primary_shard_id);
-
-    void create_msg_from_peer_list_request(flatbuffers::FlatBufferBuilder &container_builder, const p2p::sequence_hash &last_primary_shard_id);
-
-    void create_msg_from_peer_list_response(flatbuffers::FlatBufferBuilder &container_builder, const std::vector<conf::peer_properties> &peers, const std::optional<conf::peer_ip_port> &skipping_ip_port,
-                                            const p2p::sequence_hash &last_primary_shard_id);
-
-    //---Conversion helpers from flatbuffers data types to std data types---//
-
-    const std::unordered_map<std::string, std::list<usr::submitted_user_input>>
-    flatbuf_user_input_group_to_user_input_map(const flatbuffers::Vector<flatbuffers::Offset<UserInputGroup>> *fbvec);
-
-    //---Conversion helpers from std data types to flatbuffers data types---//
-
-    const flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<UserInputGroup>>>
-    user_input_map_to_flatbuf_user_input_group(flatbuffers::FlatBufferBuilder &builder, const std::unordered_map<std::string, std::list<usr::submitted_user_input>> &map);
-
-    const std::vector<conf::peer_properties>
-    flatbuf_peer_propertieslist_to_peer_propertiesvector(const flatbuffers::Vector<flatbuffers::Offset<Peer_Properties>> *fbvec);
-
-    const flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<Peer_Properties>>>
-    peer_propertiesvector_to_flatbuf_peer_propertieslist(flatbuffers::FlatBufferBuilder &builder, const std::vector<conf::peer_properties> &peers, const std::optional<conf::peer_ip_port> &skipping_ip_port);
-
-    void flatbuf_hpfsfshashentry_to_hpfsfshashentry(std::unordered_map<std::string, p2p::hpfs_fs_hash_entry> &fs_entries,
-                                                    const flatbuffers::Vector<flatbuffers::Offset<Hpfs_FS_Hash_Entry>> *fhashes);
-
-    void hpfsfilehash_to_flatbuf_hpfsfilehash(flatbuffers::FlatBufferBuilder &builder, std::vector<flatbuffers::Offset<Hpfs_FS_Hash_Entry>> &list,
-                                              std::string_view full_path, bool is_file, std::string_view hash);
-
-    flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<Hpfs_FS_Hash_Entry>>>
-    hpfsfshashentry_to_flatbuff_hpfsfshashentry(
-        flatbuffers::FlatBufferBuilder &builder,
-        std::vector<hpfs::child_hash_node> &hash_nodes);
-
-} // namespace msg::fbuf::p2pmsg
-
-#endif

src/p2p/p2p.cpp

@@ -3,12 +3,15 @@
 #include "../crypto.hpp"
 #include "../util/util.hpp"
 #include "../hplog.hpp"
-#include "../msg/fbuf/p2pmsg_helpers.hpp"
+#include "../msg/fbuf/common_helpers.hpp"
+#include "../msg/fbuf/p2pmsg_conversion.hpp"
 #include "../ledger/ledger.hpp"
 #include "p2p.hpp"
 #include "self_node.hpp"
 #include "../unl.hpp"
 
+namespace p2pmsg = msg::fbuf::p2pmsg;
+
 namespace p2p
 {
 
@@ -170,10 +173,7 @@ namespace p2p
      */
     void broadcast_message(const flatbuffers::FlatBufferBuilder &fbuf, const bool send_to_self, const bool is_msg_forwarding, const bool unl_only)
     {
-        std::string_view msg = std::string_view(
-            reinterpret_cast<const char *>(fbuf.GetBufferPointer()), fbuf.GetSize());
-
-        broadcast_message(msg, send_to_self, is_msg_forwarding, unl_only);
+        broadcast_message(msg::fbuf::builder_to_string_view(fbuf), send_to_self, is_msg_forwarding, unl_only);
     }
 
     /**
@@ -206,11 +206,11 @@ namespace p2p
 
     /**
      * Check whether the given message is qualified to be forwarded to peers.
-     * @param container The message container.
-     * @param content_message_type The message type.
+     * @param msg_type The message type.
+     * @param originated_on The originated epoch of the received message.
      * @return Returns true if the message is qualified for forwarding to peers. False otherwise.
     */
-    bool validate_for_peer_msg_forwarding(const peer_comm_session &session, const msg::fbuf::p2pmsg::Container *container, const msg::fbuf::p2pmsg::Message &content_message_type)
+    bool validate_for_peer_msg_forwarding(const peer_comm_session &session, const enum msg::fbuf::p2pmsg::P2PMsgContent msg_type, const uint64_t originated_on)
     {
         // Checking whether the message forwarding is enabled.
         if (!conf::cfg.mesh.msg_forwarding)
@@ -218,20 +218,22 @@ namespace p2p
             return false;
         }
 
+        // Only the selected types of messages are forwarded.
+        if (msg_type == p2pmsg::P2PMsgContent_ProposalMsg ||
+            msg_type == p2pmsg::P2PMsgContent_NonUnlProposalMsg ||
+            msg_type == p2pmsg::P2PMsgContent_NplMsg)
+        {
+            return true;
+        }
+
         const uint64_t time_now = util::get_epoch_milliseconds();
-        // Checking the time to live of the container. The time to live for forwarding is three times the round time.
-        if (container->timestamp() < (time_now - (conf::cfg.contract.roundtime * 3)))
+        // Checking the time to live of the message. The time to live for forwarding is three times the round time.
+        if (originated_on < (time_now - (conf::cfg.contract.roundtime * 3)))
         {
             LOG_DEBUG << "Peer message is too old for forwarding.";
             return false;
         }
-        // Only the selected types of messages are forwarded.
-        if (content_message_type == msg::fbuf::p2pmsg::Message_Proposal_Message ||
-            content_message_type == msg::fbuf::p2pmsg::Message_NonUnl_Proposal_Message ||
-            content_message_type == msg::fbuf::p2pmsg::Message_Npl_Message)
-        {
-            return true;
-        }
+
         return false;
     }
 
@@ -272,15 +274,51 @@ namespace p2p
 
             //send message to selected peer.
             peer_comm_session *session = it->second;
-            std::string_view msg = std::string_view(
-                reinterpret_cast<const char *>(fbuf.GetBufferPointer()), fbuf.GetSize());
-
-            session->send(msg);
+            session->send(msg::fbuf::builder_to_string_view(fbuf));
             target_pubkey = session->uniqueid;
             break;
         }
     }
 
+    /**
+     * Handle proposal message. This is called from peer and self message handlers.
+    */
+    void handle_proposal_message(const p2p::proposal &p)
+    {
+        // Check the cap and insert proposal with lock.
+        std::scoped_lock<std::mutex> lock(ctx.collected_msgs.proposals_mutex);
+
+        // If max number of proposals reached skip the rest.
+        if (ctx.collected_msgs.proposals.size() == p2p::PROPOSAL_LIST_CAP)
+            LOG_DEBUG << "Proposal rejected. Maximum proposal count reached.";
+        else
+            ctx.collected_msgs.proposals.push_back(std::move(p));
+    }
+
+    /**
+     * Handle nonunl proposal message. This is called from peer and self message handlers.
+    */
+    void handle_nonunl_proposal_message(const p2p::nonunl_proposal &nup)
+    {
+        // Check the cap and insert proposal with lock.
+        std::scoped_lock<std::mutex> lock(ctx.collected_msgs.nonunl_proposals_mutex);
+
+        // If max number of nonunl proposals reached skip the rest.
+        if (ctx.collected_msgs.nonunl_proposals.size() == p2p::NONUNL_PROPOSAL_LIST_CAP)
+            LOG_DEBUG << "Nonunl proposal rejected. Maximum nonunl proposal count reached. self";
+        else
+            ctx.collected_msgs.nonunl_proposals.push_back(std::move(nup));
+    }
+
+    /**
+     * Handle npl message. This is called from peer and self message handlers.
+     */
+    void handle_npl_message(const p2p::npl_message &npl)
+    {
+        if (!consensus::push_npl_message(npl))
+            LOG_DEBUG << "NPL message from self enqueue failure.";
+    }
+
     /**
      * Sends the peer requirement to the given peer session. If a session is not given, broadcast to all the connected peers.
      * @param need_consensus_msg_forwarding True if the number of connections are below the threshold value.
@@ -288,18 +326,12 @@ namespace p2p
      */
     void send_peer_requirement_announcement(const bool need_consensus_msg_forwarding, peer_comm_session *session)
     {
-        flatbuffers::FlatBufferBuilder fbuf(1024);
-        msg::fbuf::p2pmsg::create_msg_from_peer_requirement_announcement(fbuf, need_consensus_msg_forwarding, ledger::ctx.get_last_primary_shard_id());
+        flatbuffers::FlatBufferBuilder fbuf;
+        p2pmsg::create_msg_from_peer_requirement_announcement(fbuf, need_consensus_msg_forwarding);
         if (session)
-        {
-            std::string_view msg = std::string_view(
-                reinterpret_cast<const char *>(fbuf.GetBufferPointer()), fbuf.GetSize());
-            session->send(msg);
-        }
+            session->send(msg::fbuf::builder_to_string_view(fbuf));
         else
-        {
             broadcast_message(fbuf, false);
-        }
     }
 
     /**
@@ -309,8 +341,8 @@ namespace p2p
     void send_available_capacity_announcement(const int16_t &available_capacity)
     {
         const uint64_t time_now = util::get_epoch_milliseconds();
-        flatbuffers::FlatBufferBuilder fbuf(1024);
-        msg::fbuf::p2pmsg::create_msg_from_available_capacity_announcement(fbuf, available_capacity, time_now, ledger::ctx.get_last_primary_shard_id());
+        flatbuffers::FlatBufferBuilder fbuf;
+        p2pmsg::create_msg_from_available_capacity_announcement(fbuf, available_capacity, time_now);
         broadcast_message(fbuf, false);
     }
 
@@ -320,11 +352,9 @@ namespace p2p
      */
     void send_known_peer_list(peer_comm_session *session)
     {
-        flatbuffers::FlatBufferBuilder fbuf(1024);
-        msg::fbuf::p2pmsg::create_msg_from_peer_list_response(fbuf, ctx.server->req_known_remotes, session->known_ipport, ledger::ctx.get_last_primary_shard_id());
-        std::string_view msg = std::string_view(
-            reinterpret_cast<const char *>(fbuf.GetBufferPointer()), fbuf.GetSize());
-        session->send(msg);
+        flatbuffers::FlatBufferBuilder fbuf;
+        p2pmsg::create_msg_from_peer_list_response(fbuf, ctx.server->req_known_remotes, session->known_ipport);
+        session->send(msg::fbuf::builder_to_string_view(fbuf));
     }
 
     /**
@@ -354,8 +384,8 @@ namespace p2p
      */
     void send_peer_list_request()
     {
-        flatbuffers::FlatBufferBuilder fbuf(1024);
-        msg::fbuf::p2pmsg::create_msg_from_peer_list_request(fbuf, ledger::ctx.get_last_primary_shard_id());
+        flatbuffers::FlatBufferBuilder fbuf;
+        p2pmsg::create_msg_from_peer_list_request(fbuf);
         std::string target_pubkey;
         send_message_to_random_peer(fbuf, target_pubkey);
         LOG_DEBUG << "Peer list request: Requesting from [" << target_pubkey.substr(0, 10) << "]";

src/p2p/p2p.hpp

@@ -5,7 +5,7 @@
 #include "../usr/user_input.hpp"
 #include "../util/h32.hpp"
 #include "../conf.hpp"
-#include "../msg/fbuf/p2pmsg_container_generated.h"
+#include "../msg/fbuf/p2pmsg_generated.h"
 #include "peer_comm_server.hpp"
 #include "peer_comm_session.hpp"
 #include "peer_session_handler.hpp"
@@ -85,18 +85,22 @@ namespace p2p
         std::string pubkey;
     };
 
-    enum LEDGER_RESPONSE_ERROR
+    struct peer_capacity_announcement
     {
-        NONE = 0,
-        INVALID_MIN_LEDGER = 1,
-        REQ_LEDGER_NOT_FOUND = 2
+        int16_t available_capacity = 0;
+        uint64_t timestamp = 0;
+    };
+
+    struct peer_requirement_announcement
+    {
+        bool need_consensus_msg_forwarding = false;
     };
 
     // Represents an NPL message sent by a peer.
     struct npl_message
     {
-        std::string pubkey;                       // Peer binary pubkey.
-        p2p::sequence_hash last_primary_shard_id; // Last primary shard of the peer.
+        std::string pubkey;        // Peer binary pubkey.
+        p2p::sequence_hash lcl_id; // lcl of the peer.
         std::string data;
     };
 
@@ -127,6 +131,13 @@ namespace p2p
         util::h32 hash;        // Hash of the bloc data.
     };
 
+    struct peer_message_info
+    {
+        const msg::fbuf::p2pmsg::P2PMsg *p2p_msg = NULL;
+        const enum msg::fbuf::p2pmsg::P2PMsgContent type = msg::fbuf::p2pmsg::P2PMsgContent_NONE;
+        const uint64_t originated_on = 0;
+    };
+
     struct message_collection
     {
         std::list<proposal> proposals;
@@ -183,7 +194,13 @@ namespace p2p
 
     void send_message_to_random_peer(const flatbuffers::FlatBufferBuilder &fbuf, std::string &target_pubkey);
 
-    bool validate_for_peer_msg_forwarding(const peer_comm_session &session, const msg::fbuf::p2pmsg::Container *container, const msg::fbuf::p2pmsg::Message &content_message_type);
+    void handle_proposal_message(const p2p::proposal &p);
+
+    void handle_nonunl_proposal_message(const p2p::nonunl_proposal &nup);
+
+    void handle_npl_message(const p2p::npl_message &npl);
+
+    bool validate_for_peer_msg_forwarding(const peer_comm_session &session, const enum msg::fbuf::p2pmsg::P2PMsgContent msg_type, const uint64_t originated_on);
 
     void send_peer_requirement_announcement(const bool need_consensus_msg_forwarding, peer_comm_session *session = NULL);
 

src/p2p/peer_comm_server.cpp

@@ -1,6 +1,5 @@
 #include "../comm/comm_server.hpp"
 #include "../util/util.hpp"
-#include "../msg/fbuf/p2pmsg_helpers.hpp"
 #include "../ledger/ledger.hpp"
 #include "../unl.hpp"
 #include "../conf.hpp"

src/p2p/peer_session_handler.cpp

@@ -5,9 +5,8 @@
 #include "../util/util.hpp"
 #include "../util/rollover_hashset.hpp"
 #include "../hplog.hpp"
-#include "../msg/fbuf/p2pmsg_container_generated.h"
-#include "../msg/fbuf/p2pmsg_content_generated.h"
-#include "../msg/fbuf/p2pmsg_helpers.hpp"
+#include "../msg/fbuf/p2pmsg_generated.h"
+#include "../msg/fbuf/p2pmsg_conversion.hpp"
 #include "../msg/fbuf/common_helpers.hpp"
 #include "../ledger/ledger.hpp"
 #include "peer_comm_session.hpp"
@@ -36,7 +35,7 @@ namespace p2p
         }
 
         // Send peer challenge.
-        flatbuffers::FlatBufferBuilder fbuf(1024);
+        flatbuffers::FlatBufferBuilder fbuf;
         p2pmsg::create_msg_from_peer_challenge(fbuf, session.issued_challenge);
         std::string_view msg = std::string_view(
             reinterpret_cast<const char *>(fbuf.GetBufferPointer()), fbuf.GetSize());
@@ -54,37 +53,35 @@ namespace p2p
         // Adding message size to peer message characters(bytes) per minute counter.
         session.increment_metric(comm::SESSION_THRESHOLDS::MAX_RAWBYTES_PER_MINUTE, message.size());
 
-        const p2pmsg::Container *container;
-        if (p2pmsg::validate_and_extract_container(&container, message) != 0)
+        if (!p2pmsg::verify_peer_message(message))
+        {
+            session.increment_metric(comm::SESSION_THRESHOLDS::MAX_BADMSGS_PER_MINUTE, 1);
+            LOG_DEBUG << "Flatbuffer verify: Bad peer message.";
             return 0;
+        }
 
-        //Get serialised message content.
-        const flatbuffers::Vector<uint8_t> *container_content = container->content();
+        const peer_message_info mi = p2pmsg::get_peer_message_info(message);
 
-        //Accessing message content and size.
-        const uint8_t *content_ptr = container_content->Data();
-        const flatbuffers::uoffset_t content_size = container_content->size();
-
-        const p2pmsg::Content *content;
-        if (p2pmsg::validate_and_extract_content(&content, content_ptr, content_size) != 0)
+        if (mi.type == p2pmsg::P2PMsgContent_NONE)
+        {
+            session.increment_metric(comm::SESSION_THRESHOLDS::MAX_BADMSGS_PER_MINUTE, 1);
+            LOG_DEBUG << "Received invalid peer message type. " << session.display_name();
             return 0;
-
-        if (!recent_peermsg_hashes.try_emplace(crypto::get_hash(message)))
+        }
+        else if (!recent_peermsg_hashes.try_emplace(crypto::get_hash(message)))
         {
             session.increment_metric(comm::SESSION_THRESHOLDS::MAX_DUPMSGS_PER_MINUTE, 1);
             LOG_DEBUG << "Duplicate peer message. " << session.display_name();
             return 0;
         }
 
-        const p2pmsg::Message content_message_type = content->message_type(); //i.e - proposal, npl, state request, state response, etc
-
         // Check whether the message is qualified for message forwarding.
-        if (p2p::validate_for_peer_msg_forwarding(session, container, content_message_type))
+        if (p2p::validate_for_peer_msg_forwarding(session, mi.type, mi.originated_on))
         {
             // Npl messages and consensus proposals are forwarded only to unl nodes if relavent flags (npl and consensus) are set to private.
             // If consensus and npl flags are public, these messages are forward to all the connected nodes.
-            const bool unl_only = (!conf::cfg.contract.is_npl_public && content_message_type == p2pmsg::Message_Npl_Message) ||
-                                  (!conf::cfg.contract.is_consensus_public && content_message_type == p2pmsg::Message_Proposal_Message);
+            const bool unl_only = (!conf::cfg.contract.is_npl_public && mi.type == p2pmsg::P2PMsgContent_NplMsg) ||
+                                  (!conf::cfg.contract.is_consensus_public && mi.type == p2pmsg::P2PMsgContent_ProposalMsg);
             if (session.need_consensus_msg_forwarding)
             {
                 // Forward messages received by weakly connected nodes to other peers.
@@ -97,9 +94,9 @@ namespace p2p
             }
         }
 
-        if (content_message_type == p2pmsg::Message_Peer_Challenge_Message) // message is a peer challenge announcement
+        if (mi.type == p2pmsg::P2PMsgContent_PeerChallengeMsg)
         {
-            const p2p::peer_challenge chall = p2pmsg::get_peer_challenge_from_msg(*content->message_as_Peer_Challenge_Message());
+            const p2p::peer_challenge chall = p2pmsg::create_peer_challenge_from_msg(mi);
 
             // Check whether contract ids match.
             if (chall.contract_id != conf::cfg.contract.id)
@@ -112,20 +109,15 @@ namespace p2p
             session.reported_roundtime = chall.roundtime;
 
             // Sending the challenge response to the sender.
-            flatbuffers::FlatBufferBuilder fbuf(1024);
+            flatbuffers::FlatBufferBuilder fbuf;
             p2pmsg::create_peer_challenge_response_from_challenge(fbuf, chall.challenge);
-            std::string_view msg = std::string_view(
-                reinterpret_cast<const char *>(fbuf.GetBufferPointer()), fbuf.GetSize());
-            return session.send(msg);
+            return session.send(msg::fbuf::builder_to_string_view(fbuf));
         }
-        else if (content_message_type == p2pmsg::Message_Peer_Challenge_Response_Message) // message is a peer challenge response
+        else if (mi.type == p2pmsg::P2PMsgContent_PeerChallengeResponseMsg)
         {
             // Ignore if challenge is already resolved.
             if (session.challenge_status == comm::CHALLENGE_ISSUED)
-            {
-                const p2p::peer_challenge_response challenge_resp = p2pmsg::create_peer_challenge_response_from_msg(*content->message_as_Peer_Challenge_Response_Message(), container->pubkey());
-                return p2p::resolve_peer_challenge(session, challenge_resp);
-            }
+                return p2p::resolve_peer_challenge(session, p2pmsg::create_peer_challenge_response_from_msg(mi));
         }
 
         if (session.challenge_status != comm::CHALLENGE_VERIFIED)
@@ -134,64 +126,57 @@ namespace p2p
             return 0;
         }
 
-        if (content_message_type == p2pmsg::Message_Peer_List_Response_Message) // This message is the peer list response message.
+        if (mi.type == p2pmsg::P2PMsgContent_PeerListResponseMsg)
         {
-            p2p::merge_peer_list(p2pmsg::create_peer_list_response_from_msg(*content->message_as_Peer_List_Response_Message()));
+            p2p::merge_peer_list(p2pmsg::create_peer_list_response_from_msg(mi));
         }
-        else if (content_message_type == p2pmsg::Message_Peer_List_Request_Message) // This message is the peer list request message.
+        else if (mi.type == p2pmsg::P2PMsgContent_PeerListRequestMsg)
         {
             p2p::send_known_peer_list(&session);
         }
-        else if (content_message_type == p2pmsg::Message_Available_Capacity_Announcement_Message) // This message is the available capacity announcement message.
+        else if (mi.type == p2pmsg::P2PMsgContent_PeerCapacityAnnouncementMsg)
         {
             if (session.known_ipport.has_value())
             {
-                const p2pmsg::Available_Capacity_Announcement_Message *announcement_msg = content->message_as_Available_Capacity_Announcement_Message();
-                p2p::update_known_peer_available_capacity(session.known_ipport.value(), announcement_msg->available_capacity(), announcement_msg->timestamp());
+                const p2p::peer_capacity_announcement ann = p2pmsg::create_peer_capacity_announcement_from_msg(mi);
+                p2p::update_known_peer_available_capacity(session.known_ipport.value(), ann.available_capacity, ann.timestamp);
             }
         }
-        else if (content_message_type == p2pmsg::Message_Peer_Requirement_Announcement_Message) // This message is a peer requirement announcement message.
+        else if (mi.type == p2pmsg::P2PMsgContent_PeerRequirementAnnouncementMsg)
         {
-            const p2pmsg::Peer_Requirement_Announcement_Message *announcement_msg = content->message_as_Peer_Requirement_Announcement_Message();
-            session.need_consensus_msg_forwarding = announcement_msg->need_consensus_msg_forwarding();
-            if (session.need_consensus_msg_forwarding)
-                LOG_DEBUG << "Consensus message forwaring is required for " << session.display_name();
-            else
-                LOG_DEBUG << "Consensus message forwaring is not required for " << session.display_name();
+            const p2p::peer_requirement_announcement ann = p2pmsg::create_peer_requirement_announcement_from_msg(mi);
+            session.need_consensus_msg_forwarding = ann.need_consensus_msg_forwarding;
+            LOG_DEBUG << "Peer requirement: " << session.display_name() << " consensus msg forwarding:" << ann.need_consensus_msg_forwarding;
         }
-        else if (content_message_type == p2pmsg::Message_Proposal_Message) // message is a proposal message
+        else if (mi.type == p2pmsg::P2PMsgContent_NonUnlProposalMsg)
         {
-            // We only trust proposals coming from UNL peers.
-            if (p2pmsg::validate_container_trust(container) != 0)
+            handle_nonunl_proposal_message(p2pmsg::create_nonunl_proposal_from_msg(mi));
+        }
+        else if (mi.type == p2pmsg::P2PMsgContent_ProposalMsg)
+        {
+            if (!p2pmsg::verify_proposal_msg_signature(mi))
             {
                 session.increment_metric(comm::SESSION_THRESHOLDS::MAX_BADSIGMSGS_PER_MINUTE, 1);
                 LOG_DEBUG << "Proposal rejected due to trust failure. " << session.display_name();
                 return 0;
             }
 
-            if (handle_proposal_message(container, content) != 0)
-                LOG_DEBUG << "Proposal rejected. Maximum proposal count reached. " << session.display_name();
+            handle_proposal_message(p2pmsg::create_proposal_from_msg(mi));
         }
-        else if (content_message_type == p2pmsg::Message_NonUnl_Proposal_Message) //message is a non-unl proposal message
+        else if (mi.type == p2pmsg::P2PMsgContent_NplMsg)
         {
-            if (handle_nonunl_proposal_message(container, content) != 0)
-                LOG_DEBUG << "Nonunl proposal rejected. Maximum nonunl proposal count reached. " << session.display_name();
-        }
-        else if (content_message_type == p2pmsg::Message_Npl_Message) //message is a NPL message
-        {
-            if (p2pmsg::validate_container_trust(container) != 0)
+            if (!p2pmsg::verify_npl_msg_signature(mi))
             {
                 session.increment_metric(comm::SESSION_THRESHOLDS::MAX_BADSIGMSGS_PER_MINUTE, 1);
-                LOG_DEBUG << "NPL message rejected due to trust failure. " << session.display_name();
+                LOG_DEBUG << "Npl message rejected due to trust failure. " << session.display_name();
                 return 0;
             }
 
-            handle_npl_message(container, content);
+            handle_npl_message(p2pmsg::create_npl_from_msg(mi));
         }
-        else if (content_message_type == p2pmsg::Message_Hpfs_Request_Message)
+        else if (mi.type == p2pmsg::P2PMsgContent_HpfsRequestMsg)
         {
-            const msg::fbuf::p2pmsg::Content *content = msg::fbuf::p2pmsg::GetContent(content_ptr);
-            const p2p::hpfs_request hr = p2pmsg::create_hpfs_request_from_msg(*content->message_as_Hpfs_Request_Message());
+            const p2p::hpfs_request hr = p2pmsg::create_hpfs_request_from_msg(mi);
             if (hr.mount_id == sc::contract_fs.mount_id)
             {
                 // Check the cap and insert request with lock.
@@ -199,13 +184,9 @@ namespace p2p
 
                 // If max number of state requests reached skip the rest.
                 if (ctx.collected_msgs.contract_hpfs_requests.size() < p2p::HPFS_REQ_LIST_CAP)
-                {
                     ctx.collected_msgs.contract_hpfs_requests.push_back(std::make_pair(session.pubkey, std::move(hr)));
-                }
                 else
-                {
                     LOG_DEBUG << "Hpfs contract fs request rejected. Maximum hpfs contract fs request count reached. " << session.display_name();
-                }
             }
             else if (hr.mount_id == ledger::ledger_fs.mount_id)
             {
@@ -214,52 +195,37 @@ namespace p2p
 
                 // If max number of state requests reached skip the rest.
                 if (ctx.collected_msgs.ledger_hpfs_requests.size() < p2p::HPFS_REQ_LIST_CAP)
-                {
                     ctx.collected_msgs.ledger_hpfs_requests.push_back(std::make_pair(session.pubkey, std::move(hr)));
-                }
                 else
-                {
                     LOG_DEBUG << "Hpfs ledger fs request rejected. Maximum hpfs ledger fs request count reached. " << session.display_name();
-                }
             }
         }
-        else if (content_message_type == p2pmsg::Message_Hpfs_Response_Message)
+        else if (mi.type == p2pmsg::P2PMsgContent_HpfsResponseMsg)
         {
-            const msg::fbuf::p2pmsg::Content *content = msg::fbuf::p2pmsg::GetContent(content_ptr);
-            const msg::fbuf::p2pmsg::Hpfs_Response_Message *resp_msg = content->message_as_Hpfs_Response_Message();
+            const p2pmsg::HpfsResponseMsg &resp_msg = *mi.p2p_msg->content_as_HpfsResponseMsg();
 
             // Only accept hpfs responses if hpfs fs is syncing.
-            if (sc::contract_sync_worker.is_syncing && resp_msg->mount_id() == sc::contract_fs.mount_id)
+            if (sc::contract_sync_worker.is_syncing && resp_msg.mount_id() == sc::contract_fs.mount_id)
             {
                 // Check the cap and insert state_response with lock.
                 std::scoped_lock<std::mutex> lock(ctx.collected_msgs.contract_hpfs_responses_mutex);
 
                 // If max number of state responses reached skip the rest.
                 if (ctx.collected_msgs.contract_hpfs_responses.size() < p2p::HPFS_RES_LIST_CAP)
-                {
-                    std::string response(reinterpret_cast<const char *>(content_ptr), content_size);
-                    ctx.collected_msgs.contract_hpfs_responses.push_back(std::make_pair(session.uniqueid, std::move(response)));
-                }
+                    ctx.collected_msgs.contract_hpfs_responses.push_back(std::make_pair(session.uniqueid, std::string(message)));
                 else
-                {
-                    LOG_DEBUG << "Contract hpfs response rejected. Maximum contract hpfs response count reached. " << session.display_name();
-                }
+                    LOG_DEBUG << "Contract hpfs response rejected. Maximum response count reached. " << session.display_name();
             }
-            else if (ledger::ledger_sync_worker.is_syncing && resp_msg->mount_id() == ledger::ledger_fs.mount_id)
+            else if (ledger::ledger_sync_worker.is_syncing && resp_msg.mount_id() == ledger::ledger_fs.mount_id)
             {
                 // Check the cap and insert state_response with lock.
                 std::scoped_lock<std::mutex> lock(ctx.collected_msgs.ledger_hpfs_responses_mutex);
 
                 // If max number of state responses reached skip the rest.
                 if (ctx.collected_msgs.ledger_hpfs_responses.size() < p2p::HPFS_RES_LIST_CAP)
-                {
-                    std::string response(reinterpret_cast<const char *>(content_ptr), content_size);
-                    ctx.collected_msgs.ledger_hpfs_responses.push_back(std::make_pair(session.uniqueid, std::move(response)));
-                }
+                    ctx.collected_msgs.ledger_hpfs_responses.push_back(std::make_pair(session.uniqueid, std::string(message)));
                 else
-                {
-                    LOG_DEBUG << "Ledger hpfs response rejected. Maximum ledger hpfs response count reached. " << session.display_name();
-                }
+                    LOG_DEBUG << "Ledger hpfs response rejected. Maximum response count reached. " << session.display_name();
             }
         }
         else
@@ -275,96 +241,18 @@ namespace p2p
      */
     int handle_self_message(std::string_view message)
     {
-        const p2pmsg::Container *container;
-        if (p2pmsg::validate_and_extract_container(&container, message) != 0)
-            return 0;
+        const peer_message_info mi = p2pmsg::get_peer_message_info(message);
 
-        //Get serialised message content.
-        const flatbuffers::Vector<uint8_t> *container_content = container->content();
-
-        //Accessing message content and size.
-        const uint8_t *content_ptr = container_content->Data();
-        const flatbuffers::uoffset_t content_size = container_content->size();
-
-        const p2pmsg::Content *content;
-        if (p2pmsg::validate_and_extract_content(&content, content_ptr, content_size) != 0)
-            return 0;
-
-        const p2pmsg::Message content_message_type = content->message_type(); //i.e - proposal, npl, state request, state response, etc
-
-        if (content_message_type == p2pmsg::Message_Proposal_Message) // message is a proposal message
-        {
-            if (handle_proposal_message(container, content) != 0)
-                LOG_DEBUG << "Proposal rejected. Maximum proposal count reached. self";
-        }
-        else if (content_message_type == p2pmsg::Message_NonUnl_Proposal_Message) //message is a non-unl proposal message
-        {
-            if (handle_nonunl_proposal_message(container, content) != 0)
-                LOG_DEBUG << "Nonunl proposal rejected. Maximum nonunl proposal count reached. self";
-        }
-        else if (content_message_type == p2pmsg::Message_Npl_Message) //message is a NPL message
-            handle_npl_message(container, content);
+        if (mi.type == p2pmsg::P2PMsgContent_ProposalMsg)
+            handle_proposal_message(p2pmsg::create_proposal_from_msg(mi));
+        else if (mi.type == p2pmsg::P2PMsgContent_NonUnlProposalMsg)
+            handle_nonunl_proposal_message(p2pmsg::create_nonunl_proposal_from_msg(mi));
+        else if (mi.type == p2pmsg::P2PMsgContent_NplMsg)
+            handle_npl_message(p2pmsg::create_npl_from_msg(mi));
 
         return 0;
     }
 
-    /**
-     * Handle proposal message.
-     * @param container Message container.
-     * @param content Message content.
-     * @return returns 0 if proposal is pushed to the list, otherwise -1.
-    */
-    int handle_proposal_message(const p2pmsg::Container *container, const p2pmsg::Content *content)
-    {
-        // Check the cap and insert proposal with lock.
-        std::scoped_lock<std::mutex> lock(ctx.collected_msgs.proposals_mutex);
-
-        // If max number of proposals reached skip the rest.
-        if (ctx.collected_msgs.proposals.size() == p2p::PROPOSAL_LIST_CAP)
-            return -1;
-
-        ctx.collected_msgs.proposals.push_back(
-            p2pmsg::create_proposal_from_msg(*content->message_as_Proposal_Message(), container->pubkey(), container->timestamp(), *container->last_primary_shard_id()));
-
-        return 0;
-    }
-
-    /**
-     * Handle nonunl proposal message.
-     * @param container Message container.
-     * @param content Message content.
-     * @return returns 0 if nonunl proposal is pushed to the list, otherwise -1.
-    */
-    int handle_nonunl_proposal_message(const p2pmsg::Container *container, const p2pmsg::Content *content)
-    {
-        // Check the cap and insert proposal with lock.
-        std::scoped_lock<std::mutex> lock(ctx.collected_msgs.nonunl_proposals_mutex);
-
-        // If max number of nonunl proposals reached skip the rest.
-        if (ctx.collected_msgs.nonunl_proposals.size() == p2p::NONUNL_PROPOSAL_LIST_CAP)
-            return -1;
-
-        ctx.collected_msgs.nonunl_proposals.push_back(
-            p2pmsg::create_nonunl_proposal_from_msg(*content->message_as_NonUnl_Proposal_Message(), container->timestamp()));
-
-        return 0;
-    }
-
-    void handle_npl_message(const p2pmsg::Container *container, const p2pmsg::Content *content)
-    {
-        const p2pmsg::Npl_Message *npl_p2p_msg = content->message_as_Npl_Message();
-        npl_message msg;
-        msg.data = msg::fbuf::flatbuff_bytes_to_sv(npl_p2p_msg->data());
-        msg.pubkey = msg::fbuf::flatbuff_bytes_to_sv(container->pubkey());
-        msg.last_primary_shard_id.seq_no = container->last_primary_shard_id()->shard_seq_no();
-        msg.last_primary_shard_id.hash = msg::fbuf::flatbuff_bytes_to_hash(container->last_primary_shard_id()->shard_hash());
-
-        if (!consensus::push_npl_message(msg))
-        {
-            LOG_DEBUG << "NPL message from self enqueue failure.";
-        }
-    }
-
     //peer session on message callback method
     int handle_peer_close(const p2p::peer_comm_session &session)
     {

src/p2p/peer_session_handler.hpp

@@ -2,22 +2,15 @@
 #define _HP_P2P_PEER_SESSION_HANDLER_
 
 #include "../pchheader.hpp"
-#include "../msg/fbuf/p2pmsg_container_generated.h"
-#include "../msg/fbuf/p2pmsg_content_generated.h"
 #include "peer_comm_session.hpp"
 
-namespace p2pmsg = msg::fbuf::p2pmsg;
-
 namespace p2p
 {
     int handle_peer_connect(p2p::peer_comm_session &session);
     int handle_peer_message(p2p::peer_comm_session &session, std::string_view message);
     int handle_self_message(std::string_view message);
     int handle_peer_close(const p2p::peer_comm_session &session);
-    int handle_proposal_message(const p2pmsg::Container *container, const p2pmsg::Content *content);
-    int handle_nonunl_proposal_message(const p2pmsg::Container *container, const p2pmsg::Content *content);
     void handle_peer_on_verified(p2p::peer_comm_session &session);
-    void handle_npl_message(const p2pmsg::Container *container, const p2pmsg::Content *content);
 
 } // namespace p2p
 #endif

src/sc/sc.cpp

@@ -3,7 +3,7 @@
 #include "../consensus.hpp"
 #include "../hplog.hpp"
 #include "../ledger/ledger.hpp"
-#include "../msg/fbuf/p2pmsg_helpers.hpp"
+#include "../msg/fbuf/p2pmsg_conversion.hpp"
 #include "../msg/controlmsg_common.hpp"
 #include "../msg/controlmsg_parser.hpp"
 #include "../unl.hpp"
@@ -553,7 +553,7 @@ namespace sc
         p2p::npl_message npl_msg;
         if (ctx.args.npl_messages.try_dequeue(npl_msg))
         {
-            if (npl_msg.last_primary_shard_id == ctx.args.lasl_primary_shard_id)
+            if (npl_msg.lcl_id == ctx.args.lcl_id)
             {
                 const std::string pubkeyhex = util::to_hex(npl_msg.pubkey);
 
@@ -650,8 +650,8 @@ namespace sc
 
         if (!output.empty())
         {
-            flatbuffers::FlatBufferBuilder fbuf(1024);
-            msg::fbuf::p2pmsg::create_msg_from_npl_output(fbuf, output, ledger::ctx.get_last_primary_shard_id());
+            flatbuffers::FlatBufferBuilder fbuf;
+            msg::fbuf::p2pmsg::create_msg_from_npl_output(fbuf, output, ledger::ctx.get_lcl_id());
             p2p::broadcast_message(fbuf, true, false, !conf::cfg.contract.is_npl_public);
         }
     }

src/sc/sc.hpp

@@ -85,9 +85,6 @@ namespace sc
         // Current HotPocket lcl (seq no. and ledger hash hex)
         p2p::sequence_hash lcl_id;
 
-        // Current HotPocket primary shard id (struct of seq no. and ledger hash)
-        p2p::sequence_hash lasl_primary_shard_id;
-
         // State hash after execution will be copied to this (not applicable to read only mode).
         util::h32 post_execution_state_hash = util::h32_empty;
 

src/util/util.cpp

@@ -411,11 +411,34 @@ namespace util
         return fcntl(fd, F_SETLKW, &lock);
     }
 
-    /**
-     * Convert given little endian byte stream to uint64_t.
-     * @param data Byte stream to be converted.
-     * @return Returns converted uint64_t.
-    */
+    void uint32_to_bytes(uint8_t *dest, const uint32_t x)
+    {
+        dest[0] = (uint8_t)((x >> 24) & 0xff);
+        dest[1] = (uint8_t)((x >> 16) & 0xff);
+        dest[2] = (uint8_t)((x >> 8) & 0xff);
+        dest[3] = (uint8_t)((x >> 0) & 0xff);
+    }
+
+    uint32_t uint32_from_bytes(const uint8_t *data)
+    {
+        return ((uint32_t)data[0] << 24) +
+               ((uint32_t)data[1] << 16) +
+               ((uint32_t)data[2] << 8) +
+               ((uint32_t)data[3]);
+    }
+
+    void uint64_to_bytes(uint8_t *dest, const uint64_t x)
+    {
+        dest[0] = (uint8_t)((x >> 56) & 0xff);
+        dest[1] = (uint8_t)((x >> 48) & 0xff);
+        dest[2] = (uint8_t)((x >> 40) & 0xff);
+        dest[3] = (uint8_t)((x >> 32) & 0xff);
+        dest[4] = (uint8_t)((x >> 24) & 0xff);
+        dest[5] = (uint8_t)((x >> 16) & 0xff);
+        dest[6] = (uint8_t)((x >> 8) & 0xff);
+        dest[7] = (uint8_t)((x >> 0) & 0xff);
+    }
+
     uint64_t uint64_from_bytes(const uint8_t *data)
     {
         return ((uint64_t)data[0] << 56) +
@@ -428,17 +451,4 @@ namespace util
                ((uint64_t)data[7]);
     }
 
-    /**
-     * Convert given uint64_t to little endian byte stream.
-     * @param dest Byte stream to be populated.
-     * @param x uint64_t to be converted.
-    */
-    void uint64_to_bytes(uint8_t *dest, uint64_t x)
-    {
-        for (int j = 0; j < 8; j++)
-        {
-            *(dest + (7 - j)) = x & 0xff;
-            x >>= 8;
-        }
-    }
 } // namespace util

src/util/util.hpp

@@ -18,17 +18,6 @@ namespace util
     // Minimum compatible config version (this will be used to validate configs)
     constexpr const char *MIN_CONFIG_VERSION = "0.1";
 
-    // Current version of the peer message protocol.
-    constexpr uint8_t PEERMSG_VERSION = 1;
-
-    // Minimum compatible peer message version (this will be used to accept/reject incoming peer connections)
-    // (Keeping this as int for effcient msg payload and comparison)
-    constexpr uint8_t MIN_PEERMSG_VERSION = 1;
-
-    // Minimum compatible npl contract input version (this will be used to generate the npl input to feed the contract)
-    // (Keeping this as int for effcient msg payload and comparison)
-    constexpr uint8_t MIN_NPL_INPUT_VERSION = 1;
-
     /**
      * The messaging protocol used in a web socket channel.
      */
@@ -86,9 +75,11 @@ namespace util
 
     int release_lock(const int fd, struct flock &lock);
 
+    void uint32_to_bytes(uint8_t *dest, const uint32_t x);
+    uint32_t uint32_from_bytes(const uint8_t *data);
+    void uint64_to_bytes(uint8_t *dest, const uint64_t x);
     uint64_t uint64_from_bytes(const uint8_t *data);
 
-    void uint64_to_bytes(uint8_t *dest, const uint64_t x);
 } // namespace util
 
 #endif