Refactoring hpfs, hpfs sync and hpfs serve code. (#231)

* Refactoring hpfs code to a class so it can support multiple mounts.
* Refactoring hpfs serve into a class to support mulitiple mount serving.
* Refactoring hpfs sync into class to support multiple instances.
* Code improvements in hpfs_sync.
* Taking a sync target list for hpfs syncing target set.

src/msg/fbuf/p2pmsg_content.fbs

@@ -93,6 +93,7 @@ table HistoryLedgerBlock {
 }
 
 table Hpfs_Request_Message { //Hpfs request message schema
+    mount_id: int32;
     parent_path:string;
     is_file:bool;
     block_id:int32;
@@ -105,6 +106,7 @@ table Hpfs_Response_Message{
     hpfs_response:Hpfs_Response;
     hash:[ubyte];
     path: string;
+    mount_id: int32;
 }
 
 table Fs_Entry_Response{

src/msg/fbuf/p2pmsg_content_generated.h

@@ -1311,11 +1311,18 @@ inline flatbuffers::Offset<HistoryLedgerBlock> CreateHistoryLedgerBlockDirect(
 struct Hpfs_Request_Message FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
   typedef Hpfs_Request_MessageBuilder Builder;
   enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
-    VT_PARENT_PATH = 4,
-    VT_IS_FILE = 6,
-    VT_BLOCK_ID = 8,
-    VT_EXPECTED_HASH = 10
+    VT_MOUNT_ID = 4,
+    VT_PARENT_PATH = 6,
+    VT_IS_FILE = 8,
+    VT_BLOCK_ID = 10,
+    VT_EXPECTED_HASH = 12
   };
+  int32_t mount_id() const {
+    return GetField<int32_t>(VT_MOUNT_ID, 0);
+  }
+  bool mutate_mount_id(int32_t _mount_id) {
+    return SetField<int32_t>(VT_MOUNT_ID, _mount_id, 0);
+  }
   const flatbuffers::String *parent_path() const {
     return GetPointer<const flatbuffers::String *>(VT_PARENT_PATH);
   }
@@ -1342,6 +1349,7 @@ struct Hpfs_Request_Message FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table
   }
   bool Verify(flatbuffers::Verifier &verifier) const {
     return VerifyTableStart(verifier) &&
+           VerifyField<int32_t>(verifier, VT_MOUNT_ID) &&
            VerifyOffset(verifier, VT_PARENT_PATH) &&
            verifier.VerifyString(parent_path()) &&
            VerifyField<uint8_t>(verifier, VT_IS_FILE) &&
@@ -1356,6 +1364,9 @@ struct Hpfs_Request_MessageBuilder {
   typedef Hpfs_Request_Message Table;
   flatbuffers::FlatBufferBuilder &fbb_;
   flatbuffers::uoffset_t start_;
+  void add_mount_id(int32_t mount_id) {
+    fbb_.AddElement<int32_t>(Hpfs_Request_Message::VT_MOUNT_ID, mount_id, 0);
+  }
   void add_parent_path(flatbuffers::Offset<flatbuffers::String> parent_path) {
     fbb_.AddOffset(Hpfs_Request_Message::VT_PARENT_PATH, parent_path);
   }
@@ -1381,6 +1392,7 @@ struct Hpfs_Request_MessageBuilder {
 
 inline flatbuffers::Offset<Hpfs_Request_Message> CreateHpfs_Request_Message(
     flatbuffers::FlatBufferBuilder &_fbb,
+    int32_t mount_id = 0,
     flatbuffers::Offset<flatbuffers::String> parent_path = 0,
     bool is_file = false,
     int32_t block_id = 0,
@@ -1389,12 +1401,14 @@ inline flatbuffers::Offset<Hpfs_Request_Message> CreateHpfs_Request_Message(
   builder_.add_expected_hash(expected_hash);
   builder_.add_block_id(block_id);
   builder_.add_parent_path(parent_path);
+  builder_.add_mount_id(mount_id);
   builder_.add_is_file(is_file);
   return builder_.Finish();
 }
 
 inline flatbuffers::Offset<Hpfs_Request_Message> CreateHpfs_Request_MessageDirect(
     flatbuffers::FlatBufferBuilder &_fbb,
+    int32_t mount_id = 0,
     const char *parent_path = nullptr,
     bool is_file = false,
     int32_t block_id = 0,
@@ -1403,6 +1417,7 @@ inline flatbuffers::Offset<Hpfs_Request_Message> CreateHpfs_Request_MessageDirec
   auto expected_hash__ = expected_hash ? _fbb.CreateVector<uint8_t>(*expected_hash) : 0;
   return msg::fbuf::p2pmsg::CreateHpfs_Request_Message(
       _fbb,
+      mount_id,
       parent_path__,
       is_file,
       block_id,
@@ -1415,7 +1430,8 @@ struct Hpfs_Response_Message FLATBUFFERS_FINAL_CLASS : private flatbuffers::Tabl
     VT_HPFS_RESPONSE_TYPE = 4,
     VT_HPFS_RESPONSE = 6,
     VT_HASH = 8,
-    VT_PATH = 10
+    VT_PATH = 10,
+    VT_MOUNT_ID = 12
   };
   msg::fbuf::p2pmsg::Hpfs_Response hpfs_response_type() const {
     return static_cast<msg::fbuf::p2pmsg::Hpfs_Response>(GetField<uint8_t>(VT_HPFS_RESPONSE_TYPE, 0));
@@ -1448,6 +1464,12 @@ struct Hpfs_Response_Message FLATBUFFERS_FINAL_CLASS : private flatbuffers::Tabl
   flatbuffers::String *mutable_path() {
     return GetPointer<flatbuffers::String *>(VT_PATH);
   }
+  int32_t mount_id() const {
+    return GetField<int32_t>(VT_MOUNT_ID, 0);
+  }
+  bool mutate_mount_id(int32_t _mount_id) {
+    return SetField<int32_t>(VT_MOUNT_ID, _mount_id, 0);
+  }
   bool Verify(flatbuffers::Verifier &verifier) const {
     return VerifyTableStart(verifier) &&
            VerifyField<uint8_t>(verifier, VT_HPFS_RESPONSE_TYPE) &&
@@ -1457,6 +1479,7 @@ struct Hpfs_Response_Message FLATBUFFERS_FINAL_CLASS : private flatbuffers::Tabl
            verifier.VerifyVector(hash()) &&
            VerifyOffset(verifier, VT_PATH) &&
            verifier.VerifyString(path()) &&
+           VerifyField<int32_t>(verifier, VT_MOUNT_ID) &&
            verifier.EndTable();
   }
 };
@@ -1489,6 +1512,9 @@ struct Hpfs_Response_MessageBuilder {
   void add_path(flatbuffers::Offset<flatbuffers::String> path) {
     fbb_.AddOffset(Hpfs_Response_Message::VT_PATH, path);
   }
+  void add_mount_id(int32_t mount_id) {
+    fbb_.AddElement<int32_t>(Hpfs_Response_Message::VT_MOUNT_ID, mount_id, 0);
+  }
   explicit Hpfs_Response_MessageBuilder(flatbuffers::FlatBufferBuilder &_fbb)
         : fbb_(_fbb) {
     start_ = fbb_.StartTable();
@@ -1505,8 +1531,10 @@ inline flatbuffers::Offset<Hpfs_Response_Message> CreateHpfs_Response_Message(
     msg::fbuf::p2pmsg::Hpfs_Response hpfs_response_type = msg::fbuf::p2pmsg::Hpfs_Response_NONE,
     flatbuffers::Offset<void> hpfs_response = 0,
     flatbuffers::Offset<flatbuffers::Vector<uint8_t>> hash = 0,
-    flatbuffers::Offset<flatbuffers::String> path = 0) {
+    flatbuffers::Offset<flatbuffers::String> path = 0,
+    int32_t mount_id = 0) {
   Hpfs_Response_MessageBuilder builder_(_fbb);
+  builder_.add_mount_id(mount_id);
   builder_.add_path(path);
   builder_.add_hash(hash);
   builder_.add_hpfs_response(hpfs_response);
@@ -1519,7 +1547,8 @@ inline flatbuffers::Offset<Hpfs_Response_Message> CreateHpfs_Response_MessageDir
     msg::fbuf::p2pmsg::Hpfs_Response hpfs_response_type = msg::fbuf::p2pmsg::Hpfs_Response_NONE,
     flatbuffers::Offset<void> hpfs_response = 0,
     const std::vector<uint8_t> *hash = nullptr,
-    const char *path = nullptr) {
+    const char *path = nullptr,
+    int32_t mount_id = 0) {
   auto hash__ = hash ? _fbb.CreateVector<uint8_t>(*hash) : 0;
   auto path__ = path ? _fbb.CreateString(path) : 0;
   return msg::fbuf::p2pmsg::CreateHpfs_Response_Message(
@@ -1527,7 +1556,8 @@ inline flatbuffers::Offset<Hpfs_Response_Message> CreateHpfs_Response_MessageDir
       hpfs_response_type,
       hpfs_response,
       hash__,
-      path__);
+      path__,
+      mount_id);
 }
 
 struct Fs_Entry_Response FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {

src/msg/fbuf/p2pmsg_helpers.cpp

@@ -4,7 +4,6 @@
 #include "../../util/util.hpp"
 #include "../../hplog.hpp"
 #include "../../util/h32.hpp"
-#include "../../hpfs/hpfs.hpp"
 #include "../../unl.hpp"
 #include "p2pmsg_container_generated.h"
 #include "p2pmsg_content_generated.h"
@@ -275,7 +274,7 @@ namespace msg::fbuf::p2pmsg
     const p2p::hpfs_request create_hpfs_request_from_msg(const Hpfs_Request_Message &msg)
     {
         p2p::hpfs_request hr;
-
+        hr.mount_id = msg.mount_id();
         hr.block_id = msg.block_id();
         hr.is_file = msg.is_file();
         hr.parent_path = flatbuff_str_to_sv(msg.parent_path());
@@ -468,13 +467,14 @@ namespace msg::fbuf::p2pmsg
      * @param container_builder Flatbuffer builder for the container message.
      * @param hr The hpfs request struct to be placed in the container message.
      */
-    void create_msg_from_state_request(flatbuffers::FlatBufferBuilder &container_builder, const p2p::hpfs_request &hr, std::string_view lcl)
+    void create_msg_from_hpfs_request(flatbuffers::FlatBufferBuilder &container_builder, const p2p::hpfs_request &hr, std::string_view lcl)
     {
         flatbuffers::FlatBufferBuilder builder(1024);
 
         flatbuffers::Offset<Hpfs_Request_Message> srmsg =
             CreateHpfs_Request_Message(
                 builder,
+                hr.mount_id,
                 sv_to_flatbuff_str(builder, hr.parent_path),
                 hr.is_file,
                 hr.block_id,
@@ -497,7 +497,7 @@ namespace msg::fbuf::p2pmsg
  * @param lcl Lcl to be include in the container msg.
  */
     void create_msg_from_fsentry_response(
-        flatbuffers::FlatBufferBuilder &container_builder, const std::string_view path,
+        flatbuffers::FlatBufferBuilder &container_builder, const std::string_view path, const int32_t mount_id,
         std::vector<hpfs::child_hash_node> &hash_nodes, util::h32 expected_hash, std::string_view lcl)
     {
         flatbuffers::FlatBufferBuilder builder(1024);
@@ -511,7 +511,7 @@ namespace msg::fbuf::p2pmsg
             builder, Hpfs_Response_Fs_Entry_Response,
             resp.Union(),
             hash_to_flatbuff_bytes(builder, expected_hash),
-            sv_to_flatbuff_str(builder, path));
+            sv_to_flatbuff_str(builder, path), mount_id);
 
         flatbuffers::Offset<Content> message = CreateContent(builder, Message_Hpfs_Response_Message, st_resp.Union());
         builder.Finish(message); // Finished building message content to get serialised content.
@@ -529,7 +529,7 @@ namespace msg::fbuf::p2pmsg
  * @param lcl Lcl to be include in the container msg.
  */
     void create_msg_from_filehashmap_response(
-        flatbuffers::FlatBufferBuilder &container_builder, std::string_view path,
+        flatbuffers::FlatBufferBuilder &container_builder, std::string_view path, const int32_t mount_id,
         std::vector<util::h32> &hashmap, std::size_t file_length, util::h32 expected_hash, std::string_view lcl)
     {
         // todo:get a average propsal message size and allocate content builder based on that.
@@ -548,7 +548,7 @@ namespace msg::fbuf::p2pmsg
             Hpfs_Response_File_HashMap_Response,
             resp.Union(),
             hash_to_flatbuff_bytes(builder, expected_hash),
-            sv_to_flatbuff_str(builder, path));
+            sv_to_flatbuff_str(builder, path), mount_id);
 
         flatbuffers::Offset<Content> message = CreateContent(builder, Message_Hpfs_Response_Message, st_resp.Union());
         builder.Finish(message); // Finished building message content to get serialised content.
@@ -564,7 +564,7 @@ namespace msg::fbuf::p2pmsg
  * @param block_resp Block response struct to place in the message
  * @param lcl Lcl to be include in the container message.
  */
-    void create_msg_from_block_response(flatbuffers::FlatBufferBuilder &container_builder, p2p::block_response &block_resp, std::string_view lcl)
+    void create_msg_from_block_response(flatbuffers::FlatBufferBuilder &container_builder, p2p::block_response &block_resp, const int32_t mount_id, std::string_view lcl)
     {
         // todo:get a average propsal message size and allocate content builder based on that.
         flatbuffers::FlatBufferBuilder builder(1024);
@@ -580,7 +580,7 @@ namespace msg::fbuf::p2pmsg
             Hpfs_Response_Block_Response,
             resp.Union(),
             hash_to_flatbuff_bytes(builder, block_resp.hash),
-            sv_to_flatbuff_str(builder, block_resp.path));
+            sv_to_flatbuff_str(builder, block_resp.path), mount_id);
 
         flatbuffers::Offset<Content> message = CreateContent(builder, Message_Hpfs_Response_Message, st_resp.Union());
         builder.Finish(message); // Finished building message content to get serialised content.

src/msg/fbuf/p2pmsg_helpers.hpp

@@ -4,7 +4,7 @@
 #include "../../pchheader.hpp"
 #include "../../p2p/p2p.hpp"
 #include "../../util/h32.hpp"
-#include "../../hpfs/hpfs.hpp"
+#include "../../hpfs/hpfs_mount.hpp"
 #include "p2pmsg_container_generated.h"
 #include "p2pmsg_content_generated.h"
 
@@ -55,17 +55,17 @@ namespace msg::fbuf::p2pmsg
 
     void create_msg_from_npl_output(flatbuffers::FlatBufferBuilder &container_builder, const std::string_view &msg, std::string_view lcl);
 
-    void create_msg_from_state_request(flatbuffers::FlatBufferBuilder &container_builder, const p2p::hpfs_request &hr, std::string_view lcl);
+    void create_msg_from_hpfs_request(flatbuffers::FlatBufferBuilder &container_builder, const p2p::hpfs_request &hr, std::string_view lcl);
 
     void create_msg_from_fsentry_response(
-        flatbuffers::FlatBufferBuilder &container_builder, const std::string_view path,
+        flatbuffers::FlatBufferBuilder &container_builder, const std::string_view path, const int32_t mount_id,
         std::vector<hpfs::child_hash_node> &hash_nodes, util::h32 expected_hash, std::string_view lcl);
 
     void create_msg_from_filehashmap_response(
-        flatbuffers::FlatBufferBuilder &container_builder, std::string_view path,
+        flatbuffers::FlatBufferBuilder &container_builder, std::string_view path, const int32_t mount_id,
         std::vector<util::h32> &hashmap, std::size_t file_length, util::h32 expected_hash, std::string_view lcl);
 
-    void create_msg_from_block_response(flatbuffers::FlatBufferBuilder &container_builder, p2p::block_response &block_resp, std::string_view lcl);
+    void create_msg_from_block_response(flatbuffers::FlatBufferBuilder &container_builder, p2p::block_response &block_resp, const int32_t mount_id, std::string_view lcl);
 
     void create_containermsg_from_content(
         flatbuffers::FlatBufferBuilder &container_builder, const flatbuffers::FlatBufferBuilder &content_builder, std::string_view lcl, const bool sign);