Websocket re-architecture with websocketd and websocat (#89)

- Replaced beast with websocketd and websocat. #79 #83 #84
- Implemented inbound/outbound peer connection merging.
- Added graceful shutdown of hpcore with sigint. #87

CMakeLists.txt

@@ -49,7 +49,6 @@ target_link_libraries(hpstatemon
     backtrace
     pthread
     crypto
-    ssl
     ${CMAKE_DL_LIBS} # Needed for stacktrace support
 )
 
@@ -69,11 +68,9 @@ add_executable(hpcore
     src/statefs/hashtree_builder.cpp
     src/statefs/state_restore.cpp
     src/statefs/state_store.cpp
-    src/sock/socket_client.cpp
-    src/sock/socket_server.cpp
-    src/sock/socket_message.cpp
-    src/sock/socket_session.cpp
-    src/sock/socket_session_lambda.cpp
+    src/comm/comm_session.cpp
+    src/comm/comm_server.cpp
+    src/comm/comm_client.cpp
     src/fbschema/common_helpers.cpp
     src/fbschema/p2pmsg_helpers.cpp
     src/fbschema/ledger_helpers.cpp
@@ -98,7 +95,6 @@ target_link_libraries(hpcore
     backtrace
     pthread
     crypto
-    ssl
     ${CMAKE_DL_LIBS} # Needed for stacktrace support
 )
 add_dependencies(hpcore
@@ -106,11 +102,13 @@ add_dependencies(hpcore
     appbill
 )
 
-# add_custom_command(TARGET hpcore POST_BUILD
-#     COMMAND strip ./build/hpcore
-#     COMMAND strip ./build/hpstatemon
-#     COMMAND strip ./build/appbill
-# )
+add_custom_command(TARGET hpcore POST_BUILD
+    # COMMAND strip ./build/hpcore
+    # COMMAND strip ./build/hpstatemon
+    # COMMAND strip ./build/appbill
+    COMMAND cp ./test/bin/websocketd ./build/websocketd
+    COMMAND cp ./test/bin/websocat ./build/websocat
+)
     
 target_precompile_headers(hpstatemon PUBLIC src/pchheader.hpp)
 target_precompile_headers(hpcore REUSE_FROM hpstatemon)
@@ -120,7 +118,7 @@ target_precompile_headers(hpcore REUSE_FROM hpstatemon)
 add_custom_target(docker
   COMMAND mkdir -p ./test/local-cluster/bin
   COMMAND cp ./build/hpcore ./build/hpstatemon ./build/appbill ./test/local-cluster/bin/
-  COMMAND cp ./test/fusebin/fusermount3 ./test/fusebin/libfuse3.so.3 ./test/local-cluster/bin/
+  COMMAND cp ./test/bin/fusermount3 ./test/bin/libfuse3.so.3 ./test/bin/websocketd ./test/bin/websocat ./test/local-cluster/bin/
   COMMAND docker build -t hpcore:latest ./test/local-cluster
 )
 set_target_properties(docker PROPERTIES EXCLUDE_FROM_ALL TRUE)

README.md

@@ -9,13 +9,13 @@ A C++ version of hotpocket designed for production envrionments, original protot
 
 ## Libraries
 * Crypto - Libsodium https://github.com/jedisct1/libsodium
-* Websockets - Boost|Beast https://github.com/boostorg/beast
+* Websockets - [Websocketd](https://github.com/joewalnes/websocketd) | [Websocat](https://github.com/vi/websocat) | [netcat (OpenBSD)](https://man.openbsd.org/nc.1)
 * RapidJSON - http://rapidjson.org
 * P2P Protocol - https://google.github.io/flatbuffers/
-* TLS - https://www.openssl.org/
 * Fuse filesystem - https://github.com/libfuse/libfuse
+* Boost - https://www.boost.org/
 
-## Steps to setup Hot Pocket
+## Steps to setup Hot Pocket (For Ubuntu/Debian)
 
 #### Install CMAKE 3.16
 1. Download and extract [cmake-3.16.0-rc3-Linux-x86_64.tar.gz](https://github.com/Kitware/CMake/releases/download/v3.16.0-rc3/cmake-3.16.0-rc3-Linux-x86_64.tar.gz)
@@ -63,12 +63,6 @@ Example: When you make a change to `p2pmsg_content_.fbc` defnition file, you nee
 
 `flatc -o src/fbschema/ --gen-mutable --cpp src/fbschema/p2pmsg_content.fbs`
 
-#### Install OpenSSL
-1. Download and extract OpenSSL-1.1.1d source from [here](https://www.openssl.org/source/openssl-1.1.1d.tar.gz).
-2. Navigate to the extracted directory.
-3. Run `./config && make`
-4. Run `sudo make install`
-
 #### Install libfuse
 1. `sudo apt-get install -y meson ninja-build pkg-config`
 2. Download [libfuse 3.8](https://github.com/libfuse/libfuse/releases/download/fuse-3.8.0/fuse-3.8.0.tar.xz) and extract.
@@ -98,14 +92,14 @@ Code is divided into subsystems via namespaces.
 
 **proc::** Handles contract process execution and managing user/SC I/O and npl I/O. Makes use of **usr** and **p2p**.
 
-**usr::** Handles user connections. Makes use of **crypto** and **sock**.
+**usr::** Handles user connections. Makes use of **crypto** and **comm**.
 
-**p2p::** Handles peer-to-peer connections and message exchange between nodes. Makes use of **crypto** and **sock**.
+**p2p::** Handles peer-to-peer connections and message exchange between nodes. Makes use of **crypto** and **comm**.
 
 **cons::** Handles consensus and proposal rounds. Makes use of **usr**, **p2p** and **proc**
 
-**sock::** Handles generic web sockets functionality. Mainly acts as a wrapper for boost/beast.
+**comm::** Handles generic web sockets communication functionality. Mainly acts as a wrapper for websocketd/websocat.
 
 **util::** Contains shared data structures/helper functions used by multiple subsystems.
 
-**statefs::** Fuse-based state filesystem monitoring and contract state maintenence subsystem.
+**statefs::** Fuse-based state filesystem monitoring and contract state maintenance subsystem.

examples/hpclient/client.js

@@ -29,11 +29,11 @@ function main() {
     }
 
 
-    var server = 'wss://localhost:8080'
+    var server = 'ws://localhost:8080'
 
-    if (process.argv.length == 3) server = 'wss://localhost:' + process.argv[2]
+    if (process.argv.length == 3) server = 'ws://localhost:' + process.argv[2]
 
-    if (process.argv.length == 4) server = 'wss://' + process.argv[2] + ':' + process.argv[3]
+    if (process.argv.length == 4) server = 'ws://' + process.argv[2] + ':' + process.argv[3]
 
     var ws = new ws_api(server, {
         rejectUnauthorized: false

src/comm/comm_client.cpp

@@ -0,0 +1,96 @@
+#include "comm_client.hpp"
+#include "comm_session.hpp"
+#include "comm_session_handler.hpp"
+#include "../hplog.hpp"
+#include "../util.hpp"
+
+namespace comm
+{
+
+int comm_client::start(std::string_view host, const uint16_t port, const uint64_t (&metric_thresholds)[4], const uint64_t max_msg_size)
+{
+    return start_websocat_process(host, port);
+}
+
+void comm_client::stop()
+{
+    if (read_fd > 0)
+        close(read_fd);
+    if (write_fd > 0)
+        close(write_fd);
+
+    if (websocat_pid > 0)
+        kill(websocat_pid, SIGINT); // Kill websocat.
+}
+
+int comm_client::start_websocat_process(std::string_view host, const uint16_t port)
+{
+    // setup pipe I/O
+    if (pipe(read_pipe) < 0 || pipe(write_pipe) < 0)
+    {
+        LOG_ERR << errno << ": websocat pipe creation failed.";
+        return -1;
+    }
+
+    const pid_t pid = fork();
+
+    if (pid > 0)
+    {
+        // HotPocket process.
+        websocat_pid = pid;
+
+        read_fd = read_pipe[0];
+        write_fd = write_pipe[1];
+
+        // Close unused fds by us.
+        close(write_pipe[0]);
+        close(read_pipe[1]);
+
+        // Wait for some time and check if websocat is still running properly.
+        util::sleep(20);
+        int pid_status;
+        waitpid(websocat_pid, &pid_status, WNOHANG);
+        if (WIFEXITED(pid_status)) // This means websocat has exited.
+        {
+            close(read_fd);
+            close(write_fd);
+            return -1;
+        }
+    }
+    else if (pid == 0)
+    {
+        // Websocat process.
+        close(write_pipe[1]); //parent write
+        close(read_pipe[0]);  //parent read
+
+        dup2(write_pipe[0], STDIN_FILENO); //child read
+        close(write_pipe[0]);
+        dup2(read_pipe[1], STDOUT_FILENO); //child write
+        close(read_pipe[1]);
+
+        std::string url = std::string("wss://").append(host).append(":").append(std::to_string(port));
+
+        // Fill process args.
+        char *execv_args[] = {
+            conf::ctx.websocat_exe_path.data(),
+            url.data(),
+            (char *)"-k", // Accept invalid certificates
+            (char *)"-b", // Binary mode
+            (char *)"-E", // Close on EOF
+            (char *)"-q", // Quiet mode
+            NULL};
+
+        const int ret = execv(execv_args[0], execv_args);
+        LOG_ERR << errno << ": websocat process execv failed.";
+        exit(1);
+    }
+    else
+    {
+        LOG_ERR << "fork() failed when starting websocat process.";
+        return -1;
+    }
+
+    return 0;
+}
+
+} // namespace comm

src/comm/comm_client.hpp

@@ -0,0 +1,27 @@
+#ifndef _HP_COMM_CLIENT_
+#define _HP_COMM_CLIENT_
+
+#include "../pchheader.hpp"
+#include "comm_session.hpp"
+
+namespace comm
+{
+
+class comm_client
+{
+    pid_t websocat_pid = 0;
+    int read_pipe[2];  // parent to child pipe
+    int write_pipe[2]; // child to parent pipe
+
+    int start_websocat_process(std::string_view host, const uint16_t port);
+
+public:
+    int read_fd = 0, write_fd = 0;
+
+    int start(std::string_view host, const uint16_t port, const uint64_t (&metric_thresholds)[4], const uint64_t max_msg_size);
+    void stop();
+};
+
+} // namespace comm
+
+#endif

src/comm/comm_server.cpp

@@ -0,0 +1,400 @@
+#include <sys/socket.h>
+#include <sys/un.h>
+#include <poll.h>
+#include "comm_server.hpp"
+#include "comm_client.hpp"
+#include "comm_session.hpp"
+#include "comm_session_handler.hpp"
+#include "../hplog.hpp"
+#include "../util.hpp"
+#include "../bill/corebill.h"
+
+namespace comm
+{
+
+int comm_server::start(
+    const uint16_t port, const char *domain_socket_name, const SESSION_TYPE session_type, const bool is_binary,
+    const uint64_t (&metric_thresholds)[4], const std::set<conf::ip_port_pair> &req_known_remotes, const uint64_t max_msg_size)
+{
+    int accept_fd = open_domain_socket(domain_socket_name);
+    if (accept_fd > 0)
+    {
+        watchdog_thread = std::thread(
+            &comm_server::connection_watchdog, this, accept_fd, session_type, is_binary,
+            std::ref(metric_thresholds), req_known_remotes, max_msg_size);
+        return start_websocketd_process(port, domain_socket_name, is_binary);
+    }
+
+    return -1;
+}
+
+int comm_server::open_domain_socket(const char *domain_socket_name)
+{
+    int fd = socket(AF_UNIX, SOCK_STREAM, 0);
+    if (fd == -1)
+    {
+        LOG_ERR << errno << ": Domain socket open error";
+        return -1;
+    }
+
+    sockaddr_un addr;
+    memset(&addr, 0, sizeof(addr));
+    addr.sun_family = AF_UNIX;
+
+    strncpy(addr.sun_path, domain_socket_name, sizeof(addr.sun_path) - 1);
+    unlink(domain_socket_name);
+
+    if (bind(fd, (struct sockaddr *)&addr, sizeof(addr)) == -1)
+    {
+        LOG_ERR << errno << ": Domain socket bind error";
+        return -1;
+    }
+
+    if (listen(fd, 5) == -1)
+    {
+        LOG_ERR << errno << ": Domain socket listen error";
+        return -1;
+    }
+
+    // Set non-blocking behaviour.
+    // We do this so the accept() call returns immediately without blocking the listening thread.
+    int flags = fcntl(fd, F_GETFL);
+    fcntl(fd, F_SETFL, flags | O_NONBLOCK);
+
+    return fd; // This is the fd we should call accept() on.
+}
+
+void comm_server::connection_watchdog(
+    const int accept_fd, const SESSION_TYPE session_type, const bool is_binary,
+    const uint64_t (&metric_thresholds)[4], const std::set<conf::ip_port_pair> &req_known_remotes, const uint64_t max_msg_size)
+{
+    util::mask_signal();
+
+    // Map with read fd to connected session mappings.
+    std::unordered_map<int, comm_session> sessions;
+    // Map with read fd to connected comm client mappings.
+    std::unordered_map<int, comm_client> outbound_clients;
+
+    // Counter to track when to initiate outbound client connections.
+    int16_t loop_counter = -1;
+
+    while (true)
+    {
+        if (should_stop_listening)
+            break;
+
+        // Prepare poll fd list.
+        const size_t fd_count = sessions.size() + 1; //+1 for the inclusion of accept_fd
+        pollfd pollfds[fd_count];
+        if (poll_fds(pollfds, accept_fd, sessions) == -1)
+        {
+            util::sleep(10);
+            continue;
+        }
+
+        // Accept any new incoming connection if available.
+        check_for_new_connection(sessions, accept_fd, session_type, is_binary, metric_thresholds);
+
+        if (!req_known_remotes.empty())
+        {
+            // Restore any missing outbound connections every 500 iterations (including the first iteration).
+            if (loop_counter == -1 || loop_counter == 500)
+            {
+                loop_counter = 0;
+                maintain_known_connections(sessions, outbound_clients, req_known_remotes, session_type, is_binary, max_msg_size, metric_thresholds);
+            }
+            loop_counter++;
+        }
+
+        const size_t sessions_count = sessions.size();
+
+        // Loop through all fds and read any data.
+        for (size_t i = 1; i <= sessions_count; i++)
+        {
+            const short result = pollfds[i].revents;
+            const int fd = pollfds[i].fd;
+
+            const auto iter = sessions.find(fd);
+            if (iter != sessions.end())
+            {
+                comm_session &session = iter->second;
+                bool should_disconnect = (session.state == SESSION_STATE::CLOSED);
+
+                if (!should_disconnect)
+                {
+                    if (result & POLLIN)
+                        should_disconnect = (session.attempt_read(max_msg_size) == -1);
+
+                    if (result & (POLLERR | POLLHUP | POLLRDHUP | POLLNVAL))
+                        should_disconnect = true;
+                }
+
+                if (should_disconnect)
+                {
+                    // If this is an outbound session, cleanup the corresponding comm client as well.
+                    if (!session.is_inbound)
+                    {
+                        const auto client_itr = outbound_clients.find(fd);
+                        client_itr->second.stop();
+                        outbound_clients.erase(client_itr);
+                    }
+
+                    session.close();
+                    sessions.erase(fd);
+                }
+            }
+        }
+    }
+
+    // If we reach this point that means we are shutting down.
+
+    // Close all sessions and clients
+    for (auto &[fd, session] : sessions)
+        session.close(false);
+    for (auto &[fd, client] : outbound_clients)
+        client.stop();
+
+    LOG_INFO << (session_type == SESSION_TYPE::USER ? "User" : "Peer") << " listener stopped.";
+}
+
+int comm_server::poll_fds(pollfd *pollfds, const int accept_fd, const std::unordered_map<int, comm_session> &sessions)
+{
+    const short poll_events = POLLIN | POLLRDHUP;
+    pollfds[0].fd = accept_fd;
+
+    auto iter = sessions.begin();
+    for (size_t i = 1; i <= sessions.size(); i++)
+    {
+        pollfds[i].fd = iter->first;
+        pollfds[i].events = poll_events;
+        iter++;
+    }
+
+    if (poll(pollfds, sessions.size() + 1, 10) == -1) //10ms timeout
+    {
+        LOG_ERR << errno << ": Poll failed.";
+        return -1;
+    }
+
+    return 0;
+}
+
+void comm_server::check_for_new_connection(
+    std::unordered_map<int, comm_session> &sessions, const int accept_fd,
+    const SESSION_TYPE session_type, const bool is_binary, const uint64_t (&metric_thresholds)[4])
+{
+    // Accept new client connection (if available)
+    int client_fd = accept(accept_fd, NULL, NULL);
+    if (client_fd == -1 && errno != EAGAIN)
+    {
+        LOG_ERR << errno << ": Domain socket accept error";
+    }
+    else if (client_fd > 0)
+    {
+        // New client connected.
+        const std::string ip = get_cgi_ip(client_fd);
+
+        if (corebill::is_banned(ip))
+        {
+            LOG_DBG << "Dropping connection for banned host " << ip;
+            close(client_fd);
+        }
+        else
+        {
+            comm_session session(ip, client_fd, client_fd, session_type, is_binary, true, metric_thresholds);
+            if (session.on_connect() == 0)
+                sessions.try_emplace(client_fd, std::move(session));
+        }
+    }
+}
+
+void comm_server::maintain_known_connections(
+    std::unordered_map<int, comm_session> &sessions, std::unordered_map<int, comm_client> &outbound_clients,
+    const std::set<conf::ip_port_pair> &req_known_remotes, const SESSION_TYPE session_type, const bool is_binary,
+    const uint64_t max_msg_size, const uint64_t (&metric_thresholds)[4])
+{
+    // Find already connected known remote parties list
+    std::set<conf::ip_port_pair> known_remotes;
+    for (const auto &[fd, session] : sessions)
+    {
+        if (session.state != SESSION_STATE::CLOSED && !session.known_ipport.first.empty())
+            known_remotes.emplace(session.known_ipport);
+    }
+
+    for (const auto &ipport : req_known_remotes)
+    {
+        if (should_stop_listening)
+            break;
+
+        // Check if we are already connected to this remote party.
+        if (known_remotes.find(ipport) != known_remotes.end())
+            continue;
+
+        std::string_view host = ipport.first;
+        const uint16_t port = ipport.second;
+        LOG_DBG << "Trying to connect " << host << ":" << std::to_string(port);
+
+        comm::comm_client client;
+        if (client.start(host, port, metric_thresholds, conf::cfg.peermaxsize) == -1)
+        {
+            LOG_ERR << "Outbound connection attempt failed";
+        }
+        else
+        {
+            comm::comm_session session(host, client.read_fd, client.write_fd, comm::SESSION_TYPE::PEER, is_binary, false, metric_thresholds);
+            session.known_ipport = ipport;
+            if (session.on_connect() == 0)
+            {
+                sessions.try_emplace(client.read_fd, std::move(session));
+                outbound_clients.emplace(client.read_fd, std::move(client));
+                known_remotes.emplace(ipport);
+            }
+        }
+    }
+}
+
+int comm_server::start_websocketd_process(const uint16_t port, const char *domain_socket_name, const bool is_binary)
+{
+    // setup pipe for firewall
+    int firewall_pipe[2]; // parent to child pipe
+
+    if (pipe(firewall_pipe))
+    {
+        LOG_ERR << errno << ": pipe() call failed for firewall";
+    }
+    else
+    {
+        firewall_out = firewall_pipe[1];
+    }
+
+    const pid_t pid = fork();
+
+    if (pid > 0)
+    {
+        // HotPocket process.
+        websocketd_pid = pid;
+
+        // Close the child reading end of the pipe in the parent
+        if (firewall_out > 0)
+            close(firewall_pipe[0]);
+    }
+    else if (pid == 0)
+    {
+        // Websocketd process.
+        // We are using websocketd forked repo: https://github.com/codetsunami/websocketd
+
+        if (firewall_out > 0)
+        {
+            // Close parent writing end of the pipe in the child
+            close(firewall_pipe[1]);
+            // Override stdin in the child's file table
+            dup2(firewall_pipe[0], 0);
+        }
+
+        // Override stdout in the child's file table with /dev/null
+        //        int null_fd = open("/dev/null", O_WRONLY);
+        //        if (null_fd)
+        //            dup2(null_fd, 1);
+
+        // Fill process args.
+        char *execv_args[] = {
+            conf::ctx.websocketd_exe_path.data(),
+            (char *)"--port",
+            std::to_string(port).data(),
+            (char *)"--ssl",
+            (char *)"--sslcert",
+            conf::ctx.tls_cert_file.data(),
+            (char *)"--sslkey",
+            conf::ctx.tls_key_file.data(),
+            (char *)(is_binary ? "--binary=true" : "--binary=false"),
+            (char *)"--sizeheader=false",
+            (char *)"--loglevel=error",
+            (char *)"nc", // netcat (OpenBSD) is used for domain socket redirection.
+            (char *)"-U", // Use UNIX domain socket
+            (char *)domain_socket_name,
+            NULL};
+
+        const int ret = execv(execv_args[0], execv_args);
+        LOG_ERR << errno << ": websocketd process execv failed.";
+        exit(1);
+    }
+    else
+    {
+        LOG_ERR << "fork() failed when starting websocketd process.";
+        return -1;
+    }
+
+    return 0;
+}
+
+void comm_server::firewall_ban(std::string_view ip, const bool unban)
+{
+    if (firewall_out < 0)
+        return;
+
+    iovec iov[]{
+        {(void *)(unban ? "r" : "a"), 1},
+        {(void *)ip.data(), ip.length()}};
+    writev(firewall_out, iov, 2);
+}
+
+/**
+ * If the fd supplied was produced by accept()ing unix domain socket connection
+ * the process at the other end is inspected for CGI environment variables
+ * and the REMOTE_ADDR variable is returned as std::string, otherwise empty string
+ */
+std::string comm_server::get_cgi_ip(const int fd)
+{
+    socklen_t length;
+    ucred uc;
+    length = sizeof(struct ucred);
+
+    // Ask the operating system for information about the other process
+    if (getsockopt(fd, SOL_SOCKET, SO_PEERCRED, &uc, &length) == -1)
+    {
+        LOG_ERR << errno << ": Could not retrieve PID from unix domain socket";
+        return "";
+    }
+
+    // Open /proc/<pid>/environ for that process
+    std::stringstream ss;
+    ss << "/proc/" << uc.pid << "/environ";
+    std::string fn = ss.str();
+
+    const int envfd = open(fn.c_str(), O_RDONLY);
+    if (!envfd)
+    {
+        LOG_ERR << errno << ": Could not open environ block for process on other end of unix domain socket PID=" << uc.pid;
+        return "";
+    }
+
+    // Read environ block
+    char envblock[0x7fff];
+    const ssize_t bytes_read = read(envfd, envblock, 0x7fff); //0x7fff bytes is an operating system size limit for this block
+    close(envfd);
+
+    // Find the REMOTE_ADDR entry. Envrion block delimited by \0
+    for (char *upto = envblock, *last = envblock; upto - envblock < bytes_read; ++upto)
+    {
+        if (*upto == '\0')
+        {
+            if (upto - last > 12 && strncmp(last, "REMOTE_ADDR=", 12) == 0)
+                return std::string((const char *)(last + 12));
+            last = upto + 1;
+        }
+    }
+
+    LOG_ERR << "Could not find REMOTE_ADDR variable in /proc/" << uc.pid << "/environ";
+    return "";
+}
+
+void comm_server::stop()
+{
+    should_stop_listening = true;
+    watchdog_thread.join();
+
+    if (websocketd_pid > 0)
+        kill(websocketd_pid, SIGINT); // Kill websocketd.
+}
+
+} // namespace comm

src/comm/comm_server.hpp

@@ -0,0 +1,50 @@
+#ifndef _HP_COMM_SERVER_
+#define _HP_COMM_SERVER_
+
+#include "../pchheader.hpp"
+#include "comm_session.hpp"
+#include "comm_client.hpp"
+
+namespace comm
+{
+
+class comm_server
+{
+    pid_t websocketd_pid = 0;
+    int firewall_out = -1; // at some point we may want to listen for firewall_in but at the moment unimplemented
+    std::thread watchdog_thread;
+    bool should_stop_listening = false;
+
+    int open_domain_socket(const char *domain_socket_name);
+
+    void connection_watchdog(
+        const int accept_fd, const SESSION_TYPE session_type, const bool is_binary,
+        const uint64_t (&metric_thresholds)[4], const std::set<conf::ip_port_pair> &eq_known_remotes, const uint64_t max_msg_size);
+
+    int start_websocketd_process(const uint16_t port, const char *domain_socket_name, const bool is_binary);
+
+    int poll_fds(pollfd *pollfds, const int accept_fd, const std::unordered_map<int, comm_session> &sessions);
+
+    void check_for_new_connection(
+        std::unordered_map<int, comm_session> &sessions, const int accept_fd,
+        const SESSION_TYPE session_type, const bool is_binary, const uint64_t (&metric_thresholds)[4]);
+
+    void maintain_known_connections(
+        std::unordered_map<int, comm_session> &sessions, std::unordered_map<int, comm_client> &outbound_clients,
+        const std::set<conf::ip_port_pair> &req_known_remotes, const SESSION_TYPE session_type, const bool is_binary,
+        const uint64_t max_msg_size, const uint64_t (&metric_thresholds)[4]);
+
+    std::string get_cgi_ip(const int fd);
+
+public:
+    // Start accepting incoming connections
+    int start(
+        const uint16_t port, const char *domain_socket_name, const SESSION_TYPE session_type, const bool is_binary,
+        const uint64_t (&metric_thresholds)[4], const std::set<conf::ip_port_pair> &req_known_remotes, const uint64_t max_msg_size);
+    void stop();
+    void firewall_ban(std::string_view ip, const bool unban);
+};
+
+} // namespace comm
+
+#endif

src/comm/comm_session.cpp

@@ -0,0 +1,253 @@
+#include "../pchheader.hpp"
+#include "../usr/user_session_handler.hpp"
+#include "../p2p/peer_session_handler.hpp"
+#include "comm_session.hpp"
+#include "../hplog.hpp"
+#include "../util.hpp"
+#include "../conf.hpp"
+#include "../bill/corebill.h"
+
+namespace comm
+{
+
+constexpr uint32_t INTERVALMS = 60000;
+constexpr uint8_t SIZE_HEADER_LEN = 8;
+
+// Global instances of user and peer session handlers.
+usr::user_session_handler user_sess_handler;
+p2p::peer_session_handler peer_sess_handler;
+
+comm_session::comm_session(
+    std::string_view ip, const int read_fd, const int write_fd, const SESSION_TYPE session_type,
+    const bool is_binary, const bool is_inbound, const uint64_t (&metric_thresholds)[4])
+
+    : read_fd(read_fd),
+      write_fd(write_fd),
+      session_type(session_type),
+      uniqueid(std::to_string(read_fd).append(":").append(ip)),
+      is_binary(is_binary),
+      is_inbound(is_inbound)
+{
+    // Create new session_thresholds and insert it to thresholds vector.
+    // Have to maintain the SESSION_THRESHOLDS enum order in inserting new thresholds to thresholds vector
+    // since enum's value is used as index in the vector to update vector values.
+    thresholds.reserve(sizeof metric_thresholds);
+    for (size_t i = 0; i < sizeof metric_thresholds; i++)
+        thresholds.push_back(session_threshold(metric_thresholds[i], INTERVALMS));
+}
+
+int comm_session::on_connect()
+{
+    state = SESSION_STATE::ACTIVE;
+
+    if (session_type == SESSION_TYPE::USER)
+        return user_sess_handler.on_connect(*this);
+    else
+        return peer_sess_handler.on_connect(*this);
+}
+
+/**
+ * Attempts to read message data from the given socket fd and passes the message on to the session.
+ * @param should_disconnect Whether the client fd must be disconnected.
+ * @param max_msg_size The allowed max byte length of a message to be read.
+ */
+int comm_session::attempt_read(const uint64_t max_msg_size)
+{
+    size_t available_bytes = 0;
+    if (ioctl(read_fd, FIONREAD, &available_bytes) == -1 ||
+        (max_msg_size > 0 &&
+         available_bytes > (max_msg_size + (is_binary ? SIZE_HEADER_LEN : 0))))
+        return -1;
+
+    // Try to read a complete message using available bytes.
+    // If complete message is not available silently return.
+    if (available_bytes > 0)
+    {
+        const uint32_t read_len = is_binary ? get_binary_msg_read_len(available_bytes) : available_bytes;
+
+        if (read_len == -1)
+        {
+            return -1;
+        }
+        else if (read_len > 0)
+        {
+            available_bytes -= read_len;
+            if (is_binary)
+                available_bytes -= SIZE_HEADER_LEN;
+
+            char msg_buf[read_len];
+            if (read(read_fd, msg_buf, read_len) == -1)
+                return -1;
+
+            return on_message(std::string_view(msg_buf, read_len));
+        }
+    }
+
+    return 0;
+}
+
+int comm_session::on_message(std::string_view message)
+{
+    increment_metric(SESSION_THRESHOLDS::MAX_RAWBYTES_PER_MINUTE, message.length());
+
+    if (session_type == SESSION_TYPE::USER)
+        return user_sess_handler.on_message(*this, message);
+    else
+        return peer_sess_handler.on_message(*this, message);
+}
+
+int comm_session::send(std::string_view message) const
+{
+    if (state == SESSION_STATE::CLOSED)
+        return -1;
+
+    // Prepare the memory segments to map with writev().
+    iovec memsegs[2];
+
+    if (is_binary)
+    {
+        // In binary mode, we need to prefix every message with the message size header.
+        uint8_t header_buf[SIZE_HEADER_LEN] = {0, 0, 0, 0, 0, 0, 0, 0};
+        uint32_t len = message.length();
+        // Reserve the first 4 bytes for future (TODO).
+        header_buf[4] = len >> 24;
+        header_buf[5] = (len >> 16) & 0xff;
+        header_buf[6] = (len >> 8) & 0xff;
+        header_buf[7] = len & 0xff;
+
+        memsegs[0].iov_base = header_buf;
+        memsegs[0].iov_len = SIZE_HEADER_LEN;
+        memsegs[1].iov_base = (char *)message.data();
+        memsegs[1].iov_len = message.length();
+    }
+    else
+    {
+        // In text mode, we need to append every message with '\n'
+        memsegs[0].iov_base = (char *)message.data();
+        memsegs[0].iov_len = message.length();
+        memsegs[1].iov_base = (char *)"\n";
+        memsegs[1].iov_len = 1;
+    }
+
+    if (writev(write_fd, memsegs, 2) == -1)
+    {
+        LOG_ERR << errno << ": Session " << uniqueid << " send writev failed.";
+        return -1;
+    }
+    return 0;
+}
+
+void comm_session::close(const bool invoke_handler)
+{
+    if (state == SESSION_STATE::CLOSED)
+        return;
+
+    if (invoke_handler)
+    {
+        if (session_type == SESSION_TYPE::USER)
+            user_sess_handler.on_close(*this);
+        else
+            peer_sess_handler.on_close(*this);
+    }
+
+    ::close(read_fd);
+    state = SESSION_STATE::CLOSED;
+
+    LOG_DBG << (session_type == SESSION_TYPE::PEER ? "Peer" : "User") << " session closed: "
+            << uniqueid << (is_inbound ? "[in]" : "[out]") << (is_self ? "[self]" : "");
+}
+
+/**
+ * Retrieves the length of the binary message pending to be read. Only relevant for Binary mode.
+ * @param available_bytes Count of bytes that is available to read from the client socket.
+ * @return Length of the message if the complete message available to be read. 0 if reading must be skipped. -1 if client must be disconnected.
+ */
+uint32_t comm_session::get_binary_msg_read_len(const size_t available_bytes)
+{
+    // If we have previously encountered a size header and we are waiting until all message
+    // bytes are received, we must have the expected message size > 0.
+
+    // If we are not tracking a previous size header, then we must check for a size header.
+    if (expected_msg_size == 0 && available_bytes >= SIZE_HEADER_LEN)
+    {
+        // Read the size header.
+        uint8_t header_buf[SIZE_HEADER_LEN];
+        if (read(read_fd, header_buf, SIZE_HEADER_LEN) == -1)
+            return -1; // Indicates that we should disconnect the client.
+
+        // We are using last 4 bytes (big endian) in the header for the message size.
+        uint32_t upcoming_msg_size = (header_buf[4] << 24) + (header_buf[5] << 16) + (header_buf[6] << 8) + header_buf[7];
+
+        // We must read the entire message if all message bytes are available.
+        if (available_bytes >= (SIZE_HEADER_LEN + upcoming_msg_size))
+            return upcoming_msg_size;
+
+        // Remember the expected msg size until sufficient bytes are available.
+        expected_msg_size = upcoming_msg_size;
+    }
+    else if (expected_msg_size > 0 && available_bytes >= expected_msg_size)
+    {
+        // We know expected message size, and enough bytes are available to read complete expected message.
+        const uint32_t read_len = expected_msg_size;
+        expected_msg_size = 0; // reset the expected msg size.
+        return read_len;
+    }
+
+    // Skip reading
+    return 0;
+}
+
+/**
+ * Set thresholds to the socket session
+*/
+void comm_session::set_threshold(const SESSION_THRESHOLDS threshold_type, const uint64_t threshold_limit, const uint32_t intervalms)
+{
+    session_threshold &t = thresholds[threshold_type];
+    t.counter_value = 0;
+    t.intervalms = intervalms;
+    t.threshold_limit = threshold_limit;
+}
+
+/*
+* Increment the provided thresholds counter value with the provided amount and validate it against the
+* configured threshold limit.
+*/
+void comm_session::increment_metric(const SESSION_THRESHOLDS threshold_type, const uint64_t amount)
+{
+    session_threshold &t = thresholds[threshold_type];
+
+    // Ignore the counter if limit is set as 0.
+    if (t.threshold_limit == 0)
+        return;
+
+    const uint64_t time_now = util::get_epoch_milliseconds();
+
+    t.counter_value += amount;
+    if (t.timestamp == 0)
+    {
+        // Reset counter timestamp.
+        t.timestamp = time_now;
+    }
+    else
+    {
+        // Check whether we have exceeded the threshold within the monitering interval.
+        const uint64_t elapsed_time = time_now - t.timestamp;
+        if (elapsed_time <= t.intervalms && t.counter_value > t.threshold_limit)
+        {
+            this->close();
+
+            t.timestamp = 0;
+            t.counter_value = 0;
+
+            LOG_INFO << "Session " << this->uniqueid << " threshold exceeded. (type:" << threshold_type << " limit:" << t.threshold_limit << ")";
+            corebill::report_violation(this->address);
+        }
+        else if (elapsed_time > t.intervalms)
+        {
+            t.timestamp = time_now;
+            t.counter_value = amount;
+        }
+    }
+}
+
+} // namespace comm

src/comm/comm_session.hpp

@@ -0,0 +1,78 @@
+#ifndef _HP_COMM_SESSION_
+#define _HP_COMM_SESSION_
+
+#include "../pchheader.hpp"
+#include "comm_session_threshold.hpp"
+#include "../conf.hpp"
+
+namespace comm
+{
+
+/**
+ * Set of flags used to mark status information on the session.
+ * usr and p2p subsystems makes use of this to mark status information of user and peer sessions.
+ * Set flags are stored in 'flags' bitset of comm_session.
+ */
+enum SESSION_FLAG
+{
+    USER_CHALLENGE_ISSUED,
+    USER_AUTHED,
+    PEERID_RESOLVED
+};
+
+enum SESSION_STATE
+{
+    ACTIVE,
+    CLOSED
+};
+
+enum SESSION_TYPE
+{
+    USER = 0,
+    PEER = 1
+};
+
+/** 
+ * Represents an active WebSocket connection
+*/
+class comm_session
+{
+    const int read_fd = 0;
+    const int write_fd = 0; // Only valid for outgoing client connections.
+    const SESSION_TYPE session_type;
+    std::vector<session_threshold> thresholds; // track down various communication thresholds
+    uint32_t expected_msg_size = 0;            // Next expected message size based on size header.
+
+    uint32_t get_binary_msg_read_len(const size_t available_bytes);
+    int on_message(std::string_view message);
+
+public:
+    const std::string address; // IP address of the remote party.
+    const bool is_binary;
+    const bool is_inbound;
+    bool is_self = false;
+    std::string uniqueid;
+    std::string issued_challenge;
+    conf::ip_port_pair known_ipport;
+    SESSION_STATE state;
+
+    // The set of SESSION_FLAG enum flags that will be set by user-code of this calss.
+    // We mainly use this to store contexual information about this session based on the use case.
+    // Setting and reading flags to this is completely managed by user-code.
+    std::bitset<8> flags;
+
+    comm_session(
+        std::string_view ip, const int read_fd, const int write_fd, const SESSION_TYPE session_type,
+        const bool is_binary, const bool is_inbound, const uint64_t (&metric_thresholds)[4]);
+    int on_connect();
+    int attempt_read(const uint64_t max_msg_size);
+    int send(std::string_view message) const;
+    void close(const bool invoke_handler = true);
+
+    void set_threshold(const SESSION_THRESHOLDS threshold_type, const uint64_t threshold_limit, const uint32_t intervalms);
+    void increment_metric(const SESSION_THRESHOLDS threshold_type, const uint64_t amount);
+};
+
+} // namespace comm
+
+#endif

src/comm/comm_session_handler.hpp

@@ -0,0 +1,23 @@
+#ifndef _HP_COMM_SESSION_HANDLER_
+#define _HP_COMM_SESSION_HANDLER_
+
+#include "../pchheader.hpp"
+
+namespace comm
+{
+
+// Forward declaration
+class comm_session;
+
+class comm_session_handler
+{
+
+public:
+    int on_connect(comm_session &session) const;
+    int on_message(comm_session &session, std::string_view message) const;
+    void on_close(const comm_session &session) const;
+};
+
+} // namespace comm
+
+#endif

src/comm/comm_session_threshold.hpp

@@ -0,0 +1,49 @@
+#ifndef _HP_COMM_SESSION_THRESHOLD_
+#define _HP_COMM_SESSION_THRESHOLD_
+
+#include "../pchheader.hpp"
+
+namespace comm
+{
+
+/**
+ * Enum used to track down various thresholds used in socket communication.
+ */
+enum SESSION_THRESHOLDS
+{
+    // Max incoming bytes per minute.
+    MAX_RAWBYTES_PER_MINUTE = 0,
+
+    // Max duplicate messages per minute.
+    MAX_DUPMSGS_PER_MINUTE = 1,
+
+    // Max messages with invalid signature per minute.
+    MAX_BADSIGMSGS_PER_MINUTE = 2,
+
+    // Max messages with bad structure per minute.
+    MAX_BADMSGS_PER_MINUTE = 3
+};
+
+/*
+* Use this to keep in track of different thresholds which we need to deal with. e.g - maximum amount of bytes allowed per minute through a session
+* threshold_limit - Maximum threshold value which is allowed
+* counter_value - Counter which keeps incrementing per every message
+* timestamp - Timestamp when counter value changes
+* intervalms - Time interval in miliseconds in which the threshold and the counter value should be compared
+*/
+struct session_threshold
+{
+    uint64_t threshold_limit;
+    uint32_t intervalms;
+    uint64_t counter_value;
+    uint64_t timestamp;
+
+    session_threshold(const uint64_t threshold_limit, const uint32_t intervalms)
+        : threshold_limit(threshold_limit), intervalms(intervalms)
+    {
+    }
+};
+
+} // namespace comm
+
+#endif

src/conf.cpp

@@ -17,6 +17,7 @@ contract_config cfg;
 
 const static char *MODE_OBSERVER = "observer";
 const static char *MODE_PROPOSER = "proposer";
+constexpr size_t PEERID_LEN = 16;
 
 /**
  * Loads and initializes the contract config for execution. Must be called once during application startup.
@@ -34,8 +35,14 @@ int init()
 
     // Append self peer to peer list.
     const std::string portstr = std::to_string(cfg.peerport);
-    cfg.self_peer_id = "0.0.0.0:" + portstr;
-    cfg.peers.emplace(cfg.self_peer_id, std::make_pair("0.0.0.0", portstr));
+
+    // We calculate the self peer id to be a random string.
+    // Use libsodium to generate the random challenge bytes.
+    unsigned char peerid_bytes[PEERID_LEN];
+    randombytes_buf(peerid_bytes, PEERID_LEN);
+    util::bin2hex(cfg.self_peerid, peerid_bytes, PEERID_LEN);
+
+    cfg.peers.emplace(std::make_pair(SELF_HOST, cfg.peerport));
 
     // Append self pubkey to unl list.
     cfg.unl.emplace(cfg.pubkey);
@@ -90,7 +97,6 @@ int create_contract()
     binpair_to_hex();
 
     cfg.startup_mode = OPERATING_MODE::PROPOSER;
-    cfg.listenip = "0.0.0.0";
     cfg.peerport = 22860;
     cfg.roundtime = 1000;
     cfg.pubport = 8080;
@@ -138,6 +144,8 @@ void set_contract_dir_paths(std::string exepath, std::string basedir)
 
     ctx.exe_dir = boost::filesystem::path(util::realpath(exepath)).parent_path().string();
     ctx.statemon_exe_path = ctx.exe_dir + "/" + "hpstatemon";
+    ctx.websocketd_exe_path = ctx.exe_dir + "/" + "websocketd";
+    ctx.websocat_exe_path = ctx.exe_dir + "/" + "websocat";
 
     ctx.contract_dir = basedir;
     ctx.config_dir = basedir + "/cfg";
@@ -213,7 +221,6 @@ int load_config()
 
     cfg.pubkeyhex = d["pubkeyhex"].GetString();
     cfg.seckeyhex = d["seckeyhex"].GetString();
-    cfg.listenip = d["listenip"].GetString();
 
     cfg.binary = d["binary"].GetString();
     cfg.binargs = d["binargs"].GetString();
@@ -248,8 +255,8 @@ int load_config()
         boost::split(splitted_peers, ipport_concat, boost::is_any_of(":"));
         if (splitted_peers.size() == 2)
         {
-            // Push the peer address and the port to peers array
-            cfg.peers.emplace(std::make_pair(ipport_concat, std::make_pair(splitted_peers.front(), splitted_peers.back())));
+            // Push the peer address and the port to peers set
+            cfg.peers.emplace(std::make_pair(splitted_peers.front(), std::stoi(splitted_peers.back())));
             splitted_peers.clear();
         }
     }
@@ -320,14 +327,13 @@ int save_config()
     d.AddMember("binargs", rapidjson::StringRef(cfg.binargs.data()), allocator);
     d.AddMember("appbill", rapidjson::StringRef(cfg.appbill.data()), allocator);
     d.AddMember("appbillargs", rapidjson::StringRef(cfg.appbillargs.data()), allocator);
-    d.AddMember("listenip", rapidjson::StringRef(cfg.listenip.data()), allocator);
-    d.AddMember("listenip", rapidjson::StringRef(cfg.listenip.data()), allocator);
 
     rapidjson::Value peers(rapidjson::kArrayType);
-    for (const auto &[ipport_concat, ipport_pair] : cfg.peers)
+    for (const auto &ipport_pair : cfg.peers)
     {
         rapidjson::Value v;
-        v.SetString(rapidjson::StringRef(ipport_concat.data()), allocator);
+        const std::string concat_str = std::string(ipport_pair.first).append(":").append(std::to_string(ipport_pair.second));
+        v.SetString(rapidjson::StringRef(concat_str.data()), allocator);
         peers.PushBack(v, allocator);
     }
     d.AddMember("peers", peers, allocator);
@@ -459,7 +465,6 @@ int validate_config()
     bool fields_missing = false;
 
     fields_missing |= cfg.binary.empty() && std::cout << "Missing cfg field: binary\n";
-    fields_missing |= cfg.listenip.empty() && std::cout << "Missing cfg field: listenip\n";
     fields_missing |= cfg.peerport == 0 && std::cout << "Missing cfg field: peerport\n";
     fields_missing |= cfg.roundtime == 0 && std::cout << "Missing cfg field: roundtime\n";
     fields_missing |= cfg.pubport == 0 && std::cout << "Missing cfg field: pubport\n";
@@ -550,7 +555,7 @@ int is_schema_valid(const rapidjson::Document &d)
     const char *cfg_schema =
         "{"
         "\"type\": \"object\","
-        "\"required\": [ \"mode\", \"version\", \"pubkeyhex\", \"seckeyhex\", \"binary\", \"binargs\", \"appbill\", \"appbillargs\", \"listenip\""
+        "\"required\": [ \"mode\", \"version\", \"pubkeyhex\", \"seckeyhex\", \"binary\", \"binargs\", \"appbill\", \"appbillargs\""
         ", \"peers\", \"unl\", \"pubport\", \"peerport\", \"roundtime\""
         ", \"pubmaxsize\", \"pubmaxcpm\", \"pubmaxbadmpm\", \"pubmaxcons\""
         ", \"peermaxsize\", \"peermaxcpm\", \"peermaxdupmpm\", \"peermaxbadmpm\", \"peermaxbadsigpm\", \"peermaxcons\""
@@ -564,7 +569,6 @@ int is_schema_valid(const rapidjson::Document &d)
         "\"binargs\": { \"type\": \"string\" },"
         "\"appbill\": { \"type\": \"string\" },"
         "\"appbillargs\": { \"type\": \"string\" },"
-        "\"listenip\": { \"type\": \"string\" },"
         "\"peers\": {"
         "\"type\": \"array\","
         "\"items\": { \"type\": \"string\" }"
@@ -615,9 +619,9 @@ void change_operating_mode(const OPERATING_MODE mode)
     cfg.current_mode = mode;
 
     if (mode == OPERATING_MODE::OBSERVER)
-        LOG_DBG << "Switched to OBSERVER mode.";
+        LOG_INFO << "Switched to OBSERVER mode.";
     else
-        LOG_DBG << "Switched back to PROPOSER mode.";
+        LOG_INFO << "Switched back to PROPOSER mode.";
 }
 
 } // namespace conf

src/conf.hpp

@@ -10,8 +10,10 @@
 namespace conf
 {
 
+constexpr const char *SELF_HOST = "127.0.0.1";
+
 // Typedef to represent ip address and port pair.
-typedef std::pair<std::string, std::string> ip_port_pair;
+typedef std::pair<std::string, uint16_t> ip_port_pair;
 
 // The operating mode of the contract node.
 enum OPERATING_MODE
@@ -23,19 +25,21 @@ enum OPERATING_MODE
 // Holds contextual information about the currently loaded contract.
 struct contract_ctx
 {
-    std::string command;        // The CLI command issued to launch HotPocket
+    std::string command;         // The CLI command issued to launch HotPocket
     std::string exe_dir;         // Hot Pocket executable dir.
-    std::string statemon_exe_path;// State monitor executable file path.
+    std::string statemon_exe_path;   // State monitor executable file path.
+    std::string websocketd_exe_path; // Websocketd executable file path.
+    std::string websocat_exe_path;   // Websocketd executable file path.
 
     std::string contract_dir;    // Contract base directory full path
     std::string hist_dir;        // Contract ledger history dir full path
     std::string state_dir;       // Contract executing state dir full path (This is the fuse mount point)
-    std::string state_hist_dir;   // Contract state history dir full path
+    std::string state_hist_dir;  // Contract state history dir full path
     std::string log_dir;         // Contract log dir full path
     std::string config_dir;      // Contract config dir full path
     std::string config_file;     // Full path to the contract config file
-    std::string tls_key_file;     // Full path to the tls secret key file
-    std::string tls_cert_file;    // Full path to the tls certificate
+    std::string tls_key_file;    // Full path to the tls secret key file
+    std::string tls_cert_file;   // Full path to the tls certificate
 };
 
 // Holds all the contract config values.
@@ -48,7 +52,7 @@ struct contract_config
     std::vector<std::string> runtime_binexec_args;   // Contract binary execution args used during runtime.
     std::vector<std::string> runtime_appbill_args;   // Appbill execution args used during runtime.
     OPERATING_MODE current_mode;        // Current operating mode of the contract (Observer/Proposer)
-    std::string self_peer_id;           // Peer session id of this node. (format: selfip:port)
+    std::string self_peerid;           // Peer session id of this node. (format: selfip:port)
 
     // Config elements which are loaded from the config file.
 
@@ -59,9 +63,8 @@ struct contract_config
     std::string binargs;                // CLI arguments to pass to the contract binary
     std::string appbill;                // binary to execute for appbill
     std::string appbillargs;            // any arguments to supply to appbill binary by default
-    std::string listenip;               // The IPs to listen on for incoming connections
-    std::unordered_map<std::string, ip_port_pair> peers;    // Map of peers keyed by "<ip address>:<port>" concatenated format
-    std::unordered_set<std::string> unl;                    // Unique node list (list of binary public keys)
+    std::set<ip_port_pair> peers;       // Set of peers keyed by "<ip address>:<port>" concatenated format
+    std::unordered_set<std::string> unl;// Unique node list (list of binary public keys)
     uint16_t peerport;                  // Listening port for peer connections
     uint16_t roundtime;                 // Consensus round time in ms
     uint16_t pubport;                   // Listening port for public user connections

src/cons/cons.cpp

@@ -1,4 +1,3 @@
-#include <flatbuffers/flatbuffers.h>
 #include "../pchheader.hpp"
 #include "../conf.hpp"
 #include "../usr/usr.hpp"
@@ -62,13 +61,7 @@ int init()
         ctx.prev_hash_state = ctx.curr_hash_state;
     }
 
-    ctx.state_syncing_thread = std::thread([&] {
-        run_state_sync_iterator();
-        LOG_ERR << "Exit state sync thread\n";
-        exit(1);
-    });
-
-    ctx.prev_close_time = util::get_epoch_milliseconds();
+    ctx.state_syncing_thread = std::thread(&run_state_sync_iterator);
 
     // We allocate 1/5 of the round time to each stage expect stage 3. For stage 3 we allocate 2/5.
     // Stage 3 is allocated an extra stage_time unit becayse a node needs enough time to
@@ -79,16 +72,26 @@ int init()
     return 0;
 }
 
+/**
+ * Cleanup any resources.
+ */
+void deinit()
+{
+    ctx.is_shutting_down = true;
+    ctx.state_syncing_thread.join();
+}
+
 void consensus()
 {
     // A consensus round consists of 4 stages (0,1,2,3).
     // For a given stage, this function may get visited multiple times due to time-wait conditions.
 
-    if (!wait_and_proceed_stage())
+    uint64_t stage_start = 0;
+    if (!wait_and_proceed_stage(stage_start))
         return; // This means the stage has been reset.
 
     // Get the latest current time.
-    ctx.time_now = util::get_epoch_milliseconds();
+    ctx.time_now = stage_start;
     std::list<p2p::proposal> collected_proposals;
 
     // Throughout consensus, we move over the incoming proposals collected via the network so far into
@@ -148,17 +151,7 @@ void consensus()
     }
     else // Stage 1, 2, 3
     {
-        for (auto &[pubkey, proposal] : ctx.candidate_proposals)
-        {
-            bool self = proposal.pubkey == conf::cfg.pubkey;
-            LOG_DBG << "Proposal [stage" << std::to_string(proposal.stage)
-                    << "] users:" << proposal.users.size()
-                    << " hinp:" << proposal.hash_inputs.size()
-                    << " hout:" << proposal.hash_outputs.size()
-                    << " lcl:" << proposal.lcl.substr(0, 15)
-                    << " state:" << *reinterpret_cast<const hasher::B2H *>(proposal.curr_hash_state.c_str())
-                    << " self:" << self;
-        }
+        purify_candidate_proposals();
 
         // Initialize vote counters
         vote_counter votes;
@@ -174,6 +167,8 @@ void consensus()
 
             if (should_request_history)
             {
+                LOG_INFO << "Syncing lcl. Curr lcl:" << cons::ctx.lcl.substr(0, 15) << " majority:" << majority_lcl.substr(0, 15);
+
                 // TODO: If we are in a lcl fork condition try to rollback state with the help of
                 // state_restore to rollback state checkpoints before requesting new state.
 
@@ -212,7 +207,6 @@ void consensus()
 
                 if (ctx.stage == 3)
                 {
-                    ctx.prev_close_time = stg_prop.time;
                     apply_ledger(stg_prop);
 
                     // node has finished a consensus round (all 4 stages).
@@ -227,11 +221,43 @@ void consensus()
     ctx.stage = (ctx.stage + 1) % 4;
 }
 
+/**
+ * Cleanup any outdated proposals from the candidate set.
+ */
+void purify_candidate_proposals()
+{
+    auto itr = ctx.candidate_proposals.begin();
+    while (itr != ctx.candidate_proposals.end())
+    {
+        const p2p::proposal &cp = itr->second;
+
+        // only consider recent proposals and proposals from previous stage and current stage.
+        if ((ctx.time_now - cp.timestamp < conf::cfg.roundtime * 4) && cp.stage >= (ctx.stage - 1))
+        {
+            ++itr;
+
+            bool self = cp.pubkey == conf::cfg.pubkey;
+            LOG_DBG << "Proposal [stage" << std::to_string(cp.stage)
+                    << "] users:" << cp.users.size()
+                    << " hinp:" << cp.hash_inputs.size()
+                    << " hout:" << cp.hash_outputs.size()
+                    << " ts:" << std::to_string(cp.time)
+                    << " lcl:" << cp.lcl.substr(0, 15)
+                    << " state:" << *reinterpret_cast<const hasher::B2H *>(cp.curr_hash_state.c_str())
+                    << " self:" << self;
+        }
+        else
+        {
+            ctx.candidate_proposals.erase(itr++);
+        }
+    }
+}
+
 /**
  * Syncrhonise the stage/round time for fixed intervals and reset the stage.
  * @return True if consensus can proceed in the current round. False if stage is reset.
  */
-bool wait_and_proceed_stage()
+bool wait_and_proceed_stage(uint64_t &stage_start)
 {
     // Here, nodes try to synchronise nodes stages using network clock.
     // We devide universal time to windows of equal size of roundtime. Each round must be synced with the
@@ -246,7 +272,7 @@ bool wait_and_proceed_stage()
     if (ctx.stage == 0)
     {
         // Stage 0 must start in the next round window.
-        const uint64_t stage_start = current_round_start + conf::cfg.roundtime;
+        stage_start = current_round_start + conf::cfg.roundtime;
         const int64_t to_wait = stage_start - now;
 
         LOG_DBG << "Waiting " << std::to_string(to_wait) << "ms for next round stage 0";
@@ -255,7 +281,7 @@ bool wait_and_proceed_stage()
     }
     else
     {
-        const uint64_t stage_start = current_round_start + (ctx.stage * ctx.stage_time);
+        stage_start = current_round_start + (ctx.stage * ctx.stage_time);
 
         // Compute stage time wait.
         // Node wait between stages to collect enough proposals from previous stages from other nodes.
@@ -302,9 +328,9 @@ void broadcast_nonunl_proposal()
         nup.user_messages.try_emplace(user.pubkey, std::move(usermsgs));
     }
 
-    p2p::peer_outbound_message msg(std::make_shared<flatbuffers::FlatBufferBuilder>(1024));
-    p2pmsg::create_msg_from_nonunl_proposal(msg.builder(), nup);
-    p2p::broadcast_message(msg, true);
+    flatbuffers::FlatBufferBuilder fbuf(1024);
+    p2pmsg::create_msg_from_nonunl_proposal(fbuf, nup);
+    p2p::broadcast_message(fbuf, true);
 
     LOG_DBG << "NUP sent."
             << " users:" << nup.user_messages.size();
@@ -316,14 +342,17 @@ void broadcast_nonunl_proposal()
  */
 void verify_and_populate_candidate_user_inputs()
 {
+    // Lock the user sessions.
+    std::lock_guard<std::mutex> users_lock(usr::ctx.users_mutex);
+
     // Lock the list so any network activity is blocked.
-    std::lock_guard<std::mutex> lock(p2p::ctx.collected_msgs.nonunl_proposals_mutex);
+    std::lock_guard<std::mutex> nups_lock(p2p::ctx.collected_msgs.nonunl_proposals_mutex);
     for (const p2p::nonunl_proposal &p : p2p::ctx.collected_msgs.nonunl_proposals)
     {
         for (const auto &[pubkey, umsgs] : p.user_messages)
         {
             // Locate this user's socket session in case we need to send any status messages regarding user inputs.
-            sock::socket_session<usr::user_outbound_message> *session = usr::get_session_by_pubkey(pubkey);
+            const comm::comm_session *session = usr::get_session_by_pubkey(pubkey);
 
             // Populate user list with this user's pubkey.
             ctx.candidate_users.emplace(pubkey);
@@ -398,11 +427,15 @@ void verify_and_populate_candidate_user_inputs()
                     reject_reason = jusrmsg::REASON_DUPLICATE_MSG;
                 }
 
-                usr::send_request_status_result(session,
-                                                reject_reason == NULL ? jusrmsg::STATUS_ACCEPTED : jusrmsg::STATUS_REJECTED,
-                                                reject_reason == NULL ? "" : reject_reason,
-                                                jusrmsg::MSGTYPE_CONTRACT_INPUT,
-                                                jusrmsg::origin_data_for_contract_input(umsg.sig));
+                // Send the request status result if this user is connected to us.
+                if (session != NULL)
+                {
+                    usr::send_request_status_result(*session,
+                                                    reject_reason == NULL ? jusrmsg::STATUS_ACCEPTED : jusrmsg::STATUS_REJECTED,
+                                                    reject_reason == NULL ? "" : reject_reason,
+                                                    jusrmsg::MSGTYPE_CONTRACT_INPUT,
+                                                    jusrmsg::origin_data_for_contract_input(umsg.sig));
+                }
             }
         }
     }
@@ -569,11 +602,6 @@ p2p::proposal create_stage123_proposal(vote_counter &votes)
         }
     }
 
-    if (ctx.stage == 3)
-        stg_prop.time = get_ledger_time_resolution(stg_prop.time);
-    else
-        stg_prop.time = get_stage_time_resolution(stg_prop.time);
-
     return stg_prop;
 }
 
@@ -583,19 +611,20 @@ p2p::proposal create_stage123_proposal(vote_counter &votes)
  */
 void broadcast_proposal(const p2p::proposal &p)
 {
-    p2p::peer_outbound_message msg(std::make_shared<flatbuffers::FlatBufferBuilder>(1024));
-    p2pmsg::create_msg_from_proposal(msg.builder(), p);
+    flatbuffers::FlatBufferBuilder fbuf(1024);
+    p2pmsg::create_msg_from_proposal(fbuf, p);
 
     // In observer mode, we only send out the proposal to ourselves.
     if (conf::cfg.current_mode == conf::OPERATING_MODE::OBSERVER)
-        p2p::send_message_to_self(msg);
+        p2p::send_message_to_self(fbuf);
     else
-        p2p::broadcast_message(msg, true);
+        p2p::broadcast_message(fbuf, true);
 
     // LOG_DBG << "Proposed [stage" << std::to_string(p.stage)
     //         << "] users:" << p.users.size()
     //         << " hinp:" << p.hash_inputs.size()
-    //         << " hout:" << p.hash_outputs.size();
+    //         << " hout:" << p.hash_outputs.size()
+    //         << " ts:" << std::to_string(p.time);
 }
 
 /**
@@ -607,12 +636,8 @@ void check_lcl_votes(bool &is_desync, bool &should_request_history, std::string
 
     for (const auto &[pubkey, cp] : ctx.candidate_proposals)
     {
-        // only consider recent proposals and proposals from previous stage and current stage.
-        if ((ctx.time_now - cp.timestamp < conf::cfg.roundtime * 4) && cp.stage >= (ctx.stage - 1))
-        {
-            increment(votes.lcl, cp.lcl);
-            total_lcl_votes++;
-        }
+        increment(votes.lcl, cp.lcl);
+        total_lcl_votes++;
     }
 
     is_desync = false;
@@ -620,7 +645,7 @@ void check_lcl_votes(bool &is_desync, bool &should_request_history, std::string
 
     if (total_lcl_votes < (MAJORITY_THRESHOLD * conf::cfg.unl.size()))
     {
-        LOG_DBG << "Not enough peers proposer to perform consensus votes:" << std::to_string(total_lcl_votes) << " needed:" << std::to_string(MAJORITY_THRESHOLD * conf::cfg.unl.size());
+        LOG_DBG << "Not enough peers proposing to perform consensus. votes:" << std::to_string(total_lcl_votes) << " needed:" << std::to_string(MAJORITY_THRESHOLD * conf::cfg.unl.size());
         is_desync = true;
 
         //Not enough nodes are propsing. So Node is switching to Proposer if it's in observer mode.
@@ -681,39 +706,6 @@ float_t get_stage_threshold(const uint8_t stage)
     return -1;
 }
 
-/**
-* Calculate the effective ledger close time
-* After adjusting the ledger close time based on the current resolution,
-* also ensure it is sufficiently separated from the prior close time.
-* @param close_time voted/agreed closed time
-*/
-uint64_t get_ledger_time_resolution(const uint64_t time)
-{
-    uint64_t closeResolution = conf::cfg.roundtime / 4;
-    //todo: change time resolution dynamically.
-    //When nodes agree often reduce resolution time and increase if they don't.
-    uint64_t close_time = time;
-    close_time += (closeResolution / 2);
-    close_time -= (close_time % closeResolution);
-
-    return std::max(close_time, (ctx.prev_close_time + conf::cfg.roundtime));
-}
-
-/**
-* Calculate the stage time
-* Adjusting the stage time based on the current resolution.
-* @param stage_time voted/agreed closed time
-*/
-uint64_t get_stage_time_resolution(const uint64_t time)
-{
-    uint64_t closeResolution = conf::cfg.roundtime / 8;
-    uint64_t stage_time = time;
-    stage_time += (closeResolution / 2);
-    stage_time -= (stage_time % closeResolution);
-
-    return stage_time;
-}
-
 /**
  * Finalize the ledger after consensus.
  * @param cons_prop The proposal that reached consensus.
@@ -794,7 +786,7 @@ void dispatch_user_outputs(const p2p::proposal &cons_prop)
                     jusrmsg::create_contract_output_container(msg, outputtosend);
 
                     const usr::connected_user &user = user_itr->second;
-                    user.session->send(usr::user_outbound_message(std::move(msg)));
+                    user.session.send(msg);
                 }
             }
 
@@ -845,7 +837,7 @@ void check_state(vote_counter &votes)
             conf::change_operating_mode(conf::OPERATING_MODE::OBSERVER);
 
             const hasher::B2H majority_state_hash = *reinterpret_cast<const hasher::B2H *>(majority_state.c_str());
-            LOG_INFO << "Starting state sync. Curr state:" << *reinterpret_cast<const hasher::B2H *>(ctx.curr_hash_state.c_str()) << " majority:" << majority_state_hash;
+            LOG_INFO << "Syncing state. Curr state:" << *reinterpret_cast<const hasher::B2H *>(ctx.curr_hash_state.c_str()) << " majority:" << majority_state_hash;
 
             start_state_sync(majority_state_hash);
 
@@ -932,9 +924,9 @@ void broadcast_npl_output(std::string &output)
         p2p::npl_message npl;
         npl.data.swap(output);
 
-        p2p::peer_outbound_message msg(std::make_shared<flatbuffers::FlatBufferBuilder>(1024));
-        p2pmsg::create_msg_from_npl_output(msg.builder(), npl, ctx.lcl);
-        p2p::broadcast_message(msg, false);
+        flatbuffers::FlatBufferBuilder fbuf(1024);
+        p2pmsg::create_msg_from_npl_output(fbuf, npl, ctx.lcl);
+        p2p::broadcast_message(fbuf, false);
     }
 }
 

src/cons/cons.hpp

@@ -85,16 +85,18 @@ struct consensus_context
     bool is_lcl_syncing = false;
 
     //ledger close time of previous hash
-    uint64_t prev_close_time = 0;
-    uint16_t stage_time = 0;                    // Time allocated to a consensus stage.
-    uint16_t stage_reset_wait_threshold = 0;    // Minimum stage wait time to reset the stage.
+    uint16_t stage_time = 0;                 // Time allocated to a consensus stage.
+    uint16_t stage_reset_wait_threshold = 0; // Minimum stage wait time to reset the stage.
 
     bool is_state_syncing = false;
     std::string state_sync_lcl;
     std::thread state_syncing_thread;
     std::mutex state_syncing_mutex;
 
-    consensus_context() : recent_userinput_hashes(200)
+    bool is_shutting_down = false;
+
+    consensus_context()
+        : recent_userinput_hashes(200)
     {
     }
 };
@@ -113,9 +115,13 @@ extern consensus_context ctx;
 
 int init();
 
+void deinit();
+
 void consensus();
 
-bool wait_and_proceed_stage();
+void purify_candidate_proposals();
+
+bool wait_and_proceed_stage(uint64_t &stage_start);
 
 void broadcast_nonunl_proposal();
 

src/cons/ledger_handler.cpp

@@ -1,4 +1,3 @@
-#include <flatbuffers/flatbuffers.h>
 #include "../pchheader.hpp"
 #include "../conf.hpp"
 #include "../crypto.hpp"
@@ -6,6 +5,7 @@
 #include "../fbschema/common_helpers.hpp"
 #include "../fbschema/ledger_helpers.hpp"
 #include "../fbschema/p2pmsg_helpers.hpp"
+#include "../hplog.hpp"
 #include "ledger_handler.hpp"
 #include "cons.hpp"
 
@@ -232,9 +232,10 @@ void send_ledger_history_request(const std::string &minimum_lcl, const std::stri
     p2p::history_request hr;
     hr.required_lcl = required_lcl;
     hr.minimum_lcl = minimum_lcl;
-    p2p::peer_outbound_message msg(std::make_unique<flatbuffers::FlatBufferBuilder>(1024));
-    p2pmsg::create_msg_from_history_request(msg.builder(), hr);
-    p2p::send_message_to_random_peer(msg);
+    
+    flatbuffers::FlatBufferBuilder fbuf(1024);
+    p2pmsg::create_msg_from_history_request(fbuf, hr);
+    p2p::send_message_to_random_peer(fbuf);
 
     ctx.last_requested_lcl = required_lcl;
 
@@ -365,19 +366,6 @@ const p2p::history_response retrieve_ledger_history(const p2p::history_request &
     return history_response;
 }
 
-/**
- * Send ledger history response for history request.
- * @param hr lcl history request information.
- * @return peer outbound message object with ledger history response.
- */
-p2p::peer_outbound_message send_ledger_history(const p2p::history_request &hr)
-{
-    p2p::peer_outbound_message msg(std::make_unique<flatbuffers::FlatBufferBuilder>(1024));
-    p2pmsg::create_msg_from_history_response(msg.builder(), retrieve_ledger_history(hr));
-
-    return msg;
-}
-
 /**
  * Handle recieved ledger history response.
  * @param hr lcl history request information.
@@ -479,7 +467,7 @@ void handle_ledger_history_response(const p2p::history_response &hr)
         cons::ctx.led_seq_no = latest_lcl_itr->first;
     }
 
-    LOG_DBG << "Finished lcl sync. New lcl: " << cons::ctx.lcl.substr(0, 15);
+    LOG_INFO << "lcl sync complete. New lcl:" << cons::ctx.lcl.substr(0, 15);
 }
 
 } // namespace cons

src/cons/ledger_handler.hpp

@@ -39,8 +39,6 @@ bool check_required_lcl_availability(const p2p::history_request &hr);
 
 const p2p::history_response retrieve_ledger_history(const p2p::history_request &hr);
 
-p2p::peer_outbound_message send_ledger_history(const p2p::history_request &hr);
-
 void handle_ledger_history_response(const p2p::history_response &hr);
 
 } // namespace cons

src/cons/state_handler.cpp

@@ -1,4 +1,3 @@
-#include <flatbuffers/flatbuffers.h>
 #include "state_handler.hpp"
 #include "../fbschema/p2pmsg_helpers.hpp"
 #include "../fbschema/p2pmsg_content_generated.h"
@@ -7,6 +6,8 @@
 #include "../pchheader.hpp"
 #include "../cons/cons.hpp"
 #include "../statefs/state_store.hpp"
+#include "../hplog.hpp"
+#include "../util.hpp"
 
 namespace cons
 {
@@ -40,9 +41,9 @@ void request_state_from_peer(const std::string &path, const bool is_file, const
     sr.block_id = block_id;
     sr.expected_hash = expected_hash;
 
-    p2p::peer_outbound_message msg(std::make_unique<flatbuffers::FlatBufferBuilder>(1024));
-    fbschema::p2pmsg::create_msg_from_state_request(msg.builder(), sr, ctx.lcl);
-    p2p::send_message_to_random_peer(msg); //todo: send to a node that hold the majority state to improve reliability of retrieving state.
+    flatbuffers::FlatBufferBuilder fbuf(1024);
+    fbschema::p2pmsg::create_msg_from_state_request(fbuf, sr, ctx.lcl);
+    p2p::send_message_to_random_peer(fbuf); //todo: send to a node that hold the majority state to improve reliability of retrieving state.
 }
 
 /**
@@ -50,7 +51,7 @@ void request_state_from_peer(const std::string &path, const bool is_file, const
  * @param msg The peer outbound message reference to build up the reply message.
  * @param sr The state request which should be replied to.
  */
-int create_state_response(p2p::peer_outbound_message &msg, const p2p::state_request &sr)
+int create_state_response(flatbuffers::FlatBufferBuilder &fbuf, const p2p::state_request &sr)
 {
     // If block_id > -1 this means this is a file block data request.
     if (sr.block_id > -1)
@@ -67,7 +68,7 @@ int create_state_response(p2p::peer_outbound_message &msg, const p2p::state_requ
         resp.hash = sr.expected_hash;
         resp.data = std::string_view(reinterpret_cast<const char *>(block.data()), block.size());
 
-        fbschema::p2pmsg::create_msg_from_block_response(msg.builder(), resp, ctx.lcl);
+        fbschema::p2pmsg::create_msg_from_block_response(fbuf, resp, ctx.lcl);
     }
     else
     {
@@ -78,7 +79,7 @@ int create_state_response(p2p::peer_outbound_message &msg, const p2p::state_requ
             if (statefs::get_block_hash_map(existing_block_hashmap, sr.parent_path, sr.expected_hash) == -1)
                 return -1;
 
-            fbschema::p2pmsg::create_msg_from_filehashmap_response(msg.builder(), sr.parent_path, existing_block_hashmap, statefs::get_file_length(sr.parent_path), sr.expected_hash, ctx.lcl);
+            fbschema::p2pmsg::create_msg_from_filehashmap_response(fbuf, sr.parent_path, existing_block_hashmap, statefs::get_file_length(sr.parent_path), sr.expected_hash, ctx.lcl);
         }
         else
         {
@@ -87,7 +88,7 @@ int create_state_response(p2p::peer_outbound_message &msg, const p2p::state_requ
             if (statefs::get_fs_entry_hashes(existing_fs_entries, sr.parent_path, sr.expected_hash) == -1)
                 return -1;
 
-            fbschema::p2pmsg::create_msg_from_fsentry_response(msg.builder(), sr.parent_path, existing_fs_entries, sr.expected_hash, ctx.lcl);
+            fbschema::p2pmsg::create_msg_from_fsentry_response(fbuf, sr.parent_path, existing_fs_entries, sr.expected_hash, ctx.lcl);
         }
     }
 
@@ -122,8 +123,13 @@ void start_state_sync(const hasher::B2H state_hash_to_request)
  */
 int run_state_sync_iterator()
 {
+    util::mask_signal();
+
     while (true)
     {
+        if (ctx.is_shutting_down)
+            break;
+
         util::sleep(SYNC_LOOP_WAIT);
 
         // TODO: Also bypass peer session handler state responses if we're not syncing.
@@ -142,6 +148,9 @@ int run_state_sync_iterator()
 
         for (auto &response : candidate_state_responses)
         {
+            if (ctx.is_shutting_down)
+                break;
+
             const fbschema::p2pmsg::Content *content = fbschema::p2pmsg::GetContent(response.data());
             const fbschema::p2pmsg::State_Response_Message *resp_msg = content->message_as_State_Response_Message();
 
@@ -179,6 +188,9 @@ int run_state_sync_iterator()
         // Check for long-awaited responses and re-request them.
         for (auto &[hash, request] : submitted_requests)
         {
+            if (ctx.is_shutting_down)
+                break;
+
             // We wait for half of round time before each request is resubmitted.
             if (request.waiting_cycles < (conf::cfg.roundtime / (SYNC_LOOP_WAIT * 2)))
             {
@@ -200,6 +212,9 @@ int run_state_sync_iterator()
             const uint16_t available_slots = MAX_AWAITING_REQUESTS - submitted_requests.size();
             for (int i = 0; i < available_slots && !pending_requests.empty(); i++)
             {
+                if (ctx.is_shutting_down)
+                    break;
+                    
                 const backlog_item &request = pending_requests.front();
                 submit_request(request);
                 pending_requests.pop_front();

src/cons/state_handler.hpp

@@ -31,7 +31,7 @@ struct backlog_item
 
 extern std::list<std::string> candidate_state_responses;
 
-int create_state_response(p2p::peer_outbound_message &msg, const p2p::state_request &sr);
+int create_state_response(flatbuffers::FlatBufferBuilder &fbuf, const p2p::state_request &sr);
 
 void request_state_from_peer(const std::string &path, const bool is_file, const int32_t block_id, const hasher::B2H expected_hash);
 

src/fbschema/common_helpers.hpp

@@ -2,7 +2,6 @@
 #define _HP_FBSCHEMA_COMMON_HELPERS_
 
 #include "../pchheader.hpp"
-#include <flatbuffers/flatbuffers.h>
 #include "common_schema_generated.h"
 #include "../statefs/hasher.hpp"
 

src/fbschema/ledger_helpers.cpp

@@ -1,4 +1,3 @@
-#include <flatbuffers/flatbuffers.h>
 #include "../pchheader.hpp"
 #include "ledger_schema_generated.h"
 #include "../p2p/p2p.hpp"

src/fbschema/ledger_helpers.hpp

@@ -1,7 +1,6 @@
 #ifndef _HP_FBSCHEMA_LEDGER_HELPERS_
 #define _HP_FBSCHEMA_LEDGER_HELPERS_
 
-#include <flatbuffers/flatbuffers.h>
 #include "../pchheader.hpp"
 #include "ledger_schema_generated.h"
 #include "../p2p/p2p.hpp"

src/fbschema/p2pmsg_content.fbs

@@ -2,6 +2,10 @@
 include "common_schema.fbs";
 namespace fbschema.p2pmsg;
 
+table PeerId_Notify_Message {
+    peerid:[ubyte];
+}
+
 table UserSubmittedMessage {
     content:[ubyte];
     signature:[ubyte];
@@ -12,7 +16,7 @@ table UserSubmittedMessageGroup {
     messages:[UserSubmittedMessage];
 }
 
-union Message { NonUnl_Proposal_Message, Proposal_Message, Npl_Message, State_Request_Message, State_Response_Message, History_Request_Message, History_Response_Message } //message content type
+union Message { PeerId_Notify_Message, NonUnl_Proposal_Message, Proposal_Message, Npl_Message, State_Request_Message, State_Response_Message, History_Request_Message, History_Response_Message } //message content type
 
 table Content {
     message:Message;

src/fbschema/p2pmsg_content_generated.h

@@ -11,54 +11,75 @@
 namespace fbschema {
 namespace p2pmsg {
 
+struct PeerId_Notify_Message;
+struct PeerId_Notify_MessageBuilder;
+
 struct UserSubmittedMessage;
+struct UserSubmittedMessageBuilder;
 
 struct UserSubmittedMessageGroup;
+struct UserSubmittedMessageGroupBuilder;
 
 struct Content;
+struct ContentBuilder;
 
 struct NonUnl_Proposal_Message;
+struct NonUnl_Proposal_MessageBuilder;
 
 struct Proposal_Message;
+struct Proposal_MessageBuilder;
 
 struct Npl_Message;
+struct Npl_MessageBuilder;
 
 struct History_Request_Message;
+struct History_Request_MessageBuilder;
 
 struct History_Response_Message;
+struct History_Response_MessageBuilder;
 
 struct HistoryLedgerPair;
+struct HistoryLedgerPairBuilder;
 
 struct HistoryLedger;
+struct HistoryLedgerBuilder;
 
 struct State_Request_Message;
+struct State_Request_MessageBuilder;
 
 struct State_Response_Message;
+struct State_Response_MessageBuilder;
 
 struct Fs_Entry_Response;
+struct Fs_Entry_ResponseBuilder;
 
 struct File_HashMap_Response;
+struct File_HashMap_ResponseBuilder;
 
 struct Block_Response;
+struct Block_ResponseBuilder;
 
 struct State_FS_Hash_Entry;
+struct State_FS_Hash_EntryBuilder;
 
 enum Message {
   Message_NONE = 0,
-  Message_NonUnl_Proposal_Message = 1,
-  Message_Proposal_Message = 2,
-  Message_Npl_Message = 3,
-  Message_State_Request_Message = 4,
-  Message_State_Response_Message = 5,
-  Message_History_Request_Message = 6,
-  Message_History_Response_Message = 7,
+  Message_PeerId_Notify_Message = 1,
+  Message_NonUnl_Proposal_Message = 2,
+  Message_Proposal_Message = 3,
+  Message_Npl_Message = 4,
+  Message_State_Request_Message = 5,
+  Message_State_Response_Message = 6,
+  Message_History_Request_Message = 7,
+  Message_History_Response_Message = 8,
   Message_MIN = Message_NONE,
   Message_MAX = Message_History_Response_Message
 };
 
-inline const Message (&EnumValuesMessage())[8] {
+inline const Message (&EnumValuesMessage())[9] {
   static const Message values[] = {
     Message_NONE,
+    Message_PeerId_Notify_Message,
     Message_NonUnl_Proposal_Message,
     Message_Proposal_Message,
     Message_Npl_Message,
@@ -71,8 +92,9 @@ inline const Message (&EnumValuesMessage())[8] {
 }
 
 inline const char * const *EnumNamesMessage() {
-  static const char * const names[] = {
+  static const char * const names[10] = {
     "NONE",
+    "PeerId_Notify_Message",
     "NonUnl_Proposal_Message",
     "Proposal_Message",
     "Npl_Message",
@@ -86,7 +108,7 @@ inline const char * const *EnumNamesMessage() {
 }
 
 inline const char *EnumNameMessage(Message e) {
-  if (e < Message_NONE || e > Message_History_Response_Message) return "";
+  if (flatbuffers::IsOutRange(e, Message_NONE, Message_History_Response_Message)) return "";
   const size_t index = static_cast<size_t>(e);
   return EnumNamesMessage()[index];
 }
@@ -95,31 +117,35 @@ template<typename T> struct MessageTraits {
   static const Message enum_value = Message_NONE;
 };
 
-template<> struct MessageTraits<NonUnl_Proposal_Message> {
+template<> struct MessageTraits<fbschema::p2pmsg::PeerId_Notify_Message> {
+  static const Message enum_value = Message_PeerId_Notify_Message;
+};
+
+template<> struct MessageTraits<fbschema::p2pmsg::NonUnl_Proposal_Message> {
   static const Message enum_value = Message_NonUnl_Proposal_Message;
 };
 
-template<> struct MessageTraits<Proposal_Message> {
+template<> struct MessageTraits<fbschema::p2pmsg::Proposal_Message> {
   static const Message enum_value = Message_Proposal_Message;
 };
 
-template<> struct MessageTraits<Npl_Message> {
+template<> struct MessageTraits<fbschema::p2pmsg::Npl_Message> {
   static const Message enum_value = Message_Npl_Message;
 };
 
-template<> struct MessageTraits<State_Request_Message> {
+template<> struct MessageTraits<fbschema::p2pmsg::State_Request_Message> {
   static const Message enum_value = Message_State_Request_Message;
 };
 
-template<> struct MessageTraits<State_Response_Message> {
+template<> struct MessageTraits<fbschema::p2pmsg::State_Response_Message> {
   static const Message enum_value = Message_State_Response_Message;
 };
 
-template<> struct MessageTraits<History_Request_Message> {
+template<> struct MessageTraits<fbschema::p2pmsg::History_Request_Message> {
   static const Message enum_value = Message_History_Request_Message;
 };
 
-template<> struct MessageTraits<History_Response_Message> {
+template<> struct MessageTraits<fbschema::p2pmsg::History_Response_Message> {
   static const Message enum_value = Message_History_Response_Message;
 };
 
@@ -144,7 +170,7 @@ inline const Ledger_Response_Error (&EnumValuesLedger_Response_Error())[3] {
 }
 
 inline const char * const *EnumNamesLedger_Response_Error() {
-  static const char * const names[] = {
+  static const char * const names[4] = {
     "None",
     "Invalid_Min_Ledger",
     "Req_Ledger_Not_Found",
@@ -154,7 +180,7 @@ inline const char * const *EnumNamesLedger_Response_Error() {
 }
 
 inline const char *EnumNameLedger_Response_Error(Ledger_Response_Error e) {
-  if (e < Ledger_Response_Error_None || e > Ledger_Response_Error_Req_Ledger_Not_Found) return "";
+  if (flatbuffers::IsOutRange(e, Ledger_Response_Error_None, Ledger_Response_Error_Req_Ledger_Not_Found)) return "";
   const size_t index = static_cast<size_t>(e);
   return EnumNamesLedger_Response_Error()[index];
 }
@@ -179,7 +205,7 @@ inline const State_Response (&EnumValuesState_Response())[4] {
 }
 
 inline const char * const *EnumNamesState_Response() {
-  static const char * const names[] = {
+  static const char * const names[5] = {
     "NONE",
     "File_HashMap_Response",
     "Block_Response",
@@ -190,7 +216,7 @@ inline const char * const *EnumNamesState_Response() {
 }
 
 inline const char *EnumNameState_Response(State_Response e) {
-  if (e < State_Response_NONE || e > State_Response_Fs_Entry_Response) return "";
+  if (flatbuffers::IsOutRange(e, State_Response_NONE, State_Response_Fs_Entry_Response)) return "";
   const size_t index = static_cast<size_t>(e);
   return EnumNamesState_Response()[index];
 }
@@ -199,22 +225,78 @@ template<typename T> struct State_ResponseTraits {
   static const State_Response enum_value = State_Response_NONE;
 };
 
-template<> struct State_ResponseTraits<File_HashMap_Response> {
+template<> struct State_ResponseTraits<fbschema::p2pmsg::File_HashMap_Response> {
   static const State_Response enum_value = State_Response_File_HashMap_Response;
 };
 
-template<> struct State_ResponseTraits<Block_Response> {
+template<> struct State_ResponseTraits<fbschema::p2pmsg::Block_Response> {
   static const State_Response enum_value = State_Response_Block_Response;
 };
 
-template<> struct State_ResponseTraits<Fs_Entry_Response> {
+template<> struct State_ResponseTraits<fbschema::p2pmsg::Fs_Entry_Response> {
   static const State_Response enum_value = State_Response_Fs_Entry_Response;
 };
 
 bool VerifyState_Response(flatbuffers::Verifier &verifier, const void *obj, State_Response type);
 bool VerifyState_ResponseVector(flatbuffers::Verifier &verifier, const flatbuffers::Vector<flatbuffers::Offset<void>> *values, const flatbuffers::Vector<uint8_t> *types);
 
+struct PeerId_Notify_Message FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
+  typedef PeerId_Notify_MessageBuilder Builder;
+  enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
+    VT_PEERID = 4
+  };
+  const flatbuffers::Vector<uint8_t> *peerid() const {
+    return GetPointer<const flatbuffers::Vector<uint8_t> *>(VT_PEERID);
+  }
+  flatbuffers::Vector<uint8_t> *mutable_peerid() {
+    return GetPointer<flatbuffers::Vector<uint8_t> *>(VT_PEERID);
+  }
+  bool Verify(flatbuffers::Verifier &verifier) const {
+    return VerifyTableStart(verifier) &&
+           VerifyOffset(verifier, VT_PEERID) &&
+           verifier.VerifyVector(peerid()) &&
+           verifier.EndTable();
+  }
+};
+
+struct PeerId_Notify_MessageBuilder {
+  typedef PeerId_Notify_Message Table;
+  flatbuffers::FlatBufferBuilder &fbb_;
+  flatbuffers::uoffset_t start_;
+  void add_peerid(flatbuffers::Offset<flatbuffers::Vector<uint8_t>> peerid) {
+    fbb_.AddOffset(PeerId_Notify_Message::VT_PEERID, peerid);
+  }
+  explicit PeerId_Notify_MessageBuilder(flatbuffers::FlatBufferBuilder &_fbb)
+        : fbb_(_fbb) {
+    start_ = fbb_.StartTable();
+  }
+  PeerId_Notify_MessageBuilder &operator=(const PeerId_Notify_MessageBuilder &);
+  flatbuffers::Offset<PeerId_Notify_Message> Finish() {
+    const auto end = fbb_.EndTable(start_);
+    auto o = flatbuffers::Offset<PeerId_Notify_Message>(end);
+    return o;
+  }
+};
+
+inline flatbuffers::Offset<PeerId_Notify_Message> CreatePeerId_Notify_Message(
+    flatbuffers::FlatBufferBuilder &_fbb,
+    flatbuffers::Offset<flatbuffers::Vector<uint8_t>> peerid = 0) {
+  PeerId_Notify_MessageBuilder builder_(_fbb);
+  builder_.add_peerid(peerid);
+  return builder_.Finish();
+}
+
+inline flatbuffers::Offset<PeerId_Notify_Message> CreatePeerId_Notify_MessageDirect(
+    flatbuffers::FlatBufferBuilder &_fbb,
+    const std::vector<uint8_t> *peerid = nullptr) {
+  auto peerid__ = peerid ? _fbb.CreateVector<uint8_t>(*peerid) : 0;
+  return fbschema::p2pmsg::CreatePeerId_Notify_Message(
+      _fbb,
+      peerid__);
+}
+
 struct UserSubmittedMessage FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
+  typedef UserSubmittedMessageBuilder Builder;
   enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
     VT_CONTENT = 4,
     VT_SIGNATURE = 6
@@ -242,6 +324,7 @@ struct UserSubmittedMessage FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table
 };
 
 struct UserSubmittedMessageBuilder {
+  typedef UserSubmittedMessage Table;
   flatbuffers::FlatBufferBuilder &fbb_;
   flatbuffers::uoffset_t start_;
   void add_content(flatbuffers::Offset<flatbuffers::Vector<uint8_t>> content) {
@@ -285,6 +368,7 @@ inline flatbuffers::Offset<UserSubmittedMessage> CreateUserSubmittedMessageDirec
 }
 
 struct UserSubmittedMessageGroup FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
+  typedef UserSubmittedMessageGroupBuilder Builder;
   enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
     VT_PUBKEY = 4,
     VT_MESSAGES = 6
@@ -295,11 +379,11 @@ struct UserSubmittedMessageGroup FLATBUFFERS_FINAL_CLASS : private flatbuffers::
   flatbuffers::Vector<uint8_t> *mutable_pubkey() {
     return GetPointer<flatbuffers::Vector<uint8_t> *>(VT_PUBKEY);
   }
-  const flatbuffers::Vector<flatbuffers::Offset<UserSubmittedMessage>> *messages() const {
-    return GetPointer<const flatbuffers::Vector<flatbuffers::Offset<UserSubmittedMessage>> *>(VT_MESSAGES);
+  const flatbuffers::Vector<flatbuffers::Offset<fbschema::p2pmsg::UserSubmittedMessage>> *messages() const {
+    return GetPointer<const flatbuffers::Vector<flatbuffers::Offset<fbschema::p2pmsg::UserSubmittedMessage>> *>(VT_MESSAGES);
   }
-  flatbuffers::Vector<flatbuffers::Offset<UserSubmittedMessage>> *mutable_messages() {
-    return GetPointer<flatbuffers::Vector<flatbuffers::Offset<UserSubmittedMessage>> *>(VT_MESSAGES);
+  flatbuffers::Vector<flatbuffers::Offset<fbschema::p2pmsg::UserSubmittedMessage>> *mutable_messages() {
+    return GetPointer<flatbuffers::Vector<flatbuffers::Offset<fbschema::p2pmsg::UserSubmittedMessage>> *>(VT_MESSAGES);
   }
   bool Verify(flatbuffers::Verifier &verifier) const {
     return VerifyTableStart(verifier) &&
@@ -313,12 +397,13 @@ struct UserSubmittedMessageGroup FLATBUFFERS_FINAL_CLASS : private flatbuffers::
 };
 
 struct UserSubmittedMessageGroupBuilder {
+  typedef UserSubmittedMessageGroup Table;
   flatbuffers::FlatBufferBuilder &fbb_;
   flatbuffers::uoffset_t start_;
   void add_pubkey(flatbuffers::Offset<flatbuffers::Vector<uint8_t>> pubkey) {
     fbb_.AddOffset(UserSubmittedMessageGroup::VT_PUBKEY, pubkey);
   }
-  void add_messages(flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<UserSubmittedMessage>>> messages) {
+  void add_messages(flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<fbschema::p2pmsg::UserSubmittedMessage>>> messages) {
     fbb_.AddOffset(UserSubmittedMessageGroup::VT_MESSAGES, messages);
   }
   explicit UserSubmittedMessageGroupBuilder(flatbuffers::FlatBufferBuilder &_fbb)
@@ -336,7 +421,7 @@ struct UserSubmittedMessageGroupBuilder {
 inline flatbuffers::Offset<UserSubmittedMessageGroup> CreateUserSubmittedMessageGroup(
     flatbuffers::FlatBufferBuilder &_fbb,
     flatbuffers::Offset<flatbuffers::Vector<uint8_t>> pubkey = 0,
-    flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<UserSubmittedMessage>>> messages = 0) {
+    flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<fbschema::p2pmsg::UserSubmittedMessage>>> messages = 0) {
   UserSubmittedMessageGroupBuilder builder_(_fbb);
   builder_.add_messages(messages);
   builder_.add_pubkey(pubkey);
@@ -346,9 +431,9 @@ inline flatbuffers::Offset<UserSubmittedMessageGroup> CreateUserSubmittedMessage
 inline flatbuffers::Offset<UserSubmittedMessageGroup> CreateUserSubmittedMessageGroupDirect(
     flatbuffers::FlatBufferBuilder &_fbb,
     const std::vector<uint8_t> *pubkey = nullptr,
-    const std::vector<flatbuffers::Offset<UserSubmittedMessage>> *messages = nullptr) {
+    const std::vector<flatbuffers::Offset<fbschema::p2pmsg::UserSubmittedMessage>> *messages = nullptr) {
   auto pubkey__ = pubkey ? _fbb.CreateVector<uint8_t>(*pubkey) : 0;
-  auto messages__ = messages ? _fbb.CreateVector<flatbuffers::Offset<UserSubmittedMessage>>(*messages) : 0;
+  auto messages__ = messages ? _fbb.CreateVector<flatbuffers::Offset<fbschema::p2pmsg::UserSubmittedMessage>>(*messages) : 0;
   return fbschema::p2pmsg::CreateUserSubmittedMessageGroup(
       _fbb,
       pubkey__,
@@ -356,40 +441,41 @@ inline flatbuffers::Offset<UserSubmittedMessageGroup> CreateUserSubmittedMessage
 }
 
 struct Content FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
+  typedef ContentBuilder Builder;
   enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
     VT_MESSAGE_TYPE = 4,
     VT_MESSAGE = 6
   };
-  Message message_type() const {
-    return static_cast<Message>(GetField<uint8_t>(VT_MESSAGE_TYPE, 0));
-  }
-  bool mutate_message_type(Message _message_type) {
-    return SetField<uint8_t>(VT_MESSAGE_TYPE, static_cast<uint8_t>(_message_type), 0);
+  fbschema::p2pmsg::Message message_type() const {
+    return static_cast<fbschema::p2pmsg::Message>(GetField<uint8_t>(VT_MESSAGE_TYPE, 0));
   }
   const void *message() const {
     return GetPointer<const void *>(VT_MESSAGE);
   }
   template<typename T> const T *message_as() const;
-  const NonUnl_Proposal_Message *message_as_NonUnl_Proposal_Message() const {
-    return message_type() == Message_NonUnl_Proposal_Message ? static_cast<const NonUnl_Proposal_Message *>(message()) : nullptr;
+  const fbschema::p2pmsg::PeerId_Notify_Message *message_as_PeerId_Notify_Message() const {
+    return message_type() == fbschema::p2pmsg::Message_PeerId_Notify_Message ? static_cast<const fbschema::p2pmsg::PeerId_Notify_Message *>(message()) : nullptr;
   }
-  const Proposal_Message *message_as_Proposal_Message() const {
-    return message_type() == Message_Proposal_Message ? static_cast<const Proposal_Message *>(message()) : nullptr;
+  const fbschema::p2pmsg::NonUnl_Proposal_Message *message_as_NonUnl_Proposal_Message() const {
+    return message_type() == fbschema::p2pmsg::Message_NonUnl_Proposal_Message ? static_cast<const fbschema::p2pmsg::NonUnl_Proposal_Message *>(message()) : nullptr;
   }
-  const Npl_Message *message_as_Npl_Message() const {
-    return message_type() == Message_Npl_Message ? static_cast<const Npl_Message *>(message()) : nullptr;
+  const fbschema::p2pmsg::Proposal_Message *message_as_Proposal_Message() const {
+    return message_type() == fbschema::p2pmsg::Message_Proposal_Message ? static_cast<const fbschema::p2pmsg::Proposal_Message *>(message()) : nullptr;
   }
-  const State_Request_Message *message_as_State_Request_Message() const {
-    return message_type() == Message_State_Request_Message ? static_cast<const State_Request_Message *>(message()) : nullptr;
+  const fbschema::p2pmsg::Npl_Message *message_as_Npl_Message() const {
+    return message_type() == fbschema::p2pmsg::Message_Npl_Message ? static_cast<const fbschema::p2pmsg::Npl_Message *>(message()) : nullptr;
   }
-  const State_Response_Message *message_as_State_Response_Message() const {
-    return message_type() == Message_State_Response_Message ? static_cast<const State_Response_Message *>(message()) : nullptr;
+  const fbschema::p2pmsg::State_Request_Message *message_as_State_Request_Message() const {
+    return message_type() == fbschema::p2pmsg::Message_State_Request_Message ? static_cast<const fbschema::p2pmsg::State_Request_Message *>(message()) : nullptr;
   }
-  const History_Request_Message *message_as_History_Request_Message() const {
-    return message_type() == Message_History_Request_Message ? static_cast<const History_Request_Message *>(message()) : nullptr;
+  const fbschema::p2pmsg::State_Response_Message *message_as_State_Response_Message() const {
+    return message_type() == fbschema::p2pmsg::Message_State_Response_Message ? static_cast<const fbschema::p2pmsg::State_Response_Message *>(message()) : nullptr;
   }
-  const History_Response_Message *message_as_History_Response_Message() const {
-    return message_type() == Message_History_Response_Message ? static_cast<const History_Response_Message *>(message()) : nullptr;
+  const fbschema::p2pmsg::History_Request_Message *message_as_History_Request_Message() const {
+    return message_type() == fbschema::p2pmsg::Message_History_Request_Message ? static_cast<const fbschema::p2pmsg::History_Request_Message *>(message()) : nullptr;
+  }
+  const fbschema::p2pmsg::History_Response_Message *message_as_History_Response_Message() const {
+    return message_type() == fbschema::p2pmsg::Message_History_Response_Message ? static_cast<const fbschema::p2pmsg::History_Response_Message *>(message()) : nullptr;
   }
   void *mutable_message() {
     return GetPointer<void *>(VT_MESSAGE);
@@ -403,38 +489,43 @@ struct Content FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
   }
 };
 
-template<> inline const NonUnl_Proposal_Message *Content::message_as<NonUnl_Proposal_Message>() const {
+template<> inline const fbschema::p2pmsg::PeerId_Notify_Message *Content::message_as<fbschema::p2pmsg::PeerId_Notify_Message>() const {
+  return message_as_PeerId_Notify_Message();
+}
+
+template<> inline const fbschema::p2pmsg::NonUnl_Proposal_Message *Content::message_as<fbschema::p2pmsg::NonUnl_Proposal_Message>() const {
   return message_as_NonUnl_Proposal_Message();
 }
 
-template<> inline const Proposal_Message *Content::message_as<Proposal_Message>() const {
+template<> inline const fbschema::p2pmsg::Proposal_Message *Content::message_as<fbschema::p2pmsg::Proposal_Message>() const {
   return message_as_Proposal_Message();
 }
 
-template<> inline const Npl_Message *Content::message_as<Npl_Message>() const {
+template<> inline const fbschema::p2pmsg::Npl_Message *Content::message_as<fbschema::p2pmsg::Npl_Message>() const {
   return message_as_Npl_Message();
 }
 
-template<> inline const State_Request_Message *Content::message_as<State_Request_Message>() const {
+template<> inline const fbschema::p2pmsg::State_Request_Message *Content::message_as<fbschema::p2pmsg::State_Request_Message>() const {
   return message_as_State_Request_Message();
 }
 
-template<> inline const State_Response_Message *Content::message_as<State_Response_Message>() const {
+template<> inline const fbschema::p2pmsg::State_Response_Message *Content::message_as<fbschema::p2pmsg::State_Response_Message>() const {
   return message_as_State_Response_Message();
 }
 
-template<> inline const History_Request_Message *Content::message_as<History_Request_Message>() const {
+template<> inline const fbschema::p2pmsg::History_Request_Message *Content::message_as<fbschema::p2pmsg::History_Request_Message>() const {
   return message_as_History_Request_Message();
 }
 
-template<> inline const History_Response_Message *Content::message_as<History_Response_Message>() const {
+template<> inline const fbschema::p2pmsg::History_Response_Message *Content::message_as<fbschema::p2pmsg::History_Response_Message>() const {
   return message_as_History_Response_Message();
 }
 
 struct ContentBuilder {
+  typedef Content Table;
   flatbuffers::FlatBufferBuilder &fbb_;
   flatbuffers::uoffset_t start_;
-  void add_message_type(Message message_type) {
+  void add_message_type(fbschema::p2pmsg::Message message_type) {
     fbb_.AddElement<uint8_t>(Content::VT_MESSAGE_TYPE, static_cast<uint8_t>(message_type), 0);
   }
   void add_message(flatbuffers::Offset<void> message) {
@@ -454,7 +545,7 @@ struct ContentBuilder {
 
 inline flatbuffers::Offset<Content> CreateContent(
     flatbuffers::FlatBufferBuilder &_fbb,
-    Message message_type = Message_NONE,
+    fbschema::p2pmsg::Message message_type = fbschema::p2pmsg::Message_NONE,
     flatbuffers::Offset<void> message = 0) {
   ContentBuilder builder_(_fbb);
   builder_.add_message(message);
@@ -463,14 +554,15 @@ inline flatbuffers::Offset<Content> CreateContent(
 }
 
 struct NonUnl_Proposal_Message FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
+  typedef NonUnl_Proposal_MessageBuilder Builder;
   enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
     VT_USERMESSAGES = 4
   };
-  const flatbuffers::Vector<flatbuffers::Offset<UserSubmittedMessageGroup>> *usermessages() const {
-    return GetPointer<const flatbuffers::Vector<flatbuffers::Offset<UserSubmittedMessageGroup>> *>(VT_USERMESSAGES);
+  const flatbuffers::Vector<flatbuffers::Offset<fbschema::p2pmsg::UserSubmittedMessageGroup>> *usermessages() const {
+    return GetPointer<const flatbuffers::Vector<flatbuffers::Offset<fbschema::p2pmsg::UserSubmittedMessageGroup>> *>(VT_USERMESSAGES);
   }
-  flatbuffers::Vector<flatbuffers::Offset<UserSubmittedMessageGroup>> *mutable_usermessages() {
-    return GetPointer<flatbuffers::Vector<flatbuffers::Offset<UserSubmittedMessageGroup>> *>(VT_USERMESSAGES);
+  flatbuffers::Vector<flatbuffers::Offset<fbschema::p2pmsg::UserSubmittedMessageGroup>> *mutable_usermessages() {
+    return GetPointer<flatbuffers::Vector<flatbuffers::Offset<fbschema::p2pmsg::UserSubmittedMessageGroup>> *>(VT_USERMESSAGES);
   }
   bool Verify(flatbuffers::Verifier &verifier) const {
     return VerifyTableStart(verifier) &&
@@ -482,9 +574,10 @@ struct NonUnl_Proposal_Message FLATBUFFERS_FINAL_CLASS : private flatbuffers::Ta
 };
 
 struct NonUnl_Proposal_MessageBuilder {
+  typedef NonUnl_Proposal_Message Table;
   flatbuffers::FlatBufferBuilder &fbb_;
   flatbuffers::uoffset_t start_;
-  void add_usermessages(flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<UserSubmittedMessageGroup>>> usermessages) {
+  void add_usermessages(flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<fbschema::p2pmsg::UserSubmittedMessageGroup>>> usermessages) {
     fbb_.AddOffset(NonUnl_Proposal_Message::VT_USERMESSAGES, usermessages);
   }
   explicit NonUnl_Proposal_MessageBuilder(flatbuffers::FlatBufferBuilder &_fbb)
@@ -501,7 +594,7 @@ struct NonUnl_Proposal_MessageBuilder {
 
 inline flatbuffers::Offset<NonUnl_Proposal_Message> CreateNonUnl_Proposal_Message(
     flatbuffers::FlatBufferBuilder &_fbb,
-    flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<UserSubmittedMessageGroup>>> usermessages = 0) {
+    flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<fbschema::p2pmsg::UserSubmittedMessageGroup>>> usermessages = 0) {
   NonUnl_Proposal_MessageBuilder builder_(_fbb);
   builder_.add_usermessages(usermessages);
   return builder_.Finish();
@@ -509,14 +602,15 @@ inline flatbuffers::Offset<NonUnl_Proposal_Message> CreateNonUnl_Proposal_Messag
 
 inline flatbuffers::Offset<NonUnl_Proposal_Message> CreateNonUnl_Proposal_MessageDirect(
     flatbuffers::FlatBufferBuilder &_fbb,
-    const std::vector<flatbuffers::Offset<UserSubmittedMessageGroup>> *usermessages = nullptr) {
-  auto usermessages__ = usermessages ? _fbb.CreateVector<flatbuffers::Offset<UserSubmittedMessageGroup>>(*usermessages) : 0;
+    const std::vector<flatbuffers::Offset<fbschema::p2pmsg::UserSubmittedMessageGroup>> *usermessages = nullptr) {
+  auto usermessages__ = usermessages ? _fbb.CreateVector<flatbuffers::Offset<fbschema::p2pmsg::UserSubmittedMessageGroup>>(*usermessages) : 0;
   return fbschema::p2pmsg::CreateNonUnl_Proposal_Message(
       _fbb,
       usermessages__);
 }
 
 struct Proposal_Message FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
+  typedef Proposal_MessageBuilder Builder;
   enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
     VT_STAGE = 4,
     VT_TIME = 6,
@@ -581,6 +675,7 @@ struct Proposal_Message FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
 };
 
 struct Proposal_MessageBuilder {
+  typedef Proposal_Message Table;
   flatbuffers::FlatBufferBuilder &fbb_;
   flatbuffers::uoffset_t start_;
   void add_stage(uint8_t stage) {
@@ -654,6 +749,7 @@ inline flatbuffers::Offset<Proposal_Message> CreateProposal_MessageDirect(
 }
 
 struct Npl_Message FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
+  typedef Npl_MessageBuilder Builder;
   enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
     VT_DATA = 4
   };
@@ -672,6 +768,7 @@ struct Npl_Message FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
 };
 
 struct Npl_MessageBuilder {
+  typedef Npl_Message Table;
   flatbuffers::FlatBufferBuilder &fbb_;
   flatbuffers::uoffset_t start_;
   void add_data(flatbuffers::Offset<flatbuffers::Vector<uint8_t>> data) {
@@ -707,6 +804,7 @@ inline flatbuffers::Offset<Npl_Message> CreateNpl_MessageDirect(
 }
 
 struct History_Request_Message FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
+  typedef History_Request_MessageBuilder Builder;
   enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
     VT_MINIMUM_LCL = 4,
     VT_REQUIRED_LCL = 6
@@ -734,6 +832,7 @@ struct History_Request_Message FLATBUFFERS_FINAL_CLASS : private flatbuffers::Ta
 };
 
 struct History_Request_MessageBuilder {
+  typedef History_Request_Message Table;
   flatbuffers::FlatBufferBuilder &fbb_;
   flatbuffers::uoffset_t start_;
   void add_minimum_lcl(flatbuffers::Offset<flatbuffers::Vector<uint8_t>> minimum_lcl) {
@@ -777,20 +876,21 @@ inline flatbuffers::Offset<History_Request_Message> CreateHistory_Request_Messag
 }
 
 struct History_Response_Message FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
+  typedef History_Response_MessageBuilder Builder;
   enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
     VT_HIST_LEDGERS = 4,
     VT_ERROR = 6
   };
-  const flatbuffers::Vector<flatbuffers::Offset<HistoryLedgerPair>> *hist_ledgers() const {
-    return GetPointer<const flatbuffers::Vector<flatbuffers::Offset<HistoryLedgerPair>> *>(VT_HIST_LEDGERS);
+  const flatbuffers::Vector<flatbuffers::Offset<fbschema::p2pmsg::HistoryLedgerPair>> *hist_ledgers() const {
+    return GetPointer<const flatbuffers::Vector<flatbuffers::Offset<fbschema::p2pmsg::HistoryLedgerPair>> *>(VT_HIST_LEDGERS);
   }
-  flatbuffers::Vector<flatbuffers::Offset<HistoryLedgerPair>> *mutable_hist_ledgers() {
-    return GetPointer<flatbuffers::Vector<flatbuffers::Offset<HistoryLedgerPair>> *>(VT_HIST_LEDGERS);
+  flatbuffers::Vector<flatbuffers::Offset<fbschema::p2pmsg::HistoryLedgerPair>> *mutable_hist_ledgers() {
+    return GetPointer<flatbuffers::Vector<flatbuffers::Offset<fbschema::p2pmsg::HistoryLedgerPair>> *>(VT_HIST_LEDGERS);
   }
-  Ledger_Response_Error error() const {
-    return static_cast<Ledger_Response_Error>(GetField<uint8_t>(VT_ERROR, 0));
+  fbschema::p2pmsg::Ledger_Response_Error error() const {
+    return static_cast<fbschema::p2pmsg::Ledger_Response_Error>(GetField<uint8_t>(VT_ERROR, 0));
   }
-  bool mutate_error(Ledger_Response_Error _error) {
+  bool mutate_error(fbschema::p2pmsg::Ledger_Response_Error _error) {
     return SetField<uint8_t>(VT_ERROR, static_cast<uint8_t>(_error), 0);
   }
   bool Verify(flatbuffers::Verifier &verifier) const {
@@ -804,12 +904,13 @@ struct History_Response_Message FLATBUFFERS_FINAL_CLASS : private flatbuffers::T
 };
 
 struct History_Response_MessageBuilder {
+  typedef History_Response_Message Table;
   flatbuffers::FlatBufferBuilder &fbb_;
   flatbuffers::uoffset_t start_;
-  void add_hist_ledgers(flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<HistoryLedgerPair>>> hist_ledgers) {
+  void add_hist_ledgers(flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<fbschema::p2pmsg::HistoryLedgerPair>>> hist_ledgers) {
     fbb_.AddOffset(History_Response_Message::VT_HIST_LEDGERS, hist_ledgers);
   }
-  void add_error(Ledger_Response_Error error) {
+  void add_error(fbschema::p2pmsg::Ledger_Response_Error error) {
     fbb_.AddElement<uint8_t>(History_Response_Message::VT_ERROR, static_cast<uint8_t>(error), 0);
   }
   explicit History_Response_MessageBuilder(flatbuffers::FlatBufferBuilder &_fbb)
@@ -826,8 +927,8 @@ struct History_Response_MessageBuilder {
 
 inline flatbuffers::Offset<History_Response_Message> CreateHistory_Response_Message(
     flatbuffers::FlatBufferBuilder &_fbb,
-    flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<HistoryLedgerPair>>> hist_ledgers = 0,
-    Ledger_Response_Error error = Ledger_Response_Error_None) {
+    flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<fbschema::p2pmsg::HistoryLedgerPair>>> hist_ledgers = 0,
+    fbschema::p2pmsg::Ledger_Response_Error error = fbschema::p2pmsg::Ledger_Response_Error_None) {
   History_Response_MessageBuilder builder_(_fbb);
   builder_.add_hist_ledgers(hist_ledgers);
   builder_.add_error(error);
@@ -836,9 +937,9 @@ inline flatbuffers::Offset<History_Response_Message> CreateHistory_Response_Mess
 
 inline flatbuffers::Offset<History_Response_Message> CreateHistory_Response_MessageDirect(
     flatbuffers::FlatBufferBuilder &_fbb,
-    const std::vector<flatbuffers::Offset<HistoryLedgerPair>> *hist_ledgers = nullptr,
-    Ledger_Response_Error error = Ledger_Response_Error_None) {
-  auto hist_ledgers__ = hist_ledgers ? _fbb.CreateVector<flatbuffers::Offset<HistoryLedgerPair>>(*hist_ledgers) : 0;
+    const std::vector<flatbuffers::Offset<fbschema::p2pmsg::HistoryLedgerPair>> *hist_ledgers = nullptr,
+    fbschema::p2pmsg::Ledger_Response_Error error = fbschema::p2pmsg::Ledger_Response_Error_None) {
+  auto hist_ledgers__ = hist_ledgers ? _fbb.CreateVector<flatbuffers::Offset<fbschema::p2pmsg::HistoryLedgerPair>>(*hist_ledgers) : 0;
   return fbschema::p2pmsg::CreateHistory_Response_Message(
       _fbb,
       hist_ledgers__,
@@ -846,6 +947,7 @@ inline flatbuffers::Offset<History_Response_Message> CreateHistory_Response_Mess
 }
 
 struct HistoryLedgerPair FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
+  typedef HistoryLedgerPairBuilder Builder;
   enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
     VT_SEQ_NO = 4,
     VT_LEDGER = 6
@@ -856,11 +958,11 @@ struct HistoryLedgerPair FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
   bool mutate_seq_no(uint64_t _seq_no) {
     return SetField<uint64_t>(VT_SEQ_NO, _seq_no, 0);
   }
-  const HistoryLedger *ledger() const {
-    return GetPointer<const HistoryLedger *>(VT_LEDGER);
+  const fbschema::p2pmsg::HistoryLedger *ledger() const {
+    return GetPointer<const fbschema::p2pmsg::HistoryLedger *>(VT_LEDGER);
   }
-  HistoryLedger *mutable_ledger() {
-    return GetPointer<HistoryLedger *>(VT_LEDGER);
+  fbschema::p2pmsg::HistoryLedger *mutable_ledger() {
+    return GetPointer<fbschema::p2pmsg::HistoryLedger *>(VT_LEDGER);
   }
   bool Verify(flatbuffers::Verifier &verifier) const {
     return VerifyTableStart(verifier) &&
@@ -872,12 +974,13 @@ struct HistoryLedgerPair FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
 };
 
 struct HistoryLedgerPairBuilder {
+  typedef HistoryLedgerPair Table;
   flatbuffers::FlatBufferBuilder &fbb_;
   flatbuffers::uoffset_t start_;
   void add_seq_no(uint64_t seq_no) {
     fbb_.AddElement<uint64_t>(HistoryLedgerPair::VT_SEQ_NO, seq_no, 0);
   }
-  void add_ledger(flatbuffers::Offset<HistoryLedger> ledger) {
+  void add_ledger(flatbuffers::Offset<fbschema::p2pmsg::HistoryLedger> ledger) {
     fbb_.AddOffset(HistoryLedgerPair::VT_LEDGER, ledger);
   }
   explicit HistoryLedgerPairBuilder(flatbuffers::FlatBufferBuilder &_fbb)
@@ -895,7 +998,7 @@ struct HistoryLedgerPairBuilder {
 inline flatbuffers::Offset<HistoryLedgerPair> CreateHistoryLedgerPair(
     flatbuffers::FlatBufferBuilder &_fbb,
     uint64_t seq_no = 0,
-    flatbuffers::Offset<HistoryLedger> ledger = 0) {
+    flatbuffers::Offset<fbschema::p2pmsg::HistoryLedger> ledger = 0) {
   HistoryLedgerPairBuilder builder_(_fbb);
   builder_.add_seq_no(seq_no);
   builder_.add_ledger(ledger);
@@ -903,6 +1006,7 @@ inline flatbuffers::Offset<HistoryLedgerPair> CreateHistoryLedgerPair(
 }
 
 struct HistoryLedger FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
+  typedef HistoryLedgerBuilder Builder;
   enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
     VT_STATE = 4,
     VT_LCL = 6,
@@ -939,6 +1043,7 @@ struct HistoryLedger FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
 };
 
 struct HistoryLedgerBuilder {
+  typedef HistoryLedger Table;
   flatbuffers::FlatBufferBuilder &fbb_;
   flatbuffers::uoffset_t start_;
   void add_state(flatbuffers::Offset<flatbuffers::Vector<uint8_t>> state) {
@@ -990,6 +1095,7 @@ inline flatbuffers::Offset<HistoryLedger> CreateHistoryLedgerDirect(
 }
 
 struct State_Request_Message FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
+  typedef State_Request_MessageBuilder Builder;
   enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
     VT_PARENT_PATH = 4,
     VT_IS_FILE = 6,
@@ -1033,6 +1139,7 @@ struct State_Request_Message FLATBUFFERS_FINAL_CLASS : private flatbuffers::Tabl
 };
 
 struct State_Request_MessageBuilder {
+  typedef State_Request_Message Table;
   flatbuffers::FlatBufferBuilder &fbb_;
   flatbuffers::uoffset_t start_;
   void add_parent_path(flatbuffers::Offset<flatbuffers::String> parent_path) {
@@ -1090,29 +1197,27 @@ inline flatbuffers::Offset<State_Request_Message> CreateState_Request_MessageDir
 }
 
 struct State_Response_Message FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
+  typedef State_Response_MessageBuilder Builder;
   enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
     VT_STATE_RESPONSE_TYPE = 4,
     VT_STATE_RESPONSE = 6,
     VT_HASH = 8
   };
-  State_Response state_response_type() const {
-    return static_cast<State_Response>(GetField<uint8_t>(VT_STATE_RESPONSE_TYPE, 0));
-  }
-  bool mutate_state_response_type(State_Response _state_response_type) {
-    return SetField<uint8_t>(VT_STATE_RESPONSE_TYPE, static_cast<uint8_t>(_state_response_type), 0);
+  fbschema::p2pmsg::State_Response state_response_type() const {
+    return static_cast<fbschema::p2pmsg::State_Response>(GetField<uint8_t>(VT_STATE_RESPONSE_TYPE, 0));
   }
   const void *state_response() const {
     return GetPointer<const void *>(VT_STATE_RESPONSE);
   }
   template<typename T> const T *state_response_as() const;
-  const File_HashMap_Response *state_response_as_File_HashMap_Response() const {
-    return state_response_type() == State_Response_File_HashMap_Response ? static_cast<const File_HashMap_Response *>(state_response()) : nullptr;
+  const fbschema::p2pmsg::File_HashMap_Response *state_response_as_File_HashMap_Response() const {
+    return state_response_type() == fbschema::p2pmsg::State_Response_File_HashMap_Response ? static_cast<const fbschema::p2pmsg::File_HashMap_Response *>(state_response()) : nullptr;
   }
-  const Block_Response *state_response_as_Block_Response() const {
-    return state_response_type() == State_Response_Block_Response ? static_cast<const Block_Response *>(state_response()) : nullptr;
+  const fbschema::p2pmsg::Block_Response *state_response_as_Block_Response() const {
+    return state_response_type() == fbschema::p2pmsg::State_Response_Block_Response ? static_cast<const fbschema::p2pmsg::Block_Response *>(state_response()) : nullptr;
   }
-  const Fs_Entry_Response *state_response_as_Fs_Entry_Response() const {
-    return state_response_type() == State_Response_Fs_Entry_Response ? static_cast<const Fs_Entry_Response *>(state_response()) : nullptr;
+  const fbschema::p2pmsg::Fs_Entry_Response *state_response_as_Fs_Entry_Response() const {
+    return state_response_type() == fbschema::p2pmsg::State_Response_Fs_Entry_Response ? static_cast<const fbschema::p2pmsg::Fs_Entry_Response *>(state_response()) : nullptr;
   }
   void *mutable_state_response() {
     return GetPointer<void *>(VT_STATE_RESPONSE);
@@ -1134,22 +1239,23 @@ struct State_Response_Message FLATBUFFERS_FINAL_CLASS : private flatbuffers::Tab
   }
 };
 
-template<> inline const File_HashMap_Response *State_Response_Message::state_response_as<File_HashMap_Response>() const {
+template<> inline const fbschema::p2pmsg::File_HashMap_Response *State_Response_Message::state_response_as<fbschema::p2pmsg::File_HashMap_Response>() const {
   return state_response_as_File_HashMap_Response();
 }
 
-template<> inline const Block_Response *State_Response_Message::state_response_as<Block_Response>() const {
+template<> inline const fbschema::p2pmsg::Block_Response *State_Response_Message::state_response_as<fbschema::p2pmsg::Block_Response>() const {
   return state_response_as_Block_Response();
 }
 
-template<> inline const Fs_Entry_Response *State_Response_Message::state_response_as<Fs_Entry_Response>() const {
+template<> inline const fbschema::p2pmsg::Fs_Entry_Response *State_Response_Message::state_response_as<fbschema::p2pmsg::Fs_Entry_Response>() const {
   return state_response_as_Fs_Entry_Response();
 }
 
 struct State_Response_MessageBuilder {
+  typedef State_Response_Message Table;
   flatbuffers::FlatBufferBuilder &fbb_;
   flatbuffers::uoffset_t start_;
-  void add_state_response_type(State_Response state_response_type) {
+  void add_state_response_type(fbschema::p2pmsg::State_Response state_response_type) {
     fbb_.AddElement<uint8_t>(State_Response_Message::VT_STATE_RESPONSE_TYPE, static_cast<uint8_t>(state_response_type), 0);
   }
   void add_state_response(flatbuffers::Offset<void> state_response) {
@@ -1172,7 +1278,7 @@ struct State_Response_MessageBuilder {
 
 inline flatbuffers::Offset<State_Response_Message> CreateState_Response_Message(
     flatbuffers::FlatBufferBuilder &_fbb,
-    State_Response state_response_type = State_Response_NONE,
+    fbschema::p2pmsg::State_Response state_response_type = fbschema::p2pmsg::State_Response_NONE,
     flatbuffers::Offset<void> state_response = 0,
     flatbuffers::Offset<flatbuffers::Vector<uint8_t>> hash = 0) {
   State_Response_MessageBuilder builder_(_fbb);
@@ -1184,7 +1290,7 @@ inline flatbuffers::Offset<State_Response_Message> CreateState_Response_Message(
 
 inline flatbuffers::Offset<State_Response_Message> CreateState_Response_MessageDirect(
     flatbuffers::FlatBufferBuilder &_fbb,
-    State_Response state_response_type = State_Response_NONE,
+    fbschema::p2pmsg::State_Response state_response_type = fbschema::p2pmsg::State_Response_NONE,
     flatbuffers::Offset<void> state_response = 0,
     const std::vector<uint8_t> *hash = nullptr) {
   auto hash__ = hash ? _fbb.CreateVector<uint8_t>(*hash) : 0;
@@ -1196,6 +1302,7 @@ inline flatbuffers::Offset<State_Response_Message> CreateState_Response_MessageD
 }
 
 struct Fs_Entry_Response FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
+  typedef Fs_Entry_ResponseBuilder Builder;
   enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
     VT_PATH = 4,
     VT_ENTRIES = 6
@@ -1206,11 +1313,11 @@ struct Fs_Entry_Response FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
   flatbuffers::String *mutable_path() {
     return GetPointer<flatbuffers::String *>(VT_PATH);
   }
-  const flatbuffers::Vector<flatbuffers::Offset<State_FS_Hash_Entry>> *entries() const {
-    return GetPointer<const flatbuffers::Vector<flatbuffers::Offset<State_FS_Hash_Entry>> *>(VT_ENTRIES);
+  const flatbuffers::Vector<flatbuffers::Offset<fbschema::p2pmsg::State_FS_Hash_Entry>> *entries() const {
+    return GetPointer<const flatbuffers::Vector<flatbuffers::Offset<fbschema::p2pmsg::State_FS_Hash_Entry>> *>(VT_ENTRIES);
   }
-  flatbuffers::Vector<flatbuffers::Offset<State_FS_Hash_Entry>> *mutable_entries() {
-    return GetPointer<flatbuffers::Vector<flatbuffers::Offset<State_FS_Hash_Entry>> *>(VT_ENTRIES);
+  flatbuffers::Vector<flatbuffers::Offset<fbschema::p2pmsg::State_FS_Hash_Entry>> *mutable_entries() {
+    return GetPointer<flatbuffers::Vector<flatbuffers::Offset<fbschema::p2pmsg::State_FS_Hash_Entry>> *>(VT_ENTRIES);
   }
   bool Verify(flatbuffers::Verifier &verifier) const {
     return VerifyTableStart(verifier) &&
@@ -1224,12 +1331,13 @@ struct Fs_Entry_Response FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
 };
 
 struct Fs_Entry_ResponseBuilder {
+  typedef Fs_Entry_Response Table;
   flatbuffers::FlatBufferBuilder &fbb_;
   flatbuffers::uoffset_t start_;
   void add_path(flatbuffers::Offset<flatbuffers::String> path) {
     fbb_.AddOffset(Fs_Entry_Response::VT_PATH, path);
   }
-  void add_entries(flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<State_FS_Hash_Entry>>> entries) {
+  void add_entries(flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<fbschema::p2pmsg::State_FS_Hash_Entry>>> entries) {
     fbb_.AddOffset(Fs_Entry_Response::VT_ENTRIES, entries);
   }
   explicit Fs_Entry_ResponseBuilder(flatbuffers::FlatBufferBuilder &_fbb)
@@ -1247,7 +1355,7 @@ struct Fs_Entry_ResponseBuilder {
 inline flatbuffers::Offset<Fs_Entry_Response> CreateFs_Entry_Response(
     flatbuffers::FlatBufferBuilder &_fbb,
     flatbuffers::Offset<flatbuffers::String> path = 0,
-    flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<State_FS_Hash_Entry>>> entries = 0) {
+    flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<fbschema::p2pmsg::State_FS_Hash_Entry>>> entries = 0) {
   Fs_Entry_ResponseBuilder builder_(_fbb);
   builder_.add_entries(entries);
   builder_.add_path(path);
@@ -1257,9 +1365,9 @@ inline flatbuffers::Offset<Fs_Entry_Response> CreateFs_Entry_Response(
 inline flatbuffers::Offset<Fs_Entry_Response> CreateFs_Entry_ResponseDirect(
     flatbuffers::FlatBufferBuilder &_fbb,
     const char *path = nullptr,
-    const std::vector<flatbuffers::Offset<State_FS_Hash_Entry>> *entries = nullptr) {
+    const std::vector<flatbuffers::Offset<fbschema::p2pmsg::State_FS_Hash_Entry>> *entries = nullptr) {
   auto path__ = path ? _fbb.CreateString(path) : 0;
-  auto entries__ = entries ? _fbb.CreateVector<flatbuffers::Offset<State_FS_Hash_Entry>>(*entries) : 0;
+  auto entries__ = entries ? _fbb.CreateVector<flatbuffers::Offset<fbschema::p2pmsg::State_FS_Hash_Entry>>(*entries) : 0;
   return fbschema::p2pmsg::CreateFs_Entry_Response(
       _fbb,
       path__,
@@ -1267,6 +1375,7 @@ inline flatbuffers::Offset<Fs_Entry_Response> CreateFs_Entry_ResponseDirect(
 }
 
 struct File_HashMap_Response FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
+  typedef File_HashMap_ResponseBuilder Builder;
   enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
     VT_PATH = 4,
     VT_FILE_LENGTH = 6,
@@ -1302,6 +1411,7 @@ struct File_HashMap_Response FLATBUFFERS_FINAL_CLASS : private flatbuffers::Tabl
 };
 
 struct File_HashMap_ResponseBuilder {
+  typedef File_HashMap_Response Table;
   flatbuffers::FlatBufferBuilder &fbb_;
   flatbuffers::uoffset_t start_;
   void add_path(flatbuffers::Offset<flatbuffers::String> path) {
@@ -1352,6 +1462,7 @@ inline flatbuffers::Offset<File_HashMap_Response> CreateFile_HashMap_ResponseDir
 }
 
 struct Block_Response FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
+  typedef Block_ResponseBuilder Builder;
   enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
     VT_PATH = 4,
     VT_BLOCK_ID = 6,
@@ -1387,6 +1498,7 @@ struct Block_Response FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
 };
 
 struct Block_ResponseBuilder {
+  typedef Block_Response Table;
   flatbuffers::FlatBufferBuilder &fbb_;
   flatbuffers::uoffset_t start_;
   void add_path(flatbuffers::Offset<flatbuffers::String> path) {
@@ -1437,6 +1549,7 @@ inline flatbuffers::Offset<Block_Response> CreateBlock_ResponseDirect(
 }
 
 struct State_FS_Hash_Entry FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
+  typedef State_FS_Hash_EntryBuilder Builder;
   enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
     VT_PATH = 4,
     VT_IS_FILE = 6,
@@ -1472,6 +1585,7 @@ struct State_FS_Hash_Entry FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table
 };
 
 struct State_FS_Hash_EntryBuilder {
+  typedef State_FS_Hash_Entry Table;
   flatbuffers::FlatBufferBuilder &fbb_;
   flatbuffers::uoffset_t start_;
   void add_path(flatbuffers::Offset<flatbuffers::String> path) {
@@ -1526,35 +1640,39 @@ inline bool VerifyMessage(flatbuffers::Verifier &verifier, const void *obj, Mess
     case Message_NONE: {
       return true;
     }
+    case Message_PeerId_Notify_Message: {
+      auto ptr = reinterpret_cast<const fbschema::p2pmsg::PeerId_Notify_Message *>(obj);
+      return verifier.VerifyTable(ptr);
+    }
     case Message_NonUnl_Proposal_Message: {
-      auto ptr = reinterpret_cast<const NonUnl_Proposal_Message *>(obj);
+      auto ptr = reinterpret_cast<const fbschema::p2pmsg::NonUnl_Proposal_Message *>(obj);
       return verifier.VerifyTable(ptr);
     }
     case Message_Proposal_Message: {
-      auto ptr = reinterpret_cast<const Proposal_Message *>(obj);
+      auto ptr = reinterpret_cast<const fbschema::p2pmsg::Proposal_Message *>(obj);
       return verifier.VerifyTable(ptr);
     }
     case Message_Npl_Message: {
-      auto ptr = reinterpret_cast<const Npl_Message *>(obj);
+      auto ptr = reinterpret_cast<const fbschema::p2pmsg::Npl_Message *>(obj);
       return verifier.VerifyTable(ptr);
     }
     case Message_State_Request_Message: {
-      auto ptr = reinterpret_cast<const State_Request_Message *>(obj);
+      auto ptr = reinterpret_cast<const fbschema::p2pmsg::State_Request_Message *>(obj);
       return verifier.VerifyTable(ptr);
     }
     case Message_State_Response_Message: {
-      auto ptr = reinterpret_cast<const State_Response_Message *>(obj);
+      auto ptr = reinterpret_cast<const fbschema::p2pmsg::State_Response_Message *>(obj);
       return verifier.VerifyTable(ptr);
     }
     case Message_History_Request_Message: {
-      auto ptr = reinterpret_cast<const History_Request_Message *>(obj);
+      auto ptr = reinterpret_cast<const fbschema::p2pmsg::History_Request_Message *>(obj);
       return verifier.VerifyTable(ptr);
     }
     case Message_History_Response_Message: {
-      auto ptr = reinterpret_cast<const History_Response_Message *>(obj);
+      auto ptr = reinterpret_cast<const fbschema::p2pmsg::History_Response_Message *>(obj);
       return verifier.VerifyTable(ptr);
     }
-    default: return false;
+    default: return true;
   }
 }
 
@@ -1576,18 +1694,18 @@ inline bool VerifyState_Response(flatbuffers::Verifier &verifier, const void *ob
       return true;
     }
     case State_Response_File_HashMap_Response: {
-      auto ptr = reinterpret_cast<const File_HashMap_Response *>(obj);
+      auto ptr = reinterpret_cast<const fbschema::p2pmsg::File_HashMap_Response *>(obj);
       return verifier.VerifyTable(ptr);
     }
     case State_Response_Block_Response: {
-      auto ptr = reinterpret_cast<const Block_Response *>(obj);
+      auto ptr = reinterpret_cast<const fbschema::p2pmsg::Block_Response *>(obj);
       return verifier.VerifyTable(ptr);
     }
     case State_Response_Fs_Entry_Response: {
-      auto ptr = reinterpret_cast<const Fs_Entry_Response *>(obj);
+      auto ptr = reinterpret_cast<const fbschema::p2pmsg::Fs_Entry_Response *>(obj);
       return verifier.VerifyTable(ptr);
     }
-    default: return false;
+    default: return true;
   }
 }
 

src/fbschema/p2pmsg_helpers.cpp

@@ -1,4 +1,3 @@
-#include <flatbuffers/flatbuffers.h>
 #include "../pchheader.hpp"
 #include "../conf.hpp"
 #include "../crypto.hpp"
@@ -28,7 +27,7 @@ namespace fbschema::p2pmsg
  * received data and then interprit the 'Content' portion of it separately to read the actual content.
  */
 
-//---Message validation and reading helpers---/
+//---Message validation helpers---/
 
 /**
  * Verifies Conatiner message structure and outputs faltbuffer Container pointer to access the given buffer.
@@ -139,6 +138,13 @@ int validate_and_extract_content(const Content **content_ref, const uint8_t *con
     return 0;
 }
 
+//---Message reading helpers---/
+
+std::string_view get_peerid_from_msg(const PeerId_Notify_Message &msg)
+{
+    return flatbuff_bytes_to_sv(msg.peerid());
+}
+
 /**
  * Creates a non-unl proposal stuct from the given non-unl proposal message.
  * @param The Flatbuffer non-unl poporal received from the peer.
@@ -252,6 +258,23 @@ const p2p::block_response create_block_response_from_msg(const Block_Response &m
 
 //---Message creation helpers---//
 
+void create_msg_from_peerid(flatbuffers::FlatBufferBuilder &container_builder, std::string_view peerid)
+{
+    flatbuffers::FlatBufferBuilder builder(1024);
+
+    const flatbuffers::Offset<PeerId_Notify_Message> peerid_msg =
+        CreatePeerId_Notify_Message(
+            builder,
+            sv_to_flatbuff_bytes(builder, peerid));
+
+    const flatbuffers::Offset<Content> message = CreateContent(builder, Message_PeerId_Notify_Message, peerid_msg.Union());
+    builder.Finish(message); // Finished building message content to get serialised content.
+
+    // Now that we have built the content message,
+    // we need to sign it and place it inside a container message.
+    create_containermsg_from_content(container_builder, builder, nullptr, false);
+}
+
 void create_msg_from_nonunl_proposal(flatbuffers::FlatBufferBuilder &container_builder, const p2p::nonunl_proposal &nup)
 {
     flatbuffers::FlatBufferBuilder builder(1024);

src/fbschema/p2pmsg_helpers.hpp

@@ -1,7 +1,6 @@
 #ifndef _HP_FBSCHEMA_P2PMSG_HELPERS_
 #define _HP_FBSCHEMA_P2PMSG_HELPERS_
 
-#include <flatbuffers/flatbuffers.h>
 #include "../pchheader.hpp"
 #include "p2pmsg_container_generated.h"
 #include "p2pmsg_content_generated.h"
@@ -14,7 +13,7 @@ namespace fbschema::p2pmsg
  * This section contains Flatbuffer p2p message reading/writing helpers.
  */
 
-//---Message validation and reading helpers---/
+//---Message validation helpers---/
 
 int validate_and_extract_container(const Container **container_ref, std::string_view container_buf);
 
@@ -22,6 +21,10 @@ int validate_container_trust(const Container *container);
 
 int validate_and_extract_content(const Content **content_ref, const uint8_t *content_ptr, const flatbuffers::uoffset_t content_size);
 
+//---Message reading helpers---/
+
+std::string_view get_peerid_from_msg(const PeerId_Notify_Message &msg);
+
 const p2p::nonunl_proposal create_nonunl_proposal_from_msg(const NonUnl_Proposal_Message &msg, const uint64_t timestamp);
 
 const p2p::proposal create_proposal_from_msg(const Proposal_Message &msg, const flatbuffers::Vector<uint8_t> *pubkey, const uint64_t timestamp, const flatbuffers::Vector<uint8_t> *lcl);
@@ -36,6 +39,8 @@ const p2p::block_response create_block_response_from_msg(const Block_Response &m
 
 //---Message creation helpers---//
 
+void create_msg_from_peerid(flatbuffers::FlatBufferBuilder &container_builder, std::string_view peerid);
+
 void create_msg_from_nonunl_proposal(flatbuffers::FlatBufferBuilder &container_builder, const p2p::nonunl_proposal &nup);
 
 void create_msg_from_proposal(flatbuffers::FlatBufferBuilder &container_builder, const p2p::proposal &p);

src/main.cpp

@@ -64,6 +64,9 @@ int parse_cmd(int argc, char **argv)
  */
 void deinit()
 {
+    usr::deinit();
+    p2p::deinit();
+    cons::deinit();
     proc::deinit();
     hplog::deinit();
 }
@@ -72,6 +75,7 @@ void signal_handler(int signum)
 {
     LOG_WARN << "Interrupt signal (" << signum << ") received.";
     deinit();
+    std::cout << "hpcore exiting\n";
     exit(signum);
 }
 
@@ -192,9 +196,7 @@ int main(int argc, char **argv)
                 signal(SIGINT, signal_handler);
 
                 while (true)
-                {
                     cons::consensus();
-                }
 
                 // Free resources.
                 deinit();

src/p2p/p2p.cpp

@@ -1,6 +1,6 @@
 #include "../pchheader.hpp"
-#include "../sock/socket_server.hpp"
-#include "../sock/socket_client.hpp"
+#include "../comm/comm_server.hpp"
+#include "../comm/comm_client.hpp"
 #include "../conf.hpp"
 #include "../crypto.hpp"
 #include "../util.hpp"
@@ -8,70 +8,104 @@
 #include "p2p.hpp"
 #include "peer_session_handler.hpp"
 
-namespace ssl = boost::asio::ssl;
-
 namespace p2p
 {
 
 // Holds global connected-peers and related objects.
 connected_context ctx;
 
-// Holds objects used by socket listener.
-listener_context listener_ctx;
-
+/**
+ * Initializes the p2p subsystem. Must be called once during application startup.
+ * @return 0 for successful initialization. -1 for failure.
+ */
 int init()
 {
     //Entry point for p2p which will start peer connections to other nodes
-    start_peer_connections();
+    return start_peer_connections();
+}
 
+/**
+ * Cleanup any running processes.
+ */
+void deinit()
+{
+    ctx.listener.stop();
+}
+
+int start_peer_connections()
+{
+    const uint64_t metric_thresholds[] = {conf::cfg.peermaxcpm, conf::cfg.peermaxdupmpm, conf::cfg.peermaxbadsigpm, conf::cfg.peermaxbadmpm};
+    if (ctx.listener.start(
+            conf::cfg.peerport, ".sock-peer", comm::SESSION_TYPE::PEER, true, metric_thresholds, conf::cfg.peers, conf::cfg.peermaxsize) == -1)
+        return -1;
+
+    LOG_INFO << "Started listening for peer connections on " << std::to_string(conf::cfg.peerport);
     return 0;
 }
 
-void start_peer_connections()
+int resolve_session_peerid(comm::comm_session &session, const std::string &peerid)
 {
-    boost::asio::ip::address address = net::ip::make_address(conf::cfg.listenip);
+    const int res = peerid.compare(conf::cfg.self_peerid);
 
-    // Setting up the message max size. Retrieve it from config
-    listener_ctx.default_sess_opts.max_socket_read_len = conf::cfg.peermaxsize;
-    listener_ctx.default_sess_opts.max_rawbytes_per_minute = conf::cfg.peermaxcpm;
-    listener_ctx.default_sess_opts.max_dupmsgs_per_minute = conf::cfg.peermaxdupmpm;
-    listener_ctx.default_sess_opts.max_badmsgs_per_minute = conf::cfg.peermaxbadmpm;
-    listener_ctx.default_sess_opts.max_badsigmsgs_per_minute = conf::cfg.peermaxbadsigpm;
+    // If peerid is same as our (self) peerid, then this is the loopback connection to ourselves.
+    // Hence we must keep the connection but only one of two sessions must be added to peer_connections.
+    // If peerid is greater than our id (< 0), then we should give priority to any existing inbound connection
+    // from the same peer and drop the outbound connection.
+    // If peerid is lower than our id (> 0), then we should give priority to any existing outbound connection
+    // from the same peer and drop the inbound connection.
 
-    // Start listening to peers
-    std::make_shared<sock::socket_server<peer_outbound_message>>(
-        listener_ctx.ioc,
-        listener_ctx.ssl_ctx,
-        tcp::endpoint{address, conf::cfg.peerport},
-        listener_ctx.global_peer_session_handler,
-        listener_ctx.default_sess_opts)
-        ->run();
+    std::lock_guard<std::mutex> lock(ctx.peer_connections_mutex);
 
-    LOG_INFO << "Started listening for incoming peer connections on " << conf::cfg.listenip << ":" << conf::cfg.peerport;
-
-    // Scan peers and trying to keep up the connections if drop. This action is run on a seperate thread.
-    ctx.peer_watchdog_thread = std::thread([&] { peer_connection_watchdog(); });
-
-    // Peer listener thread.
-    listener_ctx.listener_thread = std::thread([&] { listener_ctx.ioc.run(); });
-}
-
-// Scan peer connections continually and attempt to maintain the connection if they drop
-void peer_connection_watchdog()
-{
-    while (true)
+    const auto iter = p2p::ctx.peer_connections.find(peerid);
+    if (iter == p2p::ctx.peer_connections.end())
     {
-        for (const auto &[peerid, ipport] : conf::cfg.peers)
+        // Add the new connection straight away, if we haven't seen it before.
+        session.uniqueid = peerid;
+        session.flags.set(comm::SESSION_FLAG::PEERID_RESOLVED);
+        p2p::ctx.peer_connections.try_emplace(peerid, &session);
+        return 0;
+    }
+    else if (res == 0) // New connection is self (There can be two sessions for self (inbound/outbound))
+    {
+        session.is_self = true;
+        session.uniqueid = peerid;
+        session.flags.set(comm::SESSION_FLAG::PEERID_RESOLVED);
+        return 0;
+    }
+    else // New connection is not self but with same peer id.
+    {
+        comm::comm_session &ex_session = *iter->second;
+
+        // We don't allow duplicate connections to the same peer to same direction.
+        if (ex_session.is_inbound != session.is_inbound)
         {
-            if (ctx.peer_connections.find(peerid) == ctx.peer_connections.end())
+            // Decide whether we need to replace existing session with new session.
+            const bool replace_needed = ((res < 0 && !ex_session.is_inbound) || (res > 0 && ex_session.is_inbound));
+            if (replace_needed)
             {
-                LOG_DBG << "Trying to connect : " << peerid;
-                std::make_shared<sock::socket_client<peer_outbound_message>>(listener_ctx.ioc, listener_ctx.ssl_ctx, listener_ctx.global_peer_session_handler, listener_ctx.default_sess_opts)
-                    ->run(ipport.first, ipport.second);
+                // If we happen to replace a peer session with known IP, transfer required details to the new session.
+                if (session.known_ipport.first.empty())
+                    session.known_ipport.swap(ex_session.known_ipport);
+                session.uniqueid = peerid;
+                session.flags.set(comm::SESSION_FLAG::PEERID_RESOLVED);
+
+                ex_session.close(false);
+                p2p::ctx.peer_connections.erase(iter);                   // remove existing session.
+                p2p::ctx.peer_connections.try_emplace(peerid, &session); // add new session.
+
+                LOG_DBG << "Replacing existing connection [" << peerid << "]";
+                return 0;
+            }
+            else if (ex_session.known_ipport.first.empty() || !session.known_ipport.first.empty())
+            {
+                // If we have any known ip-port info from the new session, transfer them to the existing session.
+                ex_session.known_ipport.swap(session.known_ipport);
             }
         }
 
-        util::sleep(conf::cfg.roundtime * 4);
+        // Reaching this point means we don't need the new session.
+        LOG_DBG << "Rejecting new peer connection because existing connection takes priority [" << peerid << "]";
+        return -1;
     }
 }
 
@@ -80,7 +114,7 @@ void peer_connection_watchdog()
  * @param msg Peer outbound message to be broadcasted.
  * @param send_to_self Whether to also send the message to self (this node).
  */
-void broadcast_message(const peer_outbound_message msg, const bool send_to_self)
+void broadcast_message(const flatbuffers::FlatBufferBuilder &fbuf, const bool send_to_self)
 {
     if (ctx.peer_connections.size() == 0)
     {
@@ -96,6 +130,9 @@ void broadcast_message(const peer_outbound_message msg, const bool send_to_self)
     {
         if (!send_to_self && session->is_self)
             continue;
+
+        std::string_view msg = std::string_view(
+            reinterpret_cast<const char *>(fbuf.GetBufferPointer()), fbuf.GetSize());
         session->send(msg);
     }
 }
@@ -104,16 +141,19 @@ void broadcast_message(const peer_outbound_message msg, const bool send_to_self)
  * Sends the given message to self (this node).
  * @param msg Peer outbound message to be sent to self.
  */
-void send_message_to_self(const peer_outbound_message msg)
+void send_message_to_self(const flatbuffers::FlatBufferBuilder &fbuf)
 {
     //Send while locking the peer_connections.
     std::lock_guard<std::mutex> lock(p2p::ctx.peer_connections_mutex);
 
     // Find the peer session connected to self.
-    const auto peer_itr = ctx.peer_connections.find(conf::cfg.self_peer_id);
+    const auto peer_itr = ctx.peer_connections.find(conf::cfg.self_peerid);
     if (peer_itr != ctx.peer_connections.end())
     {
-        const auto session = peer_itr->second;
+        std::string_view msg = std::string_view(
+            reinterpret_cast<const char *>(fbuf.GetBufferPointer()), fbuf.GetSize());
+
+        const comm::comm_session *session = peer_itr->second;
         session->send(msg);
     }
 }
@@ -122,7 +162,7 @@ void send_message_to_self(const peer_outbound_message msg)
  * Sends the given message to a random peer (except self).
  * @param msg Peer outbound message to be sent to peer.
  */
-void send_message_to_random_peer(const peer_outbound_message msg)
+void send_message_to_random_peer(const flatbuffers::FlatBufferBuilder &fbuf)
 {
     //Send while locking the peer_connections.
     std::lock_guard<std::mutex> lock(p2p::ctx.peer_connections_mutex);
@@ -147,9 +187,12 @@ void send_message_to_random_peer(const peer_outbound_message msg)
         std::advance(it, random_peer_index); //move iterator to point to random selected peer.
 
         //send message to selected peer.
-        const auto session = it->second;
+        const comm::comm_session *session = it->second;
         if (!session->is_self) // Exclude self peer.
         {
+            std::string_view msg = std::string_view(
+                reinterpret_cast<const char *>(fbuf.GetBufferPointer()), fbuf.GetSize());
+
             session->send(msg);
             break;
         }

src/p2p/p2p.hpp

@@ -2,10 +2,13 @@
 #define _HP_P2P_
 
 #include "../pchheader.hpp"
-#include "../sock/socket_session.hpp"
+#include "../comm/comm_server.hpp"
+#include "../comm/comm_client.hpp"
+#include "../comm/comm_session.hpp"
 #include "../usr/user_input.hpp"
 #include "peer_session_handler.hpp"
 #include "../statefs/hasher.hpp"
+#include "../conf.hpp"
 
 namespace p2p
 {
@@ -59,30 +62,29 @@ struct npl_message
     std::string data;
 };
 
-
 // Represents a state request sent to a peer.
 struct state_request
 {
-    std::string parent_path;    // The requested file or dir path.
-    bool is_file;               // Whether the path is a file or dir.
-    int32_t block_id;           // Block id of the file if we are requesting for file block. Otherwise -1.
-    hasher::B2H expected_hash;  // The expected hash of the requested result.
+    std::string parent_path;   // The requested file or dir path.
+    bool is_file;              // Whether the path is a file or dir.
+    int32_t block_id;          // Block id of the file if we are requesting for file block. Otherwise -1.
+    hasher::B2H expected_hash; // The expected hash of the requested result.
 };
 
 // Represents state file system entry.
 struct state_fs_hash_entry
 {
-    bool is_file;       // Whether this is a file or dir.
-    hasher::B2H hash;   // Hash of the file or dir.
+    bool is_file;     // Whether this is a file or dir.
+    hasher::B2H hash; // Hash of the file or dir.
 };
 
 // Represents a file block data resposne.
 struct block_response
 {
-    std::string path;       // Path of the file.
-    uint32_t block_id;      // Id of the block where the data belongs to.
-    std::string_view data;  // The block data.
-    hasher::B2H hash;       // Hash of the bloc data.
+    std::string path;      // Path of the file.
+    uint32_t block_id;     // Id of the block where the data belongs to.
+    std::string_view data; // The block data.
+    hasher::B2H hash;      // Hash of the bloc data.
 };
 
 struct message_collection
@@ -106,46 +108,29 @@ struct connected_context
     // Holds all the messages until they are processed by consensus.
     message_collection collected_msgs;
 
-    // Set of currently connected outbound peer connections mapped by the uniqueid of socket session.
-    std::unordered_map<std::string, sock::socket_session<peer_outbound_message> *> peer_connections;
+    // Set of currently connected peer connections mapped by the uniqueid of socket session.
+    std::unordered_map<std::string, comm::comm_session *> peer_connections;
+
     std::mutex peer_connections_mutex; // Mutex for peer connections access race conditions.
 
-    // Peer connection watchdog runs on this thread.
-    std::thread peer_watchdog_thread;
+    comm::comm_server listener;
 };
 
 extern connected_context ctx;
 
-struct listener_context
-{
-    // Peer session handler instance. This instance's methods will be fired for any peer socket activity.
-    p2p::peer_session_handler global_peer_session_handler;
-
-    // IO context used by the  boost library in creating sockets
-    net::io_context ioc;
-
-    // SSL context used by the boost library in providing tls support
-    ssl::context ssl_ctx{ssl::context::tlsv13};
-
-    // The thread the peer listener is running on.
-    std::thread listener_thread;
-
-    // Used to pass down the default settings to the socket session
-    sock::session_options default_sess_opts;
-};
-
 int init();
 
-//p2p message handling
-void start_peer_connections();
+void deinit();
 
-void peer_connection_watchdog();
+int start_peer_connections();
 
-void broadcast_message(const peer_outbound_message msg, const bool send_to_self);
+int resolve_session_peerid(comm::comm_session &session, const std::string &peerid);
 
-void send_message_to_self(const peer_outbound_message msg);
+void broadcast_message(const flatbuffers::FlatBufferBuilder &fbuf, const bool send_to_self);
 
-void send_message_to_random_peer(const peer_outbound_message msg);
+void send_message_to_self(const flatbuffers::FlatBufferBuilder &fbuf);
+
+void send_message_to_random_peer(const flatbuffers::FlatBufferBuilder &fbuf);
 
 } // namespace p2p
 

src/p2p/peer_session_handler.cpp

@@ -7,8 +7,8 @@
 #include "../fbschema/p2pmsg_content_generated.h"
 #include "../fbschema/p2pmsg_helpers.hpp"
 #include "../fbschema/common_helpers.hpp"
-#include "../sock/socket_message.hpp"
-#include "../sock/socket_session.hpp"
+#include "../comm/comm_session.hpp"
+#include "../comm/comm_client.hpp"
 #include "p2p.hpp"
 #include "peer_session_handler.hpp"
 #include "../cons/ledger_handler.hpp"
@@ -26,32 +26,33 @@ util::rollover_hashset recent_peermsg_hashes(200);
 /**
  * This gets hit every time a peer connects to HP via the peer port (configured in contract config).
  */
-void peer_session_handler::on_connect(sock::socket_session<peer_outbound_message> *session)
+int peer_session_handler::on_connect(comm::comm_session &session) const
 {
-    if (session->flags[sock::SESSION_FLAG::INBOUND])
+    if (session.is_inbound)
     {
         // Limit max number of inbound connections.
         if (conf::cfg.peermaxcons > 0 && ctx.peer_connections.size() >= conf::cfg.peermaxcons)
         {
-            session->close();
-            LOG_DBG << "Max peer connections reached. Dropped connection " << session->uniqueid;
+            LOG_DBG << "Max peer connections reached. Dropped connection " << session.uniqueid;
+            return -1;
         }
     }
-    else
-    {
-        std::lock_guard<std::mutex> lock(ctx.peer_connections_mutex);
-        ctx.peer_connections.try_emplace(session->uniqueid, session);
-        LOG_DBG << "Adding peer to list: " << session->uniqueid;
-    }
+
+    // Send our peer id.
+    flatbuffers::FlatBufferBuilder fbuf(1024);
+    p2pmsg::create_msg_from_peerid(fbuf, conf::cfg.self_peerid);
+    std::string_view msg = std::string_view(
+        reinterpret_cast<const char *>(fbuf.GetBufferPointer()), fbuf.GetSize());
+    return session.send(msg);
 }
 
 //peer session on message callback method
 //validate and handle each type of peer messages.
-void peer_session_handler::on_message(sock::socket_session<peer_outbound_message> *session, std::string_view message)
+int peer_session_handler::on_message(comm::comm_session &session, std::string_view message) const
 {
     const p2pmsg::Container *container;
     if (p2pmsg::validate_and_extract_container(&container, message) != 0)
-        return;
+        return 0;
 
     //Get serialised message content.
     const flatbuffers::Vector<uint8_t> *container_content = container->content();
@@ -62,25 +63,41 @@ void peer_session_handler::on_message(sock::socket_session<peer_outbound_message
 
     const p2pmsg::Content *content;
     if (p2pmsg::validate_and_extract_content(&content, content_ptr, content_size) != 0)
-        return;
+        return 0;
 
     if (!recent_peermsg_hashes.try_emplace(crypto::get_hash(message)))
     {
-        session->increment_metric(sock::SESSION_THRESHOLDS::MAX_DUPMSGS_PER_MINUTE, 1);
-        LOG_DBG << "Duplicate peer message.";
-        return;
+        session.increment_metric(comm::SESSION_THRESHOLDS::MAX_DUPMSGS_PER_MINUTE, 1);
+        LOG_DBG << "Duplicate peer message. " << session.uniqueid;
+        return 0;
     }
 
     const p2pmsg::Message content_message_type = content->message_type(); //i.e - proposal, npl, state request, state response, etc
 
-    if (content_message_type == p2pmsg::Message_Proposal_Message) //message is a proposal message
+    if (content_message_type == p2pmsg::Message_PeerId_Notify_Message) // message is a peer id announcement
+    {
+        // Ignore if Peer ID is already resolved.
+        if (!session.flags[comm::SESSION_FLAG::PEERID_RESOLVED])
+        {
+            const std::string peerid = std::string(p2pmsg::get_peerid_from_msg(*content->message_as_PeerId_Notify_Message()));
+            return p2p::resolve_session_peerid(session, peerid);
+        }
+    }
+
+    if (!session.flags[comm::SESSION_FLAG::PEERID_RESOLVED])
+    {
+        LOG_DBG << "Cannot accept messages. Peer id unresolved. " << session.uniqueid;
+        return 0;
+    }
+
+    if (content_message_type == p2pmsg::Message_Proposal_Message) // message is a proposal message
     {
         // We only trust proposals coming from trusted peers.
         if (p2pmsg::validate_container_trust(container) != 0)
         {
-            session->increment_metric(sock::SESSION_THRESHOLDS::MAX_BADSIGMSGS_PER_MINUTE, 1);
-            LOG_DBG << "Proposal rejected due to trust failure.";
-            return;
+            session.increment_metric(comm::SESSION_THRESHOLDS::MAX_BADSIGMSGS_PER_MINUTE, 1);
+            LOG_DBG << "Proposal rejected due to trust failure. " << session.uniqueid;
+            return 0;
         }
 
         std::lock_guard<std::mutex> lock(ctx.collected_msgs.proposals_mutex); // Insert proposal with lock.
@@ -99,8 +116,8 @@ void peer_session_handler::on_message(sock::socket_session<peer_outbound_message
     {
         if (p2pmsg::validate_container_trust(container) != 0)
         {
-            LOG_DBG << "NPL message rejected due to trust failure.";
-            return;
+            LOG_DBG << "NPL message rejected due to trust failure. " << session.uniqueid;
+            return 0;
         }
 
         std::lock_guard<std::mutex> lock(ctx.collected_msgs.npl_messages_mutex); // Insert npl message with lock.
@@ -114,20 +131,24 @@ void peer_session_handler::on_message(sock::socket_session<peer_outbound_message
     }
     else if (content_message_type == p2pmsg::Message_State_Request_Message)
     {
-        if (p2pmsg::validate_container_trust(container) != 0)
-        {
-            LOG_DBG << "State request message rejected due to trust failure.";
-            return;
-        }
-
         const p2p::state_request sr = p2pmsg::create_state_request_from_msg(*content->message_as_State_Request_Message());
-        p2p::peer_outbound_message msg(std::make_unique<flatbuffers::FlatBufferBuilder>(1024));
+        flatbuffers::FlatBufferBuilder fbuf(1024);
 
-        if (cons::create_state_response(msg, sr) == 0)
-            session->send(std::move(msg));
+        if (cons::create_state_response(fbuf, sr) == 0)
+        {
+            std::string_view msg = std::string_view(
+                reinterpret_cast<const char *>(fbuf.GetBufferPointer()), fbuf.GetSize());
+            session.send(msg);
+        }
     }
     else if (content_message_type == p2pmsg::Message_State_Response_Message)
     {
+        if (p2pmsg::validate_container_trust(container) != 0)
+        {
+            LOG_DBG << "State response message rejected due to trust failure. " << session.uniqueid;
+            return 0;
+        }
+
         std::lock_guard<std::mutex> lock(ctx.collected_msgs.state_response_mutex); // Insert state_response with lock.
         std::string response(reinterpret_cast<const char *>(content_ptr), content_size);
         ctx.collected_msgs.state_response.push_back(std::move(response));
@@ -139,30 +160,38 @@ void peer_session_handler::on_message(sock::socket_session<peer_outbound_message
         const bool req_lcl_avail = cons::check_required_lcl_availability(hr);
         if (req_lcl_avail)
         {
-            p2p::peer_outbound_message hr_msg = cons::send_ledger_history(hr);
-            session->send(hr_msg);
+            flatbuffers::FlatBufferBuilder fbuf(1024);
+            p2pmsg::create_msg_from_history_response(fbuf, cons::retrieve_ledger_history(hr));
+            std::string_view msg = std::string_view(
+                reinterpret_cast<const char *>(fbuf.GetBufferPointer()), fbuf.GetSize());
+
+            session.send(msg);
         }
     }
     else if (content_message_type == p2pmsg::Message_History_Response_Message) //message is a lcl history response message
     {
+        if (p2pmsg::validate_container_trust(container) != 0)
+        {
+            LOG_DBG << "History response message rejected due to trust failure. " << session.uniqueid;
+            return 0;
+        }
+
         cons::handle_ledger_history_response(
             p2pmsg::create_history_response_from_msg(*content->message_as_History_Response_Message()));
     }
     else
     {
-        session->increment_metric(sock::SESSION_THRESHOLDS::MAX_BADMSGS_PER_MINUTE, 1);
-        LOG_DBG << "Received invalid message type from peer";
+        session.increment_metric(comm::SESSION_THRESHOLDS::MAX_BADMSGS_PER_MINUTE, 1);
+        LOG_DBG << "Received invalid peer message type. " << session.uniqueid;
     }
+    return 0;
 }
 
 //peer session on message callback method
-void peer_session_handler::on_close(sock::socket_session<peer_outbound_message> *session)
+void peer_session_handler::on_close(const comm::comm_session &session) const
 {
-    {
-        std::lock_guard<std::mutex> lock(ctx.peer_connections_mutex);
-        ctx.peer_connections.erase(session->uniqueid);
-    }
-    LOG_DBG << "Peer disonnected: " << session->uniqueid;
+    std::lock_guard<std::mutex> lock(ctx.peer_connections_mutex);
+    ctx.peer_connections.erase(session.uniqueid);
 }
 
 } // namespace p2p

src/p2p/peer_session_handler.hpp

@@ -2,21 +2,18 @@
 #define _HP_PEER_SESSION_HANDLER_
 
 #include "../pchheader.hpp"
-#include "../sock/socket_session_handler.hpp"
-#include "../sock/socket_session.hpp"
-#include "../sock/socket_message.hpp"
+#include "../comm/comm_session_handler.hpp"
+#include "../comm/comm_session.hpp"
 
 namespace p2p
 {
 
-class peer_session_handler : public sock::socket_session_handler<peer_outbound_message>
+class peer_session_handler : public comm::comm_session_handler
 {
 public:
-    void on_connect(sock::socket_session<peer_outbound_message> *session);
-
-    void on_message(sock::socket_session<peer_outbound_message> *session, std::string_view message);
-
-    void on_close(sock::socket_session<peer_outbound_message> *session);
+    int on_connect(comm::comm_session &session) const;
+    int on_message(comm::comm_session &session, std::string_view message) const;
+    void on_close(const comm::comm_session &session) const;
 };
 
 } // namespace p2p

src/pchheader.hpp

@@ -8,12 +8,6 @@
 
 #include <bitset>
 #include <boost/algorithm/string.hpp>
-#include <boost/asio.hpp>
-#include <boost/beast.hpp>
-#include <boost/beast/core.hpp>
-#include <boost/beast/ssl.hpp>
-#include <boost/beast/websocket.hpp>
-#include <boost/beast/websocket/ssl.hpp>
 #include <boost/filesystem.hpp>
 #include <boost/log/core.hpp>
 #include <boost/log/expressions.hpp>
@@ -28,23 +22,25 @@
 #include <boost/log/utility/setup/console.hpp>
 #include <boost/log/utility/setup/file.hpp>
 #include <boost/stacktrace.hpp>
-#include <boost/thread/thread.hpp>
 #include <chrono>
 #include <cstdarg>
 #include <cstdio>
 #include <fcntl.h>
+#include <flatbuffers/flatbuffers.h>
 #include <fstream>
 #include <iostream>
 #include <list>
 #include <math.h>
 #include <memory>
 #include <mutex>
+#include <poll.h>
 #include <queue>
 #include <rapidjson/document.h>
 #include <rapidjson/istreamwrapper.h>
 #include <rapidjson/ostreamwrapper.h>
 #include <rapidjson/prettywriter.h>
 #include <rapidjson/schema.h>
+#include <set>
 #include <sodium.h>
 #include <sstream>
 #include <stdlib.h>

src/sock/socket_client.cpp

@@ -1,86 +0,0 @@
-#include "socket_client.hpp"
-#include "../p2p/peer_session_handler.hpp"
-#include "../usr/user_session_handler.hpp"
-
-namespace sock
-{
-
-template <class T>
-socket_client<T>::socket_client(net::io_context &ioc, ssl::context &ctx, socket_session_handler<T> &session_handler, const session_options &session_options)
-    : resolver(net::make_strand(ioc)), ws(net::make_strand(ioc), ctx), sess_handler(session_handler), sess_opts(session_options)
-{
-}
-
-/**
- * Entry point to socket client which will intiate a connection to server
-*/
-// boost async_resolve function requires a port as a string because of that port is passed as a string
-template <class T>
-void socket_client<T>::run(std::string_view host, std::string_view port)
-{
-    this->host = host;
-    this->port = port;
-
-    // Look up the domain name
-    resolver.async_resolve(
-        host,
-        port,
-        [self = this->shared_from_this()](error ec, tcp::resolver::results_type results) {
-            self->on_resolve(ec, results);
-        });
-}
-
-/**
- * Executes on completion of resolving the server
-*/
-template <class T>
-void socket_client<T>::on_resolve(error ec, tcp::resolver::results_type results)
-{
-    if (ec)
-        socket_client_fail(ec, "socket_client_resolve");
-
-    // Make the connection on the IP address we get from a lookup
-    beast::get_lowest_layer(ws).async_connect(
-        results,
-        [self = this->shared_from_this()](error ec, tcp::resolver::results_type::endpoint_type type) {
-            self->on_connect(ec, type);
-        });
-}
-
-/**
- * Executes on completion of connecting to the server
-*/
-template <class T>
-void socket_client<T>::on_connect(error ec, tcp::resolver::results_type::endpoint_type)
-{
-    if (ec)
-    {
-        socket_client_fail(ec, "socket_client_connect");
-    }
-    else
-    {
-        //Creates a new socket session object
-        std::make_shared<socket_session<T>>(
-            std::move(ws), sess_handler)
-            ->run(std::move(host), std::move(port), false, sess_opts);
-    }
-}
-
-/**
- * Executes on error
-*/
-template <class T>
-void socket_client<T>::socket_client_fail(beast::error_code ec, char const *what)
-{
-    LOG_ERR << what << ": " << ec.message();
-}
-
-/**
- * Declaring templates with possible values for T because keeping all those in hpp file makes compile take a long time
- */ 
-template socket_client<p2p::peer_outbound_message>::socket_client(net::io_context &ioc, ssl::context &ctx, socket_session_handler<p2p::peer_outbound_message> &session_handler, const session_options &session_options);
-template void socket_client<p2p::peer_outbound_message>::run(std::string_view host, std::string_view port);
-
-template socket_client<usr::user_outbound_message>::socket_client(net::io_context &ioc, ssl::context &ctx, socket_session_handler<usr::user_outbound_message> &session_handler, const session_options &session_options);
-template void socket_client<usr::user_outbound_message>::run(std::string_view host, std::string_view port);
-} // namespace sock

src/sock/socket_client.hpp

@@ -1,49 +0,0 @@
-#ifndef _HP_SOCKET_CLIENT_
-#define _HP_SOCKET_CLIENT_
-
-#include "socket_session_handler.hpp"
-#include "../hplog.hpp"
-
-namespace beast = boost::beast;
-namespace net = boost::asio;
-namespace websocket = boost::beast::websocket;
-namespace ssl = boost::asio::ssl; 
-
-using tcp = net::ip::tcp;
-using error = boost::system::error_code;
-
-namespace sock
-{
-/** 
- * Represents an active WebSocket client connection
- * Based on the implementation from https://github.com/vinniefalco/CppCon2018
-*/
-template <class T>
-class socket_client : public std::enable_shared_from_this<socket_client<T>>
-{
-    tcp::resolver resolver;                                     // resolver used to resolve host and the port
-    websocket::stream<beast::ssl_stream<beast::tcp_stream>> ws; // web socket stream used to send and receive messages
-    std::string host;                                           // address of the server in which the client connects
-    std::string port;                                           // port of the server in which client connects
-    socket_session_handler<T> &sess_handler;                    // handler passed to gain access to websocket events
-    const session_options &sess_opts;                                 // session options needed to pass to session
-
-    void on_resolve(error ec, tcp::resolver::results_type results);
-
-    void on_connect(error ec, tcp::resolver::results_type::endpoint_type);
-
-    void on_close(error ec);
-
-    void socket_client_fail(beast::error_code ec, char const *what);
-
-    void on_write(error ec, std::size_t);
-
-public:
-    // Resolver and socket require an io_context
-    socket_client(net::io_context &ioc, ssl::context &ctx, socket_session_handler<T> &session_handler, const session_options &session_options);
-
-    //Entry point to the client which requires an active host and port
-    void run(std::string_view host, std::string_view port);
-};
-} // namespace sock
-#endif

src/sock/socket_message.cpp

@@ -1,44 +0,0 @@
-#include <flatbuffers/flatbuffers.h>
-#include "../pchheader.hpp"
-#include "socket_message.hpp"
-
-namespace usr
-{
-
-user_outbound_message::user_outbound_message(std::string &&msg)
-{
-    this->msg = std::move(msg);
-}
-
-// Returns the buffer that should be written to the socket.
-std::string_view user_outbound_message::buffer()
-{
-    return msg;
-}
-
-}
-
-namespace p2p
-{
-
-peer_outbound_message::peer_outbound_message(
-    std::shared_ptr<flatbuffers::FlatBufferBuilder> fbbuilder_ptr)
-{
-    this->fbbuilder_ptr = fbbuilder_ptr;
-}
-
-// Returns a reference to the flatbuffer builder object.
-flatbuffers::FlatBufferBuilder &peer_outbound_message::builder()
-{
-    return *fbbuilder_ptr;
-}
-
-// Returns a reference to the data buffer that must be written to the socket.
-std::string_view peer_outbound_message::buffer()
-{
-    return std::string_view(
-        reinterpret_cast<const char *>(fbbuilder_ptr->GetBufferPointer()),
-        fbbuilder_ptr->GetSize());
-}
-
-}

src/sock/socket_message.hpp

@@ -1,70 +0,0 @@
-#ifndef _HP_SOCKET_MESSAGE_
-#define _HP_SOCKET_MESSAGE_
-
-#include <flatbuffers/flatbuffers.h>
-#include "../pchheader.hpp"
-
-namespace sock
-{
-
-/**
- * Represents an outbound message that is sent with a websocket.
- * We use this class to wrap different object types holding actual message contents.
- * We use this mechanism to achieve end-to-end zero-copy between original message
- * content generator and websocket flush.
- */
-class outbound_message
-{
-public:
-    // Returns a pointer to the internal buffer owned by the message object.
-    // Contents of this buffer is the message that is sent/received with the socket.
-    virtual std::string_view buffer() = 0;
-};
-
-}
-
-namespace usr
-{
-
-/**
- * Represents a message (bytes) that is sent to a user.
- */
-class user_outbound_message : public sock::outbound_message
-{
-    // Contains message bytes.
-    std::string msg;
-
-public:
-    user_outbound_message(std::string &&_msg);
-
-    // Returns the buffer that should be written to the socket.
-    virtual std::string_view buffer();
-};
-
-}
-
-namespace p2p
-{
-
-/**
- * Represents a peer message generated using flatbuffer that must be sent to the socket.
- * We keep a shared_ptr of flatbuffer builder to support broadcasting the same message
- * on multiple connections without copying buffer contents.
- */
-class peer_outbound_message : public sock::outbound_message
-{
-    std::shared_ptr<flatbuffers::FlatBufferBuilder> fbbuilder_ptr;
-
-public:
-    peer_outbound_message(std::shared_ptr<flatbuffers::FlatBufferBuilder> _fbbuilder_ptr);
-
-    // Returns a reference to the flatbuffer builder object.
-    flatbuffers::FlatBufferBuilder& builder();
-    
-    // Returns a reference to the data buffer that must be written to the socket.
-    virtual std::string_view buffer();
-};
-
-}
-
-#endif

src/sock/socket_server.cpp

@@ -1,128 +0,0 @@
-#include "socket_server.hpp"
-#include "../p2p/peer_session_handler.hpp"
-#include "../usr/user_session_handler.hpp"
-
-namespace sock
-{
-
-template <class T>
-socket_server<T>::socket_server(net::io_context &ioc, ssl::context &ctx, tcp::endpoint endpoint, socket_session_handler<T> &session_handler, const session_options &session_options)
-    : acceptor(net::make_strand(ioc)), ioc(ioc), ctx(ctx), sess_handler(session_handler), sess_opts(session_options)
-{
-    error_code ec;
-
-    // Open the acceptor
-    acceptor.open(endpoint.protocol(), ec);
-    if (ec)
-    {
-        fail(ec, "open");
-        return;
-    }
-
-    // Allow address reuse
-    acceptor.set_option(net::socket_base::reuse_address(true));
-    if (ec)
-    {
-        fail(ec, "set_option");
-        return;
-    }
-
-    // Bind to the server address
-    acceptor.bind(endpoint, ec);
-    if (ec)
-    {
-        fail(ec, "bind");
-        return;
-    }
-
-    // Start listening for connections
-    acceptor.listen(
-        net::socket_base::max_listen_connections, ec);
-    if (ec)
-    {
-        fail(ec, "listen");
-        return;
-    }
-}
-
-/**
- * Entry point to socket server which accepts new connections
-*/
-template <class T>
-void socket_server<T>::run()
-{
-    // Adding ssl context options disallowing requests which supports sslv2 and sslv3 which have security vulnerabilitis
-    ctx.set_options(
-        boost::asio::ssl::context::default_workarounds |
-        boost::asio::ssl::context::no_sslv2 |
-        boost::asio::ssl::context::no_sslv3);
-
-    //Providing the certification file for ssl context
-    ctx.use_certificate_chain_file(conf::ctx.tls_cert_file);
-
-    // Providing key file for the ssl context
-    ctx.use_private_key_file(
-        conf::ctx.tls_key_file,
-        boost::asio::ssl::context::pem);
-
-    // Start accepting a connection
-    acceptor.async_accept(
-        net::make_strand(ioc),
-        beast::bind_front_handler(
-            &socket_server<T>::on_accept,
-            this->shared_from_this()));
-}
-
-/**
- * Executes on acceptance of new connection
-*/
-template <class T>
-void socket_server<T>::on_accept(error_code ec, tcp::socket socket)
-{
-    if (ec)
-    {
-        return fail(ec, "accept");
-    }
-    else
-    {
-        const std::string port = std::to_string(socket.remote_endpoint().port());
-        const std::string address = socket.remote_endpoint().address().to_string();
-
-        //Creating websocket stream required to pass to initiate a new session
-        websocket::stream<beast::ssl_stream<beast::tcp_stream>> ws(std::move(socket), ctx);
-
-        // Launch a new session for this connection
-        std::make_shared<socket_session<T>>(std::move(ws), sess_handler)
-            ->run(std::move(address), std::move(port), true, sess_opts);
-    }
-
-    // Accept another connection
-    acceptor.async_accept(
-        net::make_strand(ioc),
-        beast::bind_front_handler(
-            &socket_server<T>::on_accept,
-            this->shared_from_this()));
-}
-
-/**
- * Executes on error
-*/
-template <class T>
-void socket_server<T>::fail(error_code ec, char const *what)
-{
-    // Don't report on canceled operations
-    if (ec == net::error::operation_aborted)
-        return;
-    LOG_ERR << what << ": " << ec.message();
-}
-
-/**
- * Declaring templates with possible values for T because keeping all those in hpp file makes compile take a long time
- */ 
-template socket_server<p2p::peer_outbound_message>::socket_server(net::io_context &ioc, ssl::context &ctx, tcp::endpoint endpoint, socket_session_handler<p2p::peer_outbound_message> &session_handler, const session_options &session_options);
-template void socket_server<p2p::peer_outbound_message>::run();
-
-template socket_server<usr::user_outbound_message>::socket_server(net::io_context &ioc, ssl::context &ctx, tcp::endpoint endpoint, socket_session_handler<usr::user_outbound_message> &session_handler, const session_options &session_options);
-template void socket_server<usr::user_outbound_message>::run();
-
-} // namespace sock

src/sock/socket_server.hpp

@@ -1,44 +0,0 @@
-#ifndef _HP_SOCKET_SERVER_
-#define _HP_SOCKET_SERVER_
-
-#include "socket_session_handler.hpp"
-#include "../conf.hpp"
-#include "../hplog.hpp"
-
-namespace net = boost::asio;      // namespace asio
-namespace ssl = boost::asio::ssl;
-
-using tcp = net::ip::tcp;
-using error_code = boost::system::error_code;
-
-namespace sock
-{
-
-/** 
- * Represents an active WebSocket server connection
- * Based on the implementation from https://github.com/vinniefalco/CppCon2018
-*/
-template <class T>
-class socket_server : public std::enable_shared_from_this<socket_server<T>>
-{
-    tcp::acceptor acceptor;                  // acceptor which accepts new connections
-    net::io_context &ioc;                    // socket in which the client connects
-    ssl::context &ctx;                       // ssl context which provides support for tls
-    socket_session_handler<T> &sess_handler; // handler passed to gain access to websocket events
-    const session_options &sess_opts;              // session options needed to pass to session
-
-    void fail(error_code ec, char const *what);
-
-    void on_accept(error_code ec, tcp::socket socket);
-
-public:
-    socket_server(net::io_context &ioc, ssl::context &ctx, tcp::endpoint endpoint, socket_session_handler<T> &session_handler, const session_options &session_options);
-
-    // Start accepting incoming connections
-    void run();
-};
-
-
-} // namespace sock
-
-#endif

src/sock/socket_session.cpp

@@ -1,336 +0,0 @@
-#include "../bill/corebill.h"
-#include "socket_session.hpp"
-#include "socket_message.hpp"
-#include "socket_session_handler.hpp"
-
-namespace sock
-{
-
-// Constructor
-template <class T>
-socket_session<T>::socket_session(websocket::stream<beast::ssl_stream<beast::tcp_stream>> websocket, socket_session_handler<T> &sess_handler)
-    : ws(std::move(websocket)), sess_handler(sess_handler)
-{
-    // We use binary data instead of ASCII/UTF8 character data.
-    ws.binary(true);
-}
-
-/**
- * Sets the largest permissible incoming data length in a single receive. If exceeds over this limit will cause
- * a protocol failure. Because this is internally handled by beast socket, we don't use socket_threshold struct
- * to handle this.
-*/
-template <class T>
-void socket_session<T>::set_max_socket_read_len(const uint64_t size)
-{
-    ws.read_message_max(size);
-}
-
-/**
- * Set thresholds to the socket session
-*/
-template <class T>
-void socket_session<T>::set_threshold(const SESSION_THRESHOLDS threshold_type, const uint64_t threshold_limit, const uint32_t intervalms)
-{
-    session_threshold &t = thresholds[threshold_type];
-    t.counter_value = 0;
-    t.intervalms = intervalms;
-    t.threshold_limit = threshold_limit;
-}
-
-/*
-* Increment the provided thresholds counter value with the provided amount and validate it against the
-* configured threshold limit.
-*/
-template <class T>
-void socket_session<T>::increment_metric(const SESSION_THRESHOLDS threshold_type, const uint64_t amount)
-{
-    session_threshold &t = thresholds[threshold_type];
-
-    // Ignore the counter if limit is set as 0.
-    if (t.threshold_limit == 0)
-        return;
-
-    const uint64_t time_now = util::get_epoch_milliseconds();
-
-    t.counter_value += amount;
-    if (t.timestamp == 0)
-    {
-        // Reset counter timestamp.
-        t.timestamp = time_now;
-    }
-    else
-    {
-        // Check whether we have exceeded the threshold within the monitering interval.
-        const uint64_t elapsed_time = time_now - t.timestamp;
-        if (elapsed_time <= t.intervalms && t.counter_value > t.threshold_limit)
-        {
-            this->close();
-
-            t.timestamp = 0;
-            t.counter_value = 0;
-
-            LOG_INFO << "Session " << this->uniqueid << " threshold exceeded. (type:" << threshold_type << " limit:" << t.threshold_limit << ")";
-            corebill::report_violation(this->address);
-        }
-        else if (elapsed_time > t.intervalms)
-        {
-            t.timestamp = time_now;
-            t.counter_value = amount;
-        }
-    }
-}
-
-//port and address will be used to identify from which remote party the message recieved in the handler
-template <class T>
-void socket_session<T>::run(const std::string &&address, const std::string &&port, const bool is_server_session, const session_options &sess_opts)
-{
-    if (sess_opts.max_socket_read_len > 0)
-    {
-        // Setting maximum data size within a single message. This is handled within the beast socket.
-        set_max_socket_read_len(sess_opts.max_socket_read_len);
-    }
-
-    // Create new session_thresholds and insert it to thresholds vector.
-    // Have to maintain the SESSION_THRESHOLDS enum order in inserting new thresholds to thresholds vector
-    // since enum's value is used as index in the vector to update vector values.
-    thresholds.reserve(4);
-    thresholds.push_back(session_threshold(sess_opts.max_rawbytes_per_minute, 60000));
-    thresholds.push_back(session_threshold(sess_opts.max_dupmsgs_per_minute, 60000));
-    thresholds.push_back(session_threshold(sess_opts.max_badsigmsgs_per_minute, 60000));
-    thresholds.push_back(session_threshold(sess_opts.max_badmsgs_per_minute, 60000));
-
-    const ssl::stream_base::handshake_type handshake_type =
-        is_server_session ? ssl::stream_base::server : ssl::stream_base::client;
-
-    // Set this flag to identify whether this socket session created when node acts as a server
-    // INBOUND true - when node acts as server
-    // INBOUND false (OUTBOUND) - when node acts as client
-    if (is_server_session)
-        flags.set(SESSION_FLAG::INBOUND);
-
-    this->port = std::move(port);
-    this->address = std::move(address);
-
-    // Create a unique id for the session combining ip and port.
-    // We prepare this appended string here because we need to use it as an identifier of the session in various places.
-    this->uniqueid.reserve(port.size() + address.size() + 1);
-    this->uniqueid.append(address).append(":").append(port);
-
-    // This indicates the connection is a self connection (node connects to the same node through server port)
-    if (this->uniqueid == conf::cfg.self_peer_id)
-        this->is_self = true;
-
-    // Set the timeout.
-    beast::get_lowest_layer(ws).expires_after(std::chrono::seconds(30));
-
-    // Perform the SSL handshake
-    ws_next_layer_async_handshake(handshake_type);
-}
-
-/*
-* Close an active websocket connection gracefully
-*/
-template <class T>
-void socket_session<T>::on_ssl_handshake(const error_code ec)
-{
-    if (ec)
-        return fail(ec, "handshake");
-
-    // Turn off the timeout on the tcp_stream, because
-    // the websocket stream has its own timeout system.
-    beast::get_lowest_layer(ws).expires_never();
-
-    if (flags[SESSION_FLAG::INBOUND])
-    {
-        // Set suggested timeout settings for the websocket
-        ws.set_option(
-            websocket::stream_base::timeout::suggested(
-                beast::role_type::server));
-
-        // Accept the websocket handshake
-        ws_async_accept();
-    }
-    else
-    {
-        ws.set_option(
-            websocket::stream_base::timeout::suggested(
-                beast::role_type::client));
-
-        // Perform the websocket handshake
-        ws_async_handshake();
-    }
-}
-
-/**
- * Executes on acceptance of new connection
-*/
-template <class T>
-void socket_session<T>::on_accept(const error_code ec)
-{
-    // Handle the error, if any
-    if (ec)
-        return fail(ec, "accept");
-
-    if (corebill::is_banned(this->address))
-    {
-        LOG_DBG << "Dropping connection for banned host " << this->address;
-        this->close();
-    }
-
-    sess_handler.on_connect(this);
-
-    // Read a message
-    ws_async_read();
-}
-
-/*
-* Executes on completion of recieiving a new message
-*/
-template <class T>
-void socket_session<T>::on_read(const error_code ec, const std::size_t)
-{
-    //if something goes wrong when trying to read, socket connection will be closed and calling this to inform it to the handler
-    // read may get called when operation_aborted as well.
-    // We don't need to process read operation in that case.
-    if (ec == net::error::operation_aborted)
-        return;
-
-    // Handle the error, if any
-    if (ec)
-    {
-        // if something goes wrong when trying to read, socket connection will be closed and calling this to inform it to the handler
-        on_close(ec, 1);
-        return fail(ec, "read");
-    }
-
-    increment_metric(SESSION_THRESHOLDS::MAX_RAWBYTES_PER_MINUTE, buffer.size());
-
-    // Wrap the buffer data in a string_view and call session handler.
-    // We DO NOT transfer ownership of buffer data to the session handler. It should
-    // read and process the message and we will clear the buffer after its done with it.
-    const char *buffer_data = net::buffer_cast<const char *>(buffer.data());
-    std::string_view message(buffer_data, buffer.size());
-    sess_handler.on_message(this, message);
-
-    // Clear the buffer
-    buffer.consume(buffer.size());
-
-    // Read another message
-    ws_async_read();
-}
-
-/*
-* Send message through an active websocket connection
-*/
-template <class T>
-void socket_session<T>::send(const T msg)
-{
-    try
-    {
-        std::lock_guard<std::mutex> lock(send_mutex);
-        
-        // Always add to queue
-        queue.push_back(std::move(msg));
-        //using sync write until async_write is properly handled for multi-threaded writes.
-        ws.write(net::buffer(queue.front().buffer()));
-        queue.erase(queue.begin());
-    }
-    catch (...)
-    {
-        this->handle_exception("sync_write");
-    }
-
-    // Are we already writing?
-    // if (queue.size() > 1)
-    //     return;
-
-    // std::string_view sv = queue.front().buffer();
-
-    // // We are not currently writing, so send this immediately
-    // ws_async_write(sv);
-}
-
-/*
-* Executes on completion of write operation to a socket
-*/
-template <class T>
-void socket_session<T>::on_write(const error_code ec, const std::size_t)
-{
-    // Handle the error, if any
-    if (ec)
-        return fail(ec, "write");
-
-    // Remove the string from the queue
-    queue.erase(queue.begin());
-
-    // Send the next message if any
-    if (!queue.empty())
-    {
-        std::string_view sv = queue.front().buffer();
-        ws_async_write(sv);
-    }
-}
-
-/*
-* Close an active websocket connection gracefully
-*/
-template <class T>
-void socket_session<T>::close()
-{
-    // Close the WebSocket connection
-    ws_async_close();
-}
-
-/*
-* Executes on completion of closing a socket connection
-*/
-//type will be used identify whether the error is due to failure in closing the web socket or transfer of another exception to this method
-template <class T>
-void socket_session<T>::on_close(const error_code ec, const int8_t type)
-{
-    sess_handler.on_close(this);
-
-    if (type == 1)
-        return;
-
-    if (ec)
-        return fail(ec, "close");
-}
-
-/**
- * Executes on error
-*/
-template <class T>
-void socket_session<T>::fail(const error_code ec, char const *what)
-{
-    LOG_ERR << what << ": " << ec.message();
-
-    // Don't report these
-    if (ec == net::error::operation_aborted ||
-        ec == websocket::error::closed)
-        return;
-}
-
-template <class T>
-void socket_session<T>::handle_exception(std::string_view event_name)
-{
-    std::exception_ptr p = std::current_exception();
-    LOG_ERR << "Socket Exception on " << event_name << ": " << (p ? p.__cxa_exception_type()->name() : "null") << " :"<< boost::stacktrace::stacktrace()<< std::endl;
-
-    // Close the socket on any event error except close event.
-    if (event_name != "close")
-        this->ws_async_close();
-}
-
-template <class T>
-socket_session<T>::~socket_session()
-{
-    sess_handler.on_close(this);
-}
-
-// Template instantiations.
-template class socket_session<p2p::peer_outbound_message>;
-template class socket_session<usr::user_outbound_message>;
-
-} // namespace sock

src/sock/socket_session.hpp

@@ -1,167 +0,0 @@
-#ifndef _HP_SOCKET_SESSION_
-#define _HP_SOCKET_SESSION_
-
-#include "../pchheader.hpp"
-#include "../util.hpp"
-#include "../hplog.hpp"
-
-namespace beast = boost::beast;
-namespace net = boost::asio;
-namespace websocket = boost::beast::websocket;
-namespace http = boost::beast::http;
-namespace ssl = boost::asio::ssl;
-using error_code = boost::system::error_code;
-
-namespace sock
-{
-
-/**
- * Set of flags used to mark status information on the session.
- * usr and p2p subsystems makes use of this to mark status information of user and peer sessions.
- * Set flags are stored in 'flags' bitset of socket_session.
- */
-enum SESSION_FLAG
-{
-    INBOUND = 0,
-    USER_CHALLENGE_ISSUED = 1,
-    USER_AUTHED = 2
-};
-
-/**
- * Enum used to track down various thresholds used in socket communication.
- */
-enum SESSION_THRESHOLDS
-{
-    // Max incoming bytes per minute.
-    MAX_RAWBYTES_PER_MINUTE = 0,
-
-    // Max duplicate messages per minute.
-    MAX_DUPMSGS_PER_MINUTE = 1,
-
-    // Max messages with invalid signature per minute.
-    MAX_BADSIGMSGS_PER_MINUTE = 2,
-
-    // Max messages with bad structure per minute.
-    MAX_BADMSGS_PER_MINUTE = 3
-};
-
-/*
-* Use this to keep in track of different thresholds which we need to deal with. e.g - maximum amount of bytes allowed per minute through a session
-* threshold_limit - Maximum threshold value which is allowed
-* counter_value - Counter which keeps incrementing per every message
-* timestamp - Timestamp when counter value changes
-* intervalms - Time interval in miliseconds in which the threshold and the counter value should be compared
-*/
-struct session_threshold
-{
-    uint64_t threshold_limit;
-    uint64_t counter_value;
-    uint64_t timestamp;
-    uint32_t intervalms;
-
-    session_threshold(uint64_t threshold_limit, uint32_t intervalms)
-    {
-        this->threshold_limit = threshold_limit;
-        this->intervalms = intervalms;
-    }
-};
-
-// Use this to feed the session with default options from the config file
-struct session_options
-{
-    uint64_t max_socket_read_len;
-    uint64_t max_rawbytes_per_minute;
-    uint64_t max_dupmsgs_per_minute;
-    uint64_t max_badsigmsgs_per_minute;
-    uint64_t max_badmsgs_per_minute;
-};
-
-//Forward Declaration
-template <class T>
-class socket_session_handler;
-
-/** 
- * Represents an active WebSocket connection
-*/
-template <class T>
-class socket_session : public std::enable_shared_from_this<socket_session<T>>
-{
-    beast::flat_buffer buffer;                                  // used to store incoming messages
-    websocket::stream<beast::ssl_stream<beast::tcp_stream>> ws; // websocket stream used send an recieve messages
-    std::vector<T> queue;                                       // used to store messages temporarily until it is sent to the relevant party
-    socket_session_handler<T> &sess_handler;                    // handler passed to gain access to websocket events
-    std::vector<session_threshold> thresholds;                  // track down various communication thresholds
-    std::mutex send_mutex;                                      // mutex for calling send()
-
-    void fail(const error_code ec, char const *what);
-
-    void on_ssl_handshake(const error_code ec);
-
-    void on_accept(const error_code ec);
-
-    void on_read(const error_code ec, const std::size_t bytes_transferred);
-
-    void on_write(const error_code ec, const std::size_t bytes_transferred);
-
-    void on_close(const error_code ec, const int8_t type);
-
-
-    // Websocket lambda expression helpers.
-    // Implementation of these are separated to a different .cpp to reduce regular compile time.
-
-    void ws_next_layer_async_handshake(const ssl::stream_base::handshake_type handshake_type);
-
-    void ws_async_accept();
-
-    void ws_async_handshake();
-
-    void ws_async_read();
-
-    void ws_async_write(std::string_view message);
-
-    void ws_async_close();
-
-    void handle_exception(std::string_view event_name);
-
-public:
-   
-    socket_session(websocket::stream<beast::ssl_stream<beast::tcp_stream>> websocket, socket_session_handler<T> &sess_handler);
-
-    ~socket_session();
-
-    // Port and the address of the remote party is being saved to used in the session handler
-    // to identify from which remote party the message recieved. Since the port is passed as a string
-    // from the parent we store as it is, since we are not going to pass it anywhere or used in a method
-
-    // The port of the remote party.
-    std::string port;
-
-    // The IP address of the remote party.
-    std::string address;
-
-    // The unique identifier of the remote party (format <ip>:<port>).
-    std::string uniqueid;
-
-    // Boolean value to store whether the session is self connection (connect to the same node)
-    bool is_self;
-
-    // The set of sock::SESSION_FLAG enum flags that will be set by user-code of this calss.
-    // We mainly use this to store contexual information about this session based on the use case.
-    // Setting and reading flags to this is completely managed by user-code.
-    std::bitset<8> flags;
-
-    void set_max_socket_read_len(const uint64_t size);
-
-    void set_threshold(const SESSION_THRESHOLDS threshold_type, const uint64_t threshold_limit, const uint32_t intervalms);
-
-    void increment_metric(const SESSION_THRESHOLDS threshold_type, const uint64_t amount);
-
-    void run(const std::string &&address, const std::string &&port, const bool is_server_session, const session_options &sess_opts);
-
-    void send(const T msg);
-
-    void close();
-};
-
-} // namespace sock
-#endif

src/sock/socket_session_handler.hpp

@@ -1,37 +0,0 @@
-#ifndef _HP_SOCKET_SESSION_HANDLER_
-#define _HP_SOCKET_SESSION_HANDLER_
-
-#include "socket_session.hpp"
-
-namespace sock
-{
-
-// Forward declaration
-template <class T>
-class socket_session;
-
-/** 
- * Represents a WebSocket sessions handler. Can inherit from this class and access websocket events
-*/
-template <class T>
-class socket_session_handler
-{
-public:
-    /**
-     * Executes on initiation of a new connection
-    */
-    virtual void on_connect(socket_session<T> *session) = 0;
-
-    /**
-     * Executes on recieval of new message
-    */
-    virtual void on_message(socket_session<T> *session, std::string_view message) = 0;
-
-    /**
-     * Executes on websocket connection close
-    */
-    virtual void on_close(socket_session<T> *session) = 0;
-};
-} // namespace sock
-
-#endif

src/sock/socket_session_lambda.cpp

@@ -1,128 +0,0 @@
-#include "../pchheader.hpp"
-#include "socket_message.hpp"
-#include "socket_session.hpp"
-
-namespace beast = boost::beast;
-namespace net = boost::asio;
-namespace websocket = boost::beast::websocket;
-namespace http = boost::beast::http;
-namespace ssl = boost::asio::ssl;
-using error_code = boost::system::error_code;
-
-namespace sock
-{
-
-// The following functions exist to separate out beast web sockets lambda expressions from other code.
-// This reduces lambda expression compilation time in regular code changes as long as this file is not touched.
-
-template <class T>
-void socket_session<T>::ws_next_layer_async_handshake(const ssl::stream_base::handshake_type handshake_type)
-{
-    try
-    {
-        // Perform the SSL handshake
-        ws.next_layer().async_handshake(
-            handshake_type,
-            [sp = this->shared_from_this()](error_code ec) {
-                sp->on_ssl_handshake(ec);
-            });
-    }
-    catch (...)
-    {
-        this->handle_exception("ssl_handshake");
-    }
-}
-
-template <class T>
-void socket_session<T>::ws_async_accept()
-{
-    try
-    {
-        ws.async_accept(
-            [sp = this->shared_from_this()](
-                error_code ec) {
-                sp->on_accept(ec);
-            });
-    }
-    catch (...)
-    {
-        this->handle_exception("accept");
-    }
-}
-
-template <class T>
-void socket_session<T>::ws_async_handshake()
-{
-    try
-    {
-        ws.async_handshake(this->address, "/",
-                           [sp = this->shared_from_this()](
-                               error_code ec) {
-                               sp->on_accept(ec);
-                           });
-    }
-    catch (...)
-    {
-        this->handle_exception("handshake");
-    }
-}
-
-template <class T>
-void socket_session<T>::ws_async_read()
-{
-    try
-    {
-        ws.async_read(
-            buffer,
-            [sp = this->shared_from_this()](
-                error_code ec, std::size_t bytes) {
-                sp->on_read(ec, bytes);
-            });
-    }
-    catch (...)
-    {
-        this->handle_exception("read");
-    }
-}
-
-template <class T>
-void socket_session<T>::ws_async_write(std::string_view message)
-{
-    try
-    {
-        ws.async_write(
-            // Project the outbound_message buffer from the queue front into the asio buffer.
-            net::buffer(message.data(), message.length()),
-            [sp = this->shared_from_this()](
-                error_code ec, std::size_t bytes) {
-                sp->on_write(ec, bytes);
-            });
-    }
-    catch (...)
-    {
-        this->handle_exception("write");
-    }
-}
-
-template <class T>
-void socket_session<T>::ws_async_close()
-{
-    try
-    {
-        ws.async_close(websocket::close_code::normal,
-                       [sp = this->shared_from_this()](
-                           error_code ec) {
-                           sp->on_close(ec, 0);
-                       });
-    }
-    catch (...)
-    {
-        this->handle_exception("close");
-    }
-}
-
-// Template instantiations.
-template class socket_session<p2p::peer_outbound_message>;
-template class socket_session<usr::user_outbound_message>;
-
-} // namespace sock

src/usr/user_session_handler.cpp

@@ -1,14 +1,11 @@
 #include "../pchheader.hpp"
 #include "../hplog.hpp"
 #include "../jsonschema/usrmsg_helpers.hpp"
-#include "../sock/socket_session.hpp"
-#include "../sock/socket_message.hpp"
 #include "../bill/corebill.h"
 #include "usr.hpp"
 #include "user_session_handler.hpp"
 
-namespace net = boost::asio;
-namespace beast = boost::beast;
+namespace jusrmsg = jsonschema::usrmsg;
 
 namespace usr
 {
@@ -16,89 +13,81 @@ namespace usr
 /**
  * This gets hit every time a client connects to HP via the public port (configured in contract config).
  */
-void user_session_handler::on_connect(sock::socket_session<user_outbound_message> *session)
+int user_session_handler::on_connect(comm::comm_session &session) const
 {
     if (conf::cfg.pubmaxcons > 0 && ctx.users.size() >= conf::cfg.pubmaxcons)
     {
-        session->close();
-        LOG_DBG << "Max user connections reached. Dropped connection " << session->uniqueid;
-        return;
+        LOG_DBG << "Max user connections reached. Dropped connection " << session.uniqueid;
+        return -1;
     }
 
-    LOG_DBG << "User client connected " << session->uniqueid;
+    LOG_DBG << "User client connected " << session.uniqueid;
 
     // As soon as a user connects, we issue them a challenge message. We remember the
     // challenge we issued and later verifies the user's response with it.
-    session->send(
-        user_outbound_message(issue_challenge(session->uniqueid)));
+    std::string msgstr;
+    jusrmsg::create_user_challenge(msgstr, session.issued_challenge);
+    session.send(msgstr);
 
     // Set the challenge-issued flag to help later checks in on_message.
-    session->flags.set(sock::SESSION_FLAG::USER_CHALLENGE_ISSUED);
+    session.flags.set(comm::SESSION_FLAG::USER_CHALLENGE_ISSUED);
+
+    return 0;
 }
 
 /**
  * This gets hit every time we receive some data from a client connected to the HP public port.
  */
-void user_session_handler::on_message(
-    sock::socket_session<user_outbound_message> *session,
-    std::string_view message)
+int user_session_handler::on_message(comm::comm_session &session, std::string_view message) const
 {
     // First check whether this session is pending challenge.
     // Meaning we have previously issued a challenge to the client,
-    if (session->flags[sock::SESSION_FLAG::USER_CHALLENGE_ISSUED])
+    if (session.flags[comm::SESSION_FLAG::USER_CHALLENGE_ISSUED])
     {
         if (verify_challenge(message, session) == 0)
-            return;
+            return 0;
     }
     // Check whether this session belongs to an authenticated (challenge-verified) user.
-    else if (session->flags[sock::SESSION_FLAG::USER_AUTHED])
+    else if (session.flags[comm::SESSION_FLAG::USER_AUTHED])
     {
         // Check whether this user is among authenticated users
         // and perform authenticated msg processing.
 
-        const auto itr = ctx.users.find(session->uniqueid);
+        const auto itr = ctx.users.find(session.uniqueid);
         if (itr != ctx.users.end())
         {
             // This is an authed user.
             connected_user &user = itr->second;
             if (handle_user_message(user, message) != 0)
             {
-                session->increment_metric(sock::SESSION_THRESHOLDS::MAX_BADMSGS_PER_MINUTE, 1);
-                LOG_DBG << "Bad message from user " << session->uniqueid;
+                session.increment_metric(comm::SESSION_THRESHOLDS::MAX_BADMSGS_PER_MINUTE, 1);
+                LOG_DBG << "Bad message from user " << session.uniqueid;
             }
         }
         else
         {
-            session->increment_metric(sock::SESSION_THRESHOLDS::MAX_BADMSGS_PER_MINUTE, 1);
-            LOG_DBG << "User session id not found: " << session->uniqueid;
+            session.increment_metric(comm::SESSION_THRESHOLDS::MAX_BADMSGS_PER_MINUTE, 1);
+            LOG_DBG << "User session id not found: " << session.uniqueid;
         }
 
-        return;
+        return 0;
     }
 
     // If for any reason we reach this point, we should drop the connection because none of the
     // valid cases match.
-    session->close();
-    LOG_DBG << "Dropped the user connection " << session->uniqueid;
-    corebill::report_violation(session->address);
+    LOG_DBG << "Dropping the user connection " << session.uniqueid;
+    corebill::report_violation(session.address);
+    return -1;
 }
 
 /**
  * This gets hit every time a client disconnects from the HP public port.
  */
-void user_session_handler::on_close(sock::socket_session<user_outbound_message> *session)
+void user_session_handler::on_close(const comm::comm_session &session) const
 {
-    // Cleanup any resources related to this session.
-
-    // Session is awaiting challenge response.
-    if (session->flags[sock::SESSION_FLAG::USER_CHALLENGE_ISSUED])
-        ctx.pending_challenges.erase(session->uniqueid);
-
     // Session belongs to an authed user.
-    else if (session->flags[sock::SESSION_FLAG::USER_AUTHED])
-        remove_user(session->uniqueid);
-
-    LOG_DBG << "User disconnected " << session->uniqueid;
+    if (session.flags[comm::SESSION_FLAG::USER_AUTHED])
+        remove_user(session.uniqueid);
 }
 
 } // namespace usr

src/usr/user_session_handler.hpp

@@ -2,19 +2,18 @@
 #define _HP_USER_SESSION_HANDLER_
 
 #include "../pchheader.hpp"
-#include "../sock/socket_session_handler.hpp"
-#include "../sock/socket_session.hpp"
-#include "../sock/socket_message.hpp"
+#include "../comm/comm_session_handler.hpp"
+#include "../comm/comm_session.hpp"
 
 namespace usr
 {
 
-class user_session_handler : public sock::socket_session_handler<user_outbound_message>
+class user_session_handler : public comm::comm_session_handler
 {
 public:
-    void on_connect(sock::socket_session<user_outbound_message> *session);
-    void on_message(sock::socket_session<user_outbound_message> *session, std::string_view message);
-    void on_close(sock::socket_session<user_outbound_message> *session);
+    int on_connect(comm::comm_session &session) const;
+    int on_message(comm::comm_session &session, std::string_view message) const;
+    void on_close(const comm::comm_session &session) const;
 };
 
 } // namespace usr

src/usr/usr.cpp

@@ -1,8 +1,7 @@
 #include "../pchheader.hpp"
 #include "../jsonschema/usrmsg_helpers.hpp"
-#include "../sock/socket_server.hpp"
-#include "../sock/socket_session.hpp"
-#include "../sock/socket_session_handler.hpp"
+#include "../comm/comm_server.hpp"
+#include "../comm/comm_session.hpp"
 #include "../util.hpp"
 #include "../conf.hpp"
 #include "../crypto.hpp"
@@ -19,9 +18,6 @@ namespace usr
 // Holds global connected-users and related objects.
 connected_context ctx;
 
-// Holds objects used by socket listener.
-listener_context listener_ctx;
-
 /**
  * Initializes the usr subsystem. Must be called once during application startup.
  * @return 0 for successful initialization. -1 for failure.
@@ -29,20 +25,29 @@ listener_context listener_ctx;
 int init()
 {
     // Start listening for incoming user connections.
-    start_listening();
-    return 0;
+    return start_listening();
 }
 
-std::string issue_challenge(const std::string sessionid)
+/**
+ * Cleanup any running processes.
+ */
+void deinit()
 {
-    std::string msgstr;
-    std::string challengehex;
-    jusrmsg::create_user_challenge(msgstr, challengehex);
+    ctx.listener.stop();
+}
 
-    // Create an entry in pending_challenges for later tracking upon challenge response.
-    ctx.pending_challenges.try_emplace(std::move(sessionid), challengehex);
+/**
+ * Starts listening for incoming user websocket connections.
+ */
+int start_listening()
+{
+    const uint64_t metric_thresholds[] = {conf::cfg.pubmaxcpm, 0, 0, conf::cfg.pubmaxbadmpm};
+    if (ctx.listener.start(
+        conf::cfg.pubport, ".sock-user", comm::SESSION_TYPE::USER, false, metric_thresholds, std::set<conf::ip_port_pair>(), conf::cfg.pubmaxsize) == -1)
+        return -1;
 
-    return msgstr;
+    LOG_INFO << "Started listening for user connections on " << std::to_string(conf::cfg.pubport);
+    return 0;
 }
 
 /**
@@ -51,18 +56,17 @@ std::string issue_challenge(const std::string sessionid)
  * @param session The socket session that received the response.
  * @return 0 for successful verification. -1 for failure.
  */
-int verify_challenge(std::string_view message, sock::socket_session<user_outbound_message> *session)
+int verify_challenge(std::string_view message, comm::comm_session &session)
 {
     // The received message must be the challenge response. We need to verify it.
-    const auto itr = ctx.pending_challenges.find(session->uniqueid);
-    if (itr == ctx.pending_challenges.end())
+    if (session.issued_challenge.empty())
     {
-        LOG_DBG << "No challenge found for the session " << session->uniqueid;
+        LOG_DBG << "No challenge found for the session " << session.uniqueid;
         return -1;
     }
 
     std::string userpubkeyhex;
-    std::string_view original_challenge = itr->second;
+    std::string_view original_challenge = session.issued_challenge;
     if (jusrmsg::verify_user_challenge_response(userpubkeyhex, message, original_challenge) == 0)
     {
         // Challenge singature verification successful.
@@ -82,23 +86,23 @@ int verify_challenge(std::string_view message, sock::socket_session<user_outboun
             // All good. Unique public key.
             // Promote the connection from pending-challenges to authenticated users.
 
-            session->flags.reset(sock::SESSION_FLAG::USER_CHALLENGE_ISSUED); // Clear challenge-issued flag
-            session->flags.set(sock::SESSION_FLAG::USER_AUTHED);             // Set the user-authed flag
-            add_user(session, userpubkey);                                   // Add the user to the global authed user list
-            ctx.pending_challenges.erase(session->uniqueid);                 // Remove the stored challenge
+            session.flags.reset(comm::SESSION_FLAG::USER_CHALLENGE_ISSUED); // Clear challenge-issued flag
+            session.flags.set(comm::SESSION_FLAG::USER_AUTHED);             // Set the user-authed flag
+            add_user(session, userpubkey);                                  // Add the user to the global authed user list
+            session.issued_challenge.clear();                               // Remove the stored challenge
 
-            LOG_DBG << "User connection " << session->uniqueid << " authenticated. Public key "
+            LOG_DBG << "User connection " << session.uniqueid << " authenticated. Public key "
                     << userpubkeyhex;
             return 0;
         }
         else
         {
-            LOG_DBG << "Duplicate user public key " << session->uniqueid;
+            LOG_DBG << "Duplicate user public key " << session.uniqueid;
         }
     }
     else
     {
-        LOG_DBG << "Challenge verification failed " << session->uniqueid;
+        LOG_DBG << "Challenge verification failed " << session.uniqueid;
     }
 
     return -1;
@@ -144,7 +148,7 @@ int handle_user_message(connected_user &user, std::string_view message)
         {
             std::string msg;
             jusrmsg::create_status_response(msg);
-            user.session->send(user_outbound_message(std::move(msg)));
+            user.session.send(msg);
             return 0;
         }
         else
@@ -165,14 +169,11 @@ int handle_user_message(connected_user &user, std::string_view message)
 /**
  * Send the specified status result via the provided session.
  */
-void send_request_status_result(sock::socket_session<user_outbound_message> *session, std::string_view status, std::string_view reason, std::string_view origin_type, std::string_view origin_extra_data)
+void send_request_status_result(const comm::comm_session &session, std::string_view status, std::string_view reason, std::string_view origin_type, std::string_view origin_extra_data)
 {
-    if (session != NULL)
-    {
-        std::string msg;
-        jusrmsg::create_request_status_result(msg, status, reason, origin_type, origin_extra_data);
-        session->send(usr::user_outbound_message(std::move(msg)));
-    }
+    std::string msg;
+    jusrmsg::create_request_status_result(msg, status, reason, origin_type, origin_extra_data);
+    session.send(msg);
 }
 
 /**
@@ -183,9 +184,9 @@ void send_request_status_result(sock::socket_session<user_outbound_message> *ses
  * @param pubkey User's binary public key.
  * @return 0 on successful additions. -1 on failure.
  */
-int add_user(sock::socket_session<user_outbound_message> *session, const std::string &pubkey)
+int add_user(const comm::comm_session &session, const std::string &pubkey)
 {
-    const std::string &sessionid = session->uniqueid;
+    const std::string &sessionid = session.uniqueid;
     if (ctx.users.count(sessionid) == 1)
     {
         LOG_INFO << sessionid << " already exist. Cannot add user.";
@@ -236,41 +237,17 @@ int remove_user(const std::string &sessionid)
  * @param pubkey User binary pubkey.
  * @return Pointer to the socket session. NULL of not found.
  */
-sock::socket_session<usr::user_outbound_message> *get_session_by_pubkey(const std::string &pubkey)
+const comm::comm_session *get_session_by_pubkey(const std::string &pubkey)
 {
-    const auto sessionid_itr = usr::ctx.sessionids.find(pubkey);
-    if (sessionid_itr != usr::ctx.sessionids.end())
+    const auto sessionid_itr = ctx.sessionids.find(pubkey);
+    if (sessionid_itr != ctx.sessionids.end())
     {
-        const auto user_itr = usr::ctx.users.find(sessionid_itr->second);
-        if (user_itr != usr::ctx.users.end())
-            return user_itr->second.session;
+        const auto user_itr = ctx.users.find(sessionid_itr->second);
+        if (user_itr != ctx.users.end())
+            return &user_itr->second.session;
     }
 
     return NULL;
 }
 
-/**
- * Starts listening for incoming user websocket connections.
- */
-void start_listening()
-{
-
-    auto address = net::ip::make_address(conf::cfg.listenip);
-    listener_ctx.default_sess_opts.max_socket_read_len = conf::cfg.pubmaxsize;
-    listener_ctx.default_sess_opts.max_rawbytes_per_minute = conf::cfg.pubmaxcpm;
-    listener_ctx.default_sess_opts.max_badmsgs_per_minute = conf::cfg.pubmaxbadmpm;
-
-    std::make_shared<sock::socket_server<user_outbound_message>>(
-        listener_ctx.ioc,
-        listener_ctx.ssl_ctx,
-        tcp::endpoint{address, conf::cfg.pubport},
-        listener_ctx.global_usr_session_handler,
-        listener_ctx.default_sess_opts)
-        ->run();
-
-    listener_ctx.listener_thread = std::thread([&] { listener_ctx.ioc.run(); });
-
-    LOG_INFO << "Started listening for incoming user connections...";
-}
-
 } // namespace usr

src/usr/usr.hpp

@@ -3,7 +3,8 @@
 
 #include "../pchheader.hpp"
 #include "../util.hpp"
-#include "../sock/socket_session.hpp"
+#include "../comm/comm_server.hpp"
+#include "../comm/comm_session.hpp"
 #include "user_session_handler.hpp"
 #include "user_input.hpp"
 
@@ -27,16 +28,15 @@ struct connected_user
 
     // Holds the websocket session of this user.
     // We don't need to own the session object since the lifetime of user and session are coupled.
-    sock::socket_session<user_outbound_message> *session;
+    const comm::comm_session &session;
 
     /**
      * @param session The web socket session the user is connected to.
      * @param pubkey The public key of the user in binary format.
      */
-    connected_user(sock::socket_session<user_outbound_message> *session, std::string_view pubkey)
-        : pubkey(pubkey)
+    connected_user(const comm::comm_session &session, std::string_view pubkey)
+        : session(session), pubkey(pubkey)
     {
-        this->session = session;
     }
 };
 
@@ -55,50 +55,27 @@ struct connected_context
     // Map key: User binary pubkey
     std::unordered_map<std::string, const std::string> sessionids;
 
-    // Keep track of verification-pending challenges issued to newly connected users.
-    // Map key: User socket session id (<ip:port>)
-    std::unordered_map<std::string, const std::string> pending_challenges;
+    comm::comm_server listener;
 };
 extern connected_context ctx;
 
-/**
- * Struct to hold objects used by socket listener.
- */
-struct listener_context
-{
-    // The SSL context holds certificates to facilitate TLS connections.
-    ssl::context ssl_ctx{ssl::context::tlsv13};
-
-    // User session handler instance. This instance's methods will be fired for any user socket activity.
-    usr::user_session_handler global_usr_session_handler;
-
-    // The IO context used by the websocket listener. (not exposed out of this namespace)
-    net::io_context ioc;
-
-    // The thread the websocket listener is running on. (not exposed out of this namespace)
-    std::thread listener_thread;
-
-    // Used to pass down the default settings to the socket session
-    sock::session_options default_sess_opts;
-};
-
 int init();
 
-std::string issue_challenge(const std::string sessionid);
+void deinit();
 
-int verify_challenge(std::string_view message, sock::socket_session<user_outbound_message> *session);
+int start_listening();
+
+int verify_challenge(std::string_view message, comm::comm_session &session);
 
 int handle_user_message(connected_user &user, std::string_view message);
 
-void send_request_status_result(sock::socket_session<usr::user_outbound_message> *session, std::string_view status, std::string_view reason, std::string_view origin_type, std::string_view origin_extra_data);
+void send_request_status_result(const comm::comm_session &session, std::string_view status, std::string_view reason, std::string_view origin_type, std::string_view origin_extra_data);
 
-int add_user(sock::socket_session<user_outbound_message> *session, const std::string &pubkey);
+int add_user(const comm::comm_session &session, const std::string &pubkey);
 
 int remove_user(const std::string &sessionid);
 
-sock::socket_session<usr::user_outbound_message> *get_session_by_pubkey(const std::string &pubkey);
-
-void start_listening();
+const comm::comm_session *get_session_by_pubkey(const std::string &pubkey);
 
 } // namespace usr
 

src/util.cpp

@@ -188,7 +188,7 @@ std::string_view getsv(const rapidjson::Value &v)
     return std::string_view(v.GetString(), v.GetStringLength());
 }
 
-// provide a safe std::string overload for realpath
+// Provide a safe std::string overload for realpath
 std::string realpath(std::string path)
 {
     std::array<char, PATH_MAX> buffer;
@@ -197,4 +197,13 @@ std::string realpath(std::string path)
     return buffer.data();
 }
 
+// Applies signal mask to the calling thread.
+void mask_signal()
+{
+    sigset_t mask;
+    sigemptyset(&mask);
+    sigaddset(&mask, SIGINT);
+    pthread_sigmask(SIG_BLOCK, &mask, NULL);
+}
+
 } // namespace util

src/util.hpp

@@ -78,6 +78,8 @@ std::string_view getsv(const rapidjson::Value &v);
 
 std::string realpath(std::string path);
 
+void mask_signal();
+
 } // namespace util
 
 #endif

test/bin/readme.txt

@@ -0,0 +1,2 @@
+Files in this directory contains binaries required for Ubuntu/Debian environment.
+This is used for test cluster setup scripts.

test/fusebin/readme.txt

@@ -1,2 +0,0 @@
-Files in this directory contains binaries required for fuse integration for Ubuntu/Debian environment.
-This is used for test cluster setup scripts.

test/local-cluster/Dockerfile

@@ -2,6 +2,10 @@
 # Otherwise, hpcore itself can run on any docker image like ubuntu or debian without NodeJs.
 FROM node:10.17.0-buster-slim
 
+RUN apt-get update
+# Install netcat for websocketd <--> domain socket redirection.
+RUN apt-get install -y netcat-openbsd
+
 # Install fuse.
 # Copy fuse shared library and register it.
 COPY ./bin/libfuse3.so.3 /usr/local/lib/
@@ -12,4 +16,7 @@ COPY ./bin/fusermount3 /usr/local/bin/
 WORKDIR /hp
 COPY ./bin/hpcore .
 COPY ./bin/hpstatemon .
+COPY ./bin/websocketd .
+COPY ./bin/websocat .
+
 ENTRYPOINT ["/hp/hpcore"]

test/local-cluster/cluster-create.sh

@@ -52,7 +52,7 @@ do
             appbillargs: '', \
             peerport: ${peerport}, \
             pubport: ${pubport}, \
-            roundtime: 1000, \
+            roundtime: 2000, \
             loglevel: 'debug', \
             loggers:['console', 'file'] \
             }, null, 2)" > hp.cfg
@@ -111,7 +111,7 @@ do
     popd > /dev/null 2>&1
 done
 
-# Setup initial state data for all nodes but one.
+# Setup initial state data for all nodes.
 for (( i=1; i<=$ncount; i++ ))
 do
 
@@ -137,4 +137,4 @@ docker network create --driver bridge hpnet > /dev/null 2>&1
 echo "Cluster generated at ${clusterloc}"
 echo "Use \"./cluster-start.sh <nodeid>\" to run each node."
 
-exit 0
+exit 0

test/vm-cluster/cluster.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+# Usage example: ./cluster.sh run 1
+
 # VM login password must exist in vmpass.txt
 vmpass=$(cat vmpass.txt)
 # List vm IP addresses of the cluster must exist in iplist.txt
@@ -12,19 +14,19 @@ mode=$1
 hpcore=$(realpath ../..)
 
 if [ "$mode" = "new" ] || [ "$mode" = "update" ] || [ "$mode" = "run" ] || [ "$mode" = "check" ] || \
-   [ "$mode" = "connect" ] || [ "$mode" = "kill" ] || [ "$mode" = "reboot" ] || [ "$mode" = "ssh" ]; then
+   [ "$mode" = "monitor" ] || [ "$mode" = "kill" ] || [ "$mode" = "reboot" ] || [ "$mode" = "ssh" ]; then
     echo "mode: $mode"
 else
-    echo "Invalid command. [ new | update | run <N> | check <N> | connect <N> | kill <N> | reboot <N> | ssh <N> <custom command> ] expected."
+    echo "Invalid command. [ new | update | run <N> | check <N> | monitor <N> | kill <N> | reboot <N> | ssh <N> <custom command> ] expected."
     exit 1
 fi
 
 # Command modes:
-# new - Install hot pocket dependencies and hot pocket to each vm.
-# update - Deploy updated hot pocket binaries into each vm.
+# new - Install hot pocket dependencies and hot pocket with example contracts to each vm.
+# update - Deploy updated hot pocket and example binaries into each vm.
 # run - Run hot pocket of specified vm node.
 # check - Check hot pocket running status of specified vm node.
-# connect - Connect to hot pocket console output (if running) of specified vm node.
+# monitor - Monitor streaming hot pocket console output (if running) of specified vm node.
 # kill - Kill hot pocket (if running) of specified vm node.
 # reboot - Reboot specified vm node.
 # ssh - Open up an ssh terminal for the specified vm node.
@@ -32,44 +34,45 @@ fi
 if [ $mode = "run" ]; then
     let nodeid=$2-1
     vmip=${vmips[$nodeid]}
-    sshpass -p $vmpass ssh geveo@$vmip 'nohup sudo ./hpcore run contract'
-    sshpass -p $vmpass ssh geveo@$vmip 'tail -f nohup.out'
+    sshpass -f vmpass.txt ssh geveo@$vmip 'nohup sudo ./hpcore run contract'
+    sshpass -f vmpass.txt ssh geveo@$vmip 'tail -f nohup.out'
     exit 0
 fi
 
 if [ $mode = "check" ]; then
     let nodeid=$2-1
     vmip=${vmips[$nodeid]}
-    sshpass -p $vmpass ssh geveo@$vmip 'echo hpcore pid:$(pidof hpcore)  hpstatemon pid:$(pidof hpstatemon)'
+    sshpass -f vmpass.txt ssh geveo@$vmip 'echo hpcore pid:$(pidof hpcore)  hpstatemon pid:$(pidof hpstatemon)  websocketd pid:$(pidof websocketd)'
     exit 0
 fi
 
-if [ $mode = "connect" ]; then
+if [ $mode = "monitor" ]; then
     let nodeid=$2-1
     vmip=${vmips[$nodeid]}
-    sshpass -p $vmpass ssh geveo@$vmip 'tail -f nohup.out'
+    sshpass -f vmpass.txt ssh geveo@$vmip 'tail -f nohup.out'
     exit 0
 fi
 
 if [ $mode = "kill" ]; then
     let nodeid=$2-1
     vmip=${vmips[$nodeid]}
-    sshpass -p $vmpass ssh geveo@$vmip 'sudo kill $(pidof hpcore) > /dev/null 2>&1'
-    sshpass -p $vmpass ssh geveo@$vmip 'sudo kill $(pidof hpstatemon) > /dev/null 2>&1'
+    sshpass -f vmpass.txt ssh geveo@$vmip 'sudo kill $(pidof hpcore) > /dev/null 2>&1'
+    sshpass -f vmpass.txt ssh geveo@$vmip 'sudo kill $(pidof hpstatemon) > /dev/null 2>&1'
+    sshpass -f vmpass.txt ssh geveo@$vmip 'sudo kill $(pidof websocketd) > /dev/null 2>&1'
     exit 0
 fi
 
 if [ $mode = "reboot" ]; then
     let nodeid=$2-1
     vmip=${vmips[$nodeid]}
-    sshpass -p $vmpass ssh geveo@$vmip 'sudo reboot'
+    sshpass -f vmpass.txt ssh geveo@$vmip 'sudo reboot'
     exit 0
 fi
 
 if [ $mode = "ssh" ]; then
     let nodeid=$2-1
     vmip=${vmips[$nodeid]}
-    sshpass -p $vmpass ssh geveo@$vmip $3
+    sshpass -f vmpass.txt ssh geveo@$vmip $3
     exit 0
 fi
 
@@ -138,6 +141,6 @@ do
 
     # Copy local cfg file back to remote vm.
     vmip=${vmips[j]}
-    sshpass -p $vmpass scp ./cfg/node$n.cfg geveo@$vmip:~/contract/cfg/hp.cfg
+    sshpass -f vmpass.txt scp ./cfg/node$n.cfg geveo@$vmip:~/contract/cfg/hp.cfg
 done
 rm -r ./cfg

test/vm-cluster/setup-hp.sh

@@ -10,15 +10,14 @@ else
    sudo apt-get install -y nodejs
 fi
 
-# if [ -x "$(command -v fusermount3)" ]; then
-#    echo "FUSE already installed."
-# else
+if [ -x "$(command -v fusermount3)" ]; then
+   echo "FUSE already installed."
+else
    echo "Installing FUSE..."
    sudo cp ./libfuse3.so.3 /usr/local/lib/
    sudo ldconfig
    sudo cp ./fusermount3 /usr/local/bin/
-# fi
-
+fi
 
 sudo rm -r ~/contract > /dev/null 2>&1
 ./hpcore new ./contract
@@ -26,19 +25,3 @@ pushd ./contract/cfg > /dev/null 2>&1
 openssl req -newkey rsa:2048 -new -nodes -x509 -days 3650 -keyout tlskey.pem -out tlscert.pem \
       -subj "/C=AU/ST=ST/L=L/O=O/OU=OU/CN=localhost/emailAddress=hp@example" > /dev/null 2>&1
 popd > /dev/null 2>&1
-
-sudo mkdir -p ./contract/statehist/0
-sudo mkdir -p ./contract/statehist/0/data
-
-FILE=fuse-3.8.0.tar.xz 
-FILE2=fuse-3.8.0
-if [ -f "$FILE" ]; then
-   if [ -f "$FILE2" ]; then
-    sudo cp -r ./fuse-3.8.0 ~/contract/statehist/0/data
-   else
-    sudo tar -xf fuse-3.8.0.tar.xz -C ~/contract/statehist/0/data
-   fi
-else
-sudo wget https://github.com/libfuse/libfuse/releases/download/fuse-3.8.0/fuse-3.8.0.tar.xz -
-sudo tar -xf fuse-3.8.0.tar.xz -C ~/contract/statehist/0/data
-fi

test/vm-cluster/setup-vm.sh

@@ -10,23 +10,23 @@ echo $nodeid. $vmip
 
 if [ $mode = "new" ]; then
 
-    sshpass -p $vmpass scp $hpcore/build/hpcore \
+    sshpass -f vmpass.txt scp $hpcore/build/hpcore \
                             $hpcore/build/hpstatemon \
                             $hpcore/examples/echo_contract/contract.js \
-                            $hpcore/examples/random_contract/rnd_contract \
-                            ../fusebin/libfuse3.so.3 \
-                            ../fusebin/fusermount3 \
+                            ../bin/libfuse3.so.3 \
+                            ../bin/fusermount3 \
+                            ../bin/websocketd \
+                            ../bin/websocat \
                             ./consensus-test-continuous.sh \
                             ./setup-hp.sh \
                             geveo@$vmip:~/
 
-    sshpass -p $vmpass ssh geveo@$vmip 'chmod 700 ~/consensus-test-continuous.sh && chmod 700 ~/setup-hp.sh && ~/setup-hp.sh'
-    sshpass -p $vmpass scp geveo@$vmip:~/contract/cfg/hp.cfg ./cfg/node$nodeid.json
+    sshpass -f vmpass.txt ssh geveo@$vmip 'chmod 700 ~/consensus-test-continuous.sh && chmod 700 ~/setup-hp.sh && ~/setup-hp.sh'
+    sshpass -f vmpass.txt scp geveo@$vmip:~/contract/cfg/hp.cfg ./cfg/node$nodeid.json
 else
-    sshpass -p $vmpass scp $hpcore/build/hpcore \
+    sshpass -f vmpass.txt scp $hpcore/build/hpcore \
                             $hpcore/build/hpstatemon \
                             $hpcore/examples/echo_contract/contract.js \
-                            $hpcore/examples/random_contract/rnd_contract \
                             ./consensus-test-continuous.sh \
                             geveo@$vmip:~/
 fi

test/vm-cluster/vmcreate.sh

@@ -1,15 +1,18 @@
 #!/bin/bash
 
-# Azure vm creation.s
-#az login
-#az account list
-#az account set --subscription ''
+# Usage example: ./vmcreate.sh hp1 aueast
+
+# Azure vm creation script
+# az login
+# az account list
+# az account set --subscription ''
+# az account list-locations
 
 name=$1
 loc=$2
 vmsize=Standard_B1ls
 vmpass=$(cat vmpass.txt)
-resgroup=HotPocket-ResGroup
+resgroup=My-ResGroup
 
 az vm create --name $name --resource-group $resgroup --size $vmsize --admin-username geveo --admin-password $vmpass --image UbuntuLTS --location $loc --generate-ssh-keys
 az vm open-port --resource-group $resgroup --name $name --port 22860 --priority 900 && \