diff --git a/CMakeLists.txt b/CMakeLists.txt index 189683a5..125bcde0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -21,7 +21,7 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY build) link_directories(/usr/local/lib) -# We have 3 executable build outputs: appbill, hpstatemon and hpcore +# We have 2 executable build outputs: appbill and hpcore #-------appbill------- @@ -30,29 +30,6 @@ add_executable(appbill src/bill/appbill.cpp ) - -#-------hpstatemon------- - -add_executable(hpstatemon - src/statefs/state_monitor/fusefs.cpp - src/statefs/state_monitor/state_monitor.cpp - src/statefs/hasher.cpp - src/statefs/state_common.cpp -) -target_link_libraries(hpstatemon - libfuse3.so.3 - libsodium.a - libboost_system.a - libboost_thread.a - libboost_filesystem.a - libboost_stacktrace_backtrace.a - backtrace - pthread - crypto - ${CMAKE_DL_LIBS} # Needed for stacktrace support -) - - #-------hpcore------- add_executable(hpcore @@ -62,12 +39,8 @@ add_executable(hpcore src/hplog.cpp src/proc.cpp src/bill/corebill.cpp - src/statefs/hasher.cpp - src/statefs/state_common.cpp - src/statefs/hashmap_builder.cpp - src/statefs/hashtree_builder.cpp - src/statefs/state_restore.cpp - src/statefs/state_store.cpp + src/hpfs/h32.cpp + src/hpfs/hpfs.cpp src/comm/comm_session.cpp src/comm/comm_server.cpp src/comm/comm_client.cpp @@ -98,27 +71,23 @@ target_link_libraries(hpcore ${CMAKE_DL_LIBS} # Needed for stacktrace support ) add_dependencies(hpcore - hpstatemon appbill ) add_custom_command(TARGET hpcore POST_BUILD # COMMAND strip ./build/hpcore - # COMMAND strip ./build/hpstatemon # COMMAND strip ./build/appbill - COMMAND cp ./test/bin/websocketd ./build/websocketd - COMMAND cp ./test/bin/websocat ./build/websocat + COMMAND cp ./test/bin/websocketd ./test/bin/websocat ./test/bin/hpfs ./build/ ) -target_precompile_headers(hpstatemon PUBLIC src/pchheader.hpp) -target_precompile_headers(hpcore REUSE_FROM hpstatemon) +target_precompile_headers(hpcore PUBLIC src/pchheader.hpp) # Create docker image from hpcore build output with 'make docker' # Requires docker to be runnable without 'sudo' add_custom_target(docker COMMAND mkdir -p ./test/local-cluster/bin - COMMAND cp ./build/hpcore ./build/hpstatemon ./build/appbill ./test/local-cluster/bin/ - COMMAND cp ./test/bin/fusermount3 ./test/bin/libfuse3.so.3 ./test/bin/websocketd ./test/bin/websocat ./test/local-cluster/bin/ + COMMAND cp ./build/hpcore ./build/appbill ./test/local-cluster/bin/ + COMMAND cp ./test/bin/fusermount3 ./test/bin/libfuse3.so.3 ./test/bin/libb2.so.1 ./test/bin/websocketd ./test/bin/websocat ./test/bin/hpfs ./test/local-cluster/bin/ COMMAND docker build -t hpcore:latest ./test/local-cluster ) set_target_properties(docker PROPERTIES EXCLUDE_FROM_ALL TRUE) diff --git a/README.md b/README.md index 033fe6a8..833058be 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ A C++ version of hotpocket designed for production envrionments, original prototype here: https://github.com/codetsunami/hotpocket -[Hot Pocket Wiki](https://github.com/HotPocketDev/core/wiki/Hot-Pocket-Wiki) +[Hot Pocket Wiki](https://github.com/HotPocketDev/core/wiki) ## Libraries * Crypto - Libsodium https://github.com/jedisct1/libsodium @@ -81,7 +81,7 @@ This will update your linker library cache and avoid potential issues when runni 1. Run `make` (Hot Pocket binary will be created as `./build/hpcore`) 1. Refer to [Running Hot Pocket](https://github.com/HotPocketDev/core/wiki/Running-Hot-Pocket) in the Wiki. -Refer to [Hot Pocket Wiki](https://github.com/HotPocketDev/core/wiki/Hot-Pocket-Wiki) for more info. +Refer to [Hot Pocket Wiki](https://github.com/HotPocketDev/core/wiki) for more info. ## Code structure Code is divided into subsystems via namespaces. @@ -102,4 +102,4 @@ Code is divided into subsystems via namespaces. **util::** Contains shared data structures/helper functions used by multiple subsystems. -**statefs::** Fuse-based state filesystem monitoring and contract state maintenance subsystem. \ No newline at end of file +**hpfs::** [hpfs](https://github.com/HotPocketDev/hpfs) state management client helpers. \ No newline at end of file diff --git a/examples/echo_contract/contract.js b/examples/echo_contract/contract.js index a2154485..65c2dfea 100644 --- a/examples/echo_contract/contract.js +++ b/examples/echo_contract/contract.js @@ -9,7 +9,7 @@ const fs = require('fs') let hpargs = JSON.parse(fs.readFileSync(0, 'utf8')); // We just save execution args as an example state file change. -fs.appendFileSync("state/exects.txt", "ts:" + hpargs.ts + "\n"); +fs.appendFileSync("exects.txt", "ts:" + hpargs.ts + "\n"); Object.keys(hpargs.usrfd).forEach(function (key, index) { let userfds = hpargs.usrfd[key]; @@ -17,7 +17,7 @@ Object.keys(hpargs.usrfd).forEach(function (key, index) { if (userinput.length > 0) { // Append user input to a state file. - fs.appendFileSync("state/userinputs.txt", userinput + "\n"); + fs.appendFileSync("userinputs.txt", userinput + "\n"); fs.writeSync(userfds[1], "Echoing: " + userinput); } }); diff --git a/examples/file_contract/contract.js b/examples/file_contract/contract.js new file mode 100644 index 00000000..3c212817 --- /dev/null +++ b/examples/file_contract/contract.js @@ -0,0 +1,26 @@ +process.on('uncaughtException', (err) => { + console.error('There was an uncaught error', err) +}) +const fs = require('fs') + +//console.log("===File contract started==="); +//console.log("Contract args received from hp: " + input); + +let hpargs = JSON.parse(fs.readFileSync(0, 'utf8')); + +// We just save execution args as an example state file change. +fs.appendFileSync("exects.txt", "ts:" + hpargs.ts + "\n"); + +Object.keys(hpargs.usrfd).forEach(function (key, index) { + let userfds = hpargs.usrfd[key]; + let fileContent = fs.readFileSync(userfds[0]); + + if (fileContent.length > 0) { + // Save the content into a new file. + var fileName = new Date().getTime().toString(); + fs.writeFileSync(fileName, fileContent); + fs.writeSync(userfds[1], "Saved file (len: " + fileContent.length / 1024 + " KB)"); + } +}); + +//console.log("===File contract ended==="); \ No newline at end of file diff --git a/examples/hpclient/file-client.js b/examples/hpclient/file-client.js new file mode 100644 index 00000000..7bf12c64 --- /dev/null +++ b/examples/hpclient/file-client.js @@ -0,0 +1,153 @@ +const fs = require('fs') +const ws_api = require('ws'); +const sodium = require('libsodium-wrappers') +const readline = require('readline') + +// sodium has a trigger when it's ready, we will wait and execute from there +sodium.ready.then(main).catch((e) => { console.log(e) }) + + +function main() { + + var keys = sodium.crypto_sign_keypair() + + + // check for client keys + if (!fs.existsSync('.hp_client_keys')) { + keys.privateKey = sodium.to_hex(keys.privateKey) + keys.publicKey = sodium.to_hex(keys.publicKey) + fs.writeFileSync('.hp_client_keys', JSON.stringify(keys)) + } else { + keys = JSON.parse(fs.readFileSync('.hp_client_keys')) + keys.privateKey = Uint8Array.from(Buffer.from(keys.privateKey, 'hex')) + keys.publicKey = Uint8Array.from(Buffer.from(keys.publicKey, 'hex')) + } + + + var server = 'wss://localhost:8080' + + if (process.argv.length == 3) server = 'wss://localhost:' + process.argv[2] + + if (process.argv.length == 4) server = 'wss://' + process.argv[2] + ':' + process.argv[3] + + var ws = new ws_api(server, { + rejectUnauthorized: false + }) + + /* anatomy of a public challenge + { + version: '0.1', + type: 'public_challenge', + challenge: '' + } + */ + + + // if the console ctrl + c's us we should close ws gracefully + process.once('SIGINT', function (code) { + console.log('SIGINT received...'); + ws.close() + }); + + function create_input_container(inp) { + + let hexInp = inp.toString('hex'); + console.log("hex " + hexInp.length); + + let inp_container = { + nonce: (new Date()).getTime().toString(), + input: hexInp, + max_ledger_seqno: 9999999 + } + let inp_container_bytes = JSON.stringify(inp_container); + let sig_bytes = sodium.crypto_sign_detached(inp_container_bytes, keys.privateKey); + + let signed_inp_container = { + type: "contract_input", + content: inp_container_bytes.toString('hex'), + sig: Buffer.from(sig_bytes).toString('hex') + } + + return JSON.stringify(signed_inp_container); + } + + function create_status_request() { + let statreq = { type: 'stat' } + return JSON.stringify(statreq); + } + + function handle_public_challange(m) { + let pkhex = 'ed' + Buffer.from(keys.publicKey).toString('hex'); + console.log('My public key is: ' + pkhex); + + // sign the challenge and send back the response + var sigbytes = sodium.crypto_sign_detached(m.challenge, keys.privateKey); + var response = { + type: 'challenge_resp', + challenge: m.challenge, + sig: Buffer.from(sigbytes).toString('hex'), + pubkey: pkhex + } + + ws.send(JSON.stringify(response)) + + // start listening for stdin + const rl = readline.createInterface({ + input: process.stdin, + output: process.stdout + }); + + console.log("Ready to accept inputs.") + + // Capture user input from the console. + var input_pump = () => { + rl.question('', (inp) => { + + let msgtosend = ""; + + if (inp == "stat") + msgtosend = create_status_request(); + else { + var fileContent = fs.readFileSync(inp); + msgtosend = create_input_container(fileContent); + + console.log("Sending file (len: " + fileContent.length / 1024 + " KB)"); + } + + ws.send(msgtosend) + + input_pump() + }) + } + input_pump(); + } + + ws.on('message', (data) => { + + try { + m = JSON.parse(data) + } catch (e) { + console.log("Exception: " + data); + return + } + + if (m.type == 'public_challenge') { + handle_public_challange(m); + } + else if (m.type == 'contract_output') { + console.log("Contract says: " + Buffer.from(m.content, 'hex').toString()); + } + else if (m.type == 'request_status_result') { + if (m.status != "accepted") + console.log("Input status: " + m.status); + } + else { + console.log(m); + } + + }); + + ws.on('close', () => { + console.log('Server disconnected.'); + }); +} diff --git a/src/comm/comm_client.cpp b/src/comm/comm_client.cpp index 78227e5b..658495d8 100644 --- a/src/comm/comm_client.cpp +++ b/src/comm/comm_client.cpp @@ -7,90 +7,89 @@ namespace comm { -int comm_client::start(std::string_view host, const uint16_t port, const uint64_t (&metric_thresholds)[4], const uint64_t max_msg_size) -{ - return start_websocat_process(host, port); -} - -void comm_client::stop() -{ - if (read_fd > 0) - close(read_fd); - if (write_fd > 0) - close(write_fd); - - if (websocat_pid > 0) - kill(websocat_pid, SIGINT); // Kill websocat. -} - -int comm_client::start_websocat_process(std::string_view host, const uint16_t port) -{ - // setup pipe I/O - if (pipe(read_pipe) < 0 || pipe(write_pipe) < 0) + int comm_client::start(std::string_view host, const uint16_t port, const uint64_t (&metric_thresholds)[4], const uint64_t max_msg_size) { - LOG_ERR << errno << ": websocat pipe creation failed."; - return -1; + return start_websocat_process(host, port); } - const pid_t pid = fork(); - - if (pid > 0) + void comm_client::stop() { - // HotPocket process. - websocat_pid = pid; - - read_fd = read_pipe[0]; - write_fd = write_pipe[1]; - - // Close unused fds by us. - close(write_pipe[0]); - close(read_pipe[1]); - - // Wait for some time and check if websocat is still running properly. - util::sleep(20); - int pid_status; - waitpid(websocat_pid, &pid_status, WNOHANG); - if (WIFEXITED(pid_status)) // This means websocat has exited. - { + if (read_fd > 0) close(read_fd); + if (write_fd > 0) close(write_fd); + + if (websocat_pid > 0) + util::kill_process(websocat_pid, false); // Kill websocat. + } + + int comm_client::start_websocat_process(std::string_view host, const uint16_t port) + { + // setup pipe I/O + if (pipe(read_pipe) < 0 || pipe(write_pipe) < 0) + { + LOG_ERR << errno << ": websocat pipe creation failed."; return -1; } + + const pid_t pid = fork(); + + if (pid > 0) + { + // HotPocket process. + + read_fd = read_pipe[0]; + write_fd = write_pipe[1]; + + // Close unused fds by us. + close(write_pipe[0]); + close(read_pipe[1]); + + // Wait for some time and check if websocat is still running properly. + util::sleep(20); + if (kill(pid, 0) == -1) + { + close(read_fd); + close(write_fd); + return -1; + } + + websocat_pid = pid; + } + else if (pid == 0) + { + // Websocat process. + close(write_pipe[1]); //parent write + close(read_pipe[0]); //parent read + + dup2(write_pipe[0], STDIN_FILENO); //child read + close(write_pipe[0]); + dup2(read_pipe[1], STDOUT_FILENO); //child write + close(read_pipe[1]); + + std::string url = std::string("wss://").append(host).append(":").append(std::to_string(port)); + + // Fill process args. + char *execv_args[] = { + conf::ctx.websocat_exe_path.data(), + url.data(), + (char *)"-k", // Accept invalid certificates + (char *)"-b", // Binary mode + (char *)"-E", // Close on EOF + (char *)"-q", // Quiet mode + NULL}; + + const int ret = execv(execv_args[0], execv_args); + LOG_ERR << errno << ": websocat process execv failed."; + exit(1); + } + else + { + LOG_ERR << "fork() failed when starting websocat process."; + return -1; + } + + return 0; } - else if (pid == 0) - { - // Websocat process. - close(write_pipe[1]); //parent write - close(read_pipe[0]); //parent read - - dup2(write_pipe[0], STDIN_FILENO); //child read - close(write_pipe[0]); - dup2(read_pipe[1], STDOUT_FILENO); //child write - close(read_pipe[1]); - - std::string url = std::string("wss://").append(host).append(":").append(std::to_string(port)); - - // Fill process args. - char *execv_args[] = { - conf::ctx.websocat_exe_path.data(), - url.data(), - (char *)"-k", // Accept invalid certificates - (char *)"-b", // Binary mode - (char *)"-E", // Close on EOF - (char *)"-q", // Quiet mode - NULL}; - - const int ret = execv(execv_args[0], execv_args); - LOG_ERR << errno << ": websocat process execv failed."; - exit(1); - } - else - { - LOG_ERR << "fork() failed when starting websocat process."; - return -1; - } - - return 0; -} } // namespace comm diff --git a/src/comm/comm_server.cpp b/src/comm/comm_server.cpp index d1f64aa7..5cf35a9d 100644 --- a/src/comm/comm_server.cpp +++ b/src/comm/comm_server.cpp @@ -12,391 +12,397 @@ namespace comm { -int comm_server::start( - const uint16_t port, const char *domain_socket_name, const SESSION_TYPE session_type, const bool is_binary, const bool use_size_header, - const uint64_t (&metric_thresholds)[4], const std::set &req_known_remotes, const uint64_t max_msg_size) -{ - int accept_fd = open_domain_socket(domain_socket_name); - if (accept_fd > 0) + int comm_server::start( + const uint16_t port, const char *domain_socket_name, const SESSION_TYPE session_type, const bool is_binary, const bool use_size_header, + const uint64_t (&metric_thresholds)[4], const std::set &req_known_remotes, const uint64_t max_msg_size) { - watchdog_thread = std::thread( - &comm_server::connection_watchdog, this, accept_fd, session_type, is_binary, - std::ref(metric_thresholds), req_known_remotes, max_msg_size); - return start_websocketd_process(port, domain_socket_name, is_binary, use_size_header); - } - - return -1; -} - -int comm_server::open_domain_socket(const char *domain_socket_name) -{ - int fd = socket(AF_UNIX, SOCK_STREAM, 0); - if (fd == -1) - { - LOG_ERR << errno << ": Domain socket open error"; - return -1; - } - - sockaddr_un addr; - memset(&addr, 0, sizeof(addr)); - addr.sun_family = AF_UNIX; - - strncpy(addr.sun_path, domain_socket_name, sizeof(addr.sun_path) - 1); - unlink(domain_socket_name); - - if (bind(fd, (struct sockaddr *)&addr, sizeof(addr)) == -1) - { - LOG_ERR << errno << ": Domain socket bind error"; - return -1; - } - - if (listen(fd, 5) == -1) - { - LOG_ERR << errno << ": Domain socket listen error"; - return -1; - } - - // Set non-blocking behaviour. - // We do this so the accept() call returns immediately without blocking the listening thread. - int flags = fcntl(fd, F_GETFL); - fcntl(fd, F_SETFL, flags | O_NONBLOCK); - - return fd; // This is the fd we should call accept() on. -} - -void comm_server::connection_watchdog( - const int accept_fd, const SESSION_TYPE session_type, const bool is_binary, - const uint64_t (&metric_thresholds)[4], const std::set &req_known_remotes, const uint64_t max_msg_size) -{ - util::mask_signal(); - - // Map with read fd to connected session mappings. - std::unordered_map sessions; - // Map with read fd to connected comm client mappings. - std::unordered_map outbound_clients; - - // Counter to track when to initiate outbound client connections. - int16_t loop_counter = -1; - - while (true) - { - if (should_stop_listening) - break; - - // Prepare poll fd list. - const size_t fd_count = sessions.size() + 1; //+1 for the inclusion of accept_fd - pollfd pollfds[fd_count]; - if (poll_fds(pollfds, accept_fd, sessions) == -1) + int accept_fd = open_domain_socket(domain_socket_name); + if (accept_fd > 0) { - util::sleep(10); - continue; + watchdog_thread = std::thread( + &comm_server::connection_watchdog, this, accept_fd, session_type, is_binary, + std::ref(metric_thresholds), req_known_remotes, max_msg_size); + return start_websocketd_process(port, domain_socket_name, is_binary, use_size_header); } - util::sleep(10); + return -1; + } - // Accept any new incoming connection if available. - check_for_new_connection(sessions, accept_fd, session_type, is_binary, metric_thresholds); - - if (!req_known_remotes.empty()) + int comm_server::open_domain_socket(const char *domain_socket_name) + { + int fd = socket(AF_UNIX, SOCK_STREAM, 0); + if (fd == -1) { - // Restore any missing outbound connections every 500 iterations (including the first iteration). - if (loop_counter == -1 || loop_counter == 500) + LOG_ERR << errno << ": Domain socket open error"; + return -1; + } + + sockaddr_un addr; + memset(&addr, 0, sizeof(addr)); + addr.sun_family = AF_UNIX; + + strncpy(addr.sun_path, domain_socket_name, sizeof(addr.sun_path) - 1); + unlink(domain_socket_name); + + if (bind(fd, (struct sockaddr *)&addr, sizeof(addr)) == -1) + { + LOG_ERR << errno << ": Domain socket bind error"; + return -1; + } + + if (listen(fd, 5) == -1) + { + LOG_ERR << errno << ": Domain socket listen error"; + return -1; + } + + // Set non-blocking behaviour. + // We do this so the accept() call returns immediately without blocking the listening thread. + int flags = fcntl(fd, F_GETFL); + fcntl(fd, F_SETFL, flags | O_NONBLOCK); + + return fd; // This is the fd we should call accept() on. + } + + void comm_server::connection_watchdog( + const int accept_fd, const SESSION_TYPE session_type, const bool is_binary, + const uint64_t (&metric_thresholds)[4], const std::set &req_known_remotes, const uint64_t max_msg_size) + { + util::mask_signal(); + + // Map with read fd to connected session mappings. + std::unordered_map sessions; + // Map with read fd to connected comm client mappings. + std::unordered_map outbound_clients; + + // Counter to track when to initiate outbound client connections. + int16_t loop_counter = -1; + + while (true) + { + if (should_stop_listening) + break; + + // Prepare poll fd list. + const size_t fd_count = sessions.size() + 1; //+1 for the inclusion of accept_fd + pollfd pollfds[fd_count]; + if (poll_fds(pollfds, accept_fd, sessions) == -1) { - loop_counter = 0; - maintain_known_connections(sessions, outbound_clients, req_known_remotes, session_type, is_binary, max_msg_size, metric_thresholds); + util::sleep(10); + continue; } - loop_counter++; - } - const size_t sessions_count = sessions.size(); + util::sleep(10); - // Loop through all fds and read any data. - for (size_t i = 1; i <= sessions_count; i++) - { - const short result = pollfds[i].revents; - const int fd = pollfds[i].fd; + // Accept any new incoming connection if available. + check_for_new_connection(sessions, accept_fd, session_type, is_binary, metric_thresholds); - const auto iter = sessions.find(fd); - if (iter != sessions.end()) + if (!req_known_remotes.empty()) { - comm_session &session = iter->second; - bool should_disconnect = (session.state == SESSION_STATE::CLOSED); - - if (!should_disconnect) + // Restore any missing outbound connections every 500 iterations (including the first iteration). + if (loop_counter == -1 || loop_counter == 500) { - if (result & POLLIN) - should_disconnect = (session.attempt_read(max_msg_size) == -1); - - if (result & (POLLERR | POLLHUP | POLLRDHUP | POLLNVAL)) - should_disconnect = true; + loop_counter = 0; + maintain_known_connections(sessions, outbound_clients, req_known_remotes, session_type, is_binary, max_msg_size, metric_thresholds); } + loop_counter++; + } - if (should_disconnect) + const size_t sessions_count = sessions.size(); + + // Loop through all fds and read any data. + for (size_t i = 1; i <= sessions_count; i++) + { + const short result = pollfds[i].revents; + const int fd = pollfds[i].fd; + + const auto iter = sessions.find(fd); + if (iter != sessions.end()) { - // If this is an outbound session, cleanup the corresponding comm client as well. - if (!session.is_inbound) + comm_session &session = iter->second; + bool should_disconnect = (session.state == SESSION_STATE::CLOSED); + + if (!should_disconnect) { - const auto client_itr = outbound_clients.find(fd); - client_itr->second.stop(); - outbound_clients.erase(client_itr); + if (result & POLLIN) + should_disconnect = (session.attempt_read(max_msg_size) == -1); + + if (result & (POLLERR | POLLHUP | POLLRDHUP | POLLNVAL)) + should_disconnect = true; } - session.close(); - sessions.erase(fd); + if (should_disconnect) + { + // If this is an outbound session, cleanup the corresponding comm client as well. + if (!session.is_inbound) + { + const auto client_itr = outbound_clients.find(fd); + client_itr->second.stop(); + outbound_clients.erase(client_itr); + } + + session.close(); + sessions.erase(fd); + } + } + } + } + + // If we reach this point that means we are shutting down. + + // Close all sessions and clients + for (auto &[fd, session] : sessions) + session.close(false); + for (auto &[fd, client] : outbound_clients) + client.stop(); + + LOG_INFO << (session_type == SESSION_TYPE::USER ? "User" : "Peer") << " listener stopped."; + } + + int comm_server::poll_fds(pollfd *pollfds, const int accept_fd, const std::unordered_map &sessions) + { + const short poll_events = POLLIN | POLLRDHUP; + pollfds[0].fd = accept_fd; + + auto iter = sessions.begin(); + for (size_t i = 1; i <= sessions.size(); i++) + { + pollfds[i].fd = iter->first; + pollfds[i].events = poll_events; + iter++; + } + + if (poll(pollfds, sessions.size() + 1, 10) == -1) //10ms timeout + { + LOG_ERR << errno << ": Poll failed."; + return -1; + } + + return 0; + } + + void comm_server::check_for_new_connection( + std::unordered_map &sessions, const int accept_fd, + const SESSION_TYPE session_type, const bool is_binary, const uint64_t (&metric_thresholds)[4]) + { + // Accept new client connection (if available) + int client_fd = accept(accept_fd, NULL, NULL); + if (client_fd == -1 && errno != EAGAIN) + { + LOG_ERR << errno << ": Domain socket accept error"; + } + else if (client_fd > 0) + { + // New client connected. + const std::string ip = get_cgi_ip(client_fd); + + if (corebill::is_banned(ip)) + { + LOG_DBG << "Dropping connection for banned host " << ip; + close(client_fd); + } + else + { + comm_session session(ip, client_fd, client_fd, session_type, is_binary, true, metric_thresholds); + if (session.on_connect() == 0) + sessions.try_emplace(client_fd, std::move(session)); + } + } + } + + void comm_server::maintain_known_connections( + std::unordered_map &sessions, std::unordered_map &outbound_clients, + const std::set &req_known_remotes, const SESSION_TYPE session_type, const bool is_binary, + const uint64_t max_msg_size, const uint64_t (&metric_thresholds)[4]) + { + // Find already connected known remote parties list + std::set known_remotes; + for (const auto &[fd, session] : sessions) + { + if (session.state != SESSION_STATE::CLOSED && !session.known_ipport.first.empty()) + known_remotes.emplace(session.known_ipport); + } + + for (const auto &ipport : req_known_remotes) + { + if (should_stop_listening) + break; + + // Check if we are already connected to this remote party. + if (known_remotes.find(ipport) != known_remotes.end()) + continue; + + std::string_view host = ipport.first; + const uint16_t port = ipport.second; + LOG_DBG << "Trying to connect " << host << ":" << std::to_string(port); + + comm::comm_client client; + if (client.start(host, port, metric_thresholds, conf::cfg.peermaxsize) == -1) + { + LOG_ERR << "Outbound connection attempt failed"; + } + else + { + comm::comm_session session(host, client.read_fd, client.write_fd, comm::SESSION_TYPE::PEER, is_binary, false, metric_thresholds); + session.known_ipport = ipport; + if (session.on_connect() == 0) + { + sessions.try_emplace(client.read_fd, std::move(session)); + outbound_clients.emplace(client.read_fd, std::move(client)); + known_remotes.emplace(ipport); } } } } - // If we reach this point that means we are shutting down. - - // Close all sessions and clients - for (auto &[fd, session] : sessions) - session.close(false); - for (auto &[fd, client] : outbound_clients) - client.stop(); - - LOG_INFO << (session_type == SESSION_TYPE::USER ? "User" : "Peer") << " listener stopped."; -} - -int comm_server::poll_fds(pollfd *pollfds, const int accept_fd, const std::unordered_map &sessions) -{ - const short poll_events = POLLIN | POLLRDHUP; - pollfds[0].fd = accept_fd; - - auto iter = sessions.begin(); - for (size_t i = 1; i <= sessions.size(); i++) + int comm_server::start_websocketd_process(const uint16_t port, const char *domain_socket_name, const bool is_binary, const bool use_size_header) { - pollfds[i].fd = iter->first; - pollfds[i].events = poll_events; - iter++; - } + // setup pipe for firewall + int firewall_pipe[2]; // parent to child pipe - if (poll(pollfds, sessions.size() + 1, 10) == -1) //10ms timeout - { - LOG_ERR << errno << ": Poll failed."; - return -1; - } - - return 0; -} - -void comm_server::check_for_new_connection( - std::unordered_map &sessions, const int accept_fd, - const SESSION_TYPE session_type, const bool is_binary, const uint64_t (&metric_thresholds)[4]) -{ - // Accept new client connection (if available) - int client_fd = accept(accept_fd, NULL, NULL); - if (client_fd == -1 && errno != EAGAIN) - { - LOG_ERR << errno << ": Domain socket accept error"; - } - else if (client_fd > 0) - { - // New client connected. - const std::string ip = get_cgi_ip(client_fd); - - if (corebill::is_banned(ip)) + if (pipe(firewall_pipe)) { - LOG_DBG << "Dropping connection for banned host " << ip; - close(client_fd); + LOG_ERR << errno << ": pipe() call failed for firewall"; } else { - comm_session session(ip, client_fd, client_fd, session_type, is_binary, true, metric_thresholds); - if (session.on_connect() == 0) - sessions.try_emplace(client_fd, std::move(session)); + firewall_out = firewall_pipe[1]; } - } -} -void comm_server::maintain_known_connections( - std::unordered_map &sessions, std::unordered_map &outbound_clients, - const std::set &req_known_remotes, const SESSION_TYPE session_type, const bool is_binary, - const uint64_t max_msg_size, const uint64_t (&metric_thresholds)[4]) -{ - // Find already connected known remote parties list - std::set known_remotes; - for (const auto &[fd, session] : sessions) - { - if (session.state != SESSION_STATE::CLOSED && !session.known_ipport.first.empty()) - known_remotes.emplace(session.known_ipport); - } + const pid_t pid = fork(); - for (const auto &ipport : req_known_remotes) - { - if (should_stop_listening) - break; - - // Check if we are already connected to this remote party. - if (known_remotes.find(ipport) != known_remotes.end()) - continue; - - std::string_view host = ipport.first; - const uint16_t port = ipport.second; - LOG_DBG << "Trying to connect " << host << ":" << std::to_string(port); - - comm::comm_client client; - if (client.start(host, port, metric_thresholds, conf::cfg.peermaxsize) == -1) + if (pid > 0) { - LOG_ERR << "Outbound connection attempt failed"; + // HotPocket process. + + // Close the child reading end of the pipe in the parent + if (firewall_out > 0) + close(firewall_pipe[0]); + + // Wait for some time and check if websocketd is still running properly. + util::sleep(20); + if (kill(pid, 0) == -1) + return -1; + + websocketd_pid = pid; } - else + else if (pid == 0) { - comm::comm_session session(host, client.read_fd, client.write_fd, comm::SESSION_TYPE::PEER, is_binary, false, metric_thresholds); - session.known_ipport = ipport; - if (session.on_connect() == 0) + // Websocketd process. + // We are using websocketd forked repo: https://github.com/codetsunami/websocketd + + if (firewall_out > 0) { - sessions.try_emplace(client.read_fd, std::move(session)); - outbound_clients.emplace(client.read_fd, std::move(client)); - known_remotes.emplace(ipport); + // Close parent writing end of the pipe in the child + close(firewall_pipe[1]); + // Override stdin in the child's file table + dup2(firewall_pipe[0], 0); } + + // Override stdout in the child's file table with /dev/null + // int null_fd = open("/dev/null", O_WRONLY); + // if (null_fd) + // dup2(null_fd, 1); + + // Fill process args. + char *execv_args[] = { + conf::ctx.websocketd_exe_path.data(), + (char *)"--port", + std::to_string(port).data(), + (char *)"--ssl", + (char *)"--sslcert", + conf::ctx.tls_cert_file.data(), + (char *)"--sslkey", + conf::ctx.tls_key_file.data(), + (char *)(is_binary ? "--binary=true" : "--binary=false"), + (char *)(use_size_header ? "--sizeheader=true" : "--sizeheader=false"), + (char *)"--loglevel=error", + (char *)"nc", // netcat (OpenBSD) is used for domain socket redirection. + (char *)"-U", // Use UNIX domain socket + (char *)domain_socket_name, + NULL}; + + const int ret = execv(execv_args[0], execv_args); + LOG_ERR << errno << ": websocketd process execv failed."; + exit(1); } - } -} - -int comm_server::start_websocketd_process(const uint16_t port, const char *domain_socket_name, const bool is_binary, const bool use_size_header) -{ - // setup pipe for firewall - int firewall_pipe[2]; // parent to child pipe - - if (pipe(firewall_pipe)) - { - LOG_ERR << errno << ": pipe() call failed for firewall"; - } - else - { - firewall_out = firewall_pipe[1]; - } - - const pid_t pid = fork(); - - if (pid > 0) - { - // HotPocket process. - websocketd_pid = pid; - - // Close the child reading end of the pipe in the parent - if (firewall_out > 0) - close(firewall_pipe[0]); - } - else if (pid == 0) - { - // Websocketd process. - // We are using websocketd forked repo: https://github.com/codetsunami/websocketd - - if (firewall_out > 0) + else { - // Close parent writing end of the pipe in the child - close(firewall_pipe[1]); - // Override stdin in the child's file table - dup2(firewall_pipe[0], 0); + LOG_ERR << "fork() failed when starting websocketd process."; + return -1; } - // Override stdout in the child's file table with /dev/null - // int null_fd = open("/dev/null", O_WRONLY); - // if (null_fd) - // dup2(null_fd, 1); - - // Fill process args. - char *execv_args[] = { - conf::ctx.websocketd_exe_path.data(), - (char *)"--port", - std::to_string(port).data(), - (char *)"--ssl", - (char *)"--sslcert", - conf::ctx.tls_cert_file.data(), - (char *)"--sslkey", - conf::ctx.tls_key_file.data(), - (char *)(is_binary ? "--binary=true" : "--binary=false"), - (char *)(use_size_header? "--sizeheader=true" : "--sizeheader=false"), - (char *)"--loglevel=error", - (char *)"nc", // netcat (OpenBSD) is used for domain socket redirection. - (char *)"-U", // Use UNIX domain socket - (char *)domain_socket_name, - NULL}; - - const int ret = execv(execv_args[0], execv_args); - LOG_ERR << errno << ": websocketd process execv failed."; - exit(1); + return 0; } - else + + void comm_server::firewall_ban(std::string_view ip, const bool unban) { - LOG_ERR << "fork() failed when starting websocketd process."; - return -1; + if (firewall_out < 0) + return; + + iovec iov[]{ + {(void *)(unban ? "r" : "a"), 1}, + {(void *)ip.data(), ip.length()}}; + writev(firewall_out, iov, 2); } - return 0; -} - -void comm_server::firewall_ban(std::string_view ip, const bool unban) -{ - if (firewall_out < 0) - return; - - iovec iov[]{ - {(void *)(unban ? "r" : "a"), 1}, - {(void *)ip.data(), ip.length()}}; - writev(firewall_out, iov, 2); -} - -/** + /** * If the fd supplied was produced by accept()ing unix domain socket connection * the process at the other end is inspected for CGI environment variables * and the REMOTE_ADDR variable is returned as std::string, otherwise empty string */ -std::string comm_server::get_cgi_ip(const int fd) -{ - socklen_t length; - ucred uc; - length = sizeof(struct ucred); - - // Ask the operating system for information about the other process - if (getsockopt(fd, SOL_SOCKET, SO_PEERCRED, &uc, &length) == -1) + std::string comm_server::get_cgi_ip(const int fd) { - LOG_ERR << errno << ": Could not retrieve PID from unix domain socket"; - return ""; - } + socklen_t length; + ucred uc; + length = sizeof(struct ucred); - // Open /proc//environ for that process - std::stringstream ss; - ss << "/proc/" << uc.pid << "/environ"; - std::string fn = ss.str(); - - const int envfd = open(fn.c_str(), O_RDONLY); - if (!envfd) - { - LOG_ERR << errno << ": Could not open environ block for process on other end of unix domain socket PID=" << uc.pid; - return ""; - } - - // Read environ block - char envblock[0x7fff]; - const ssize_t bytes_read = read(envfd, envblock, 0x7fff); //0x7fff bytes is an operating system size limit for this block - close(envfd); - - // Find the REMOTE_ADDR entry. Envrion block delimited by \0 - for (char *upto = envblock, *last = envblock; upto - envblock < bytes_read; ++upto) - { - if (*upto == '\0') + // Ask the operating system for information about the other process + if (getsockopt(fd, SOL_SOCKET, SO_PEERCRED, &uc, &length) == -1) { - if (upto - last > 12 && strncmp(last, "REMOTE_ADDR=", 12) == 0) - return std::string((const char *)(last + 12)); - last = upto + 1; + LOG_ERR << errno << ": Could not retrieve PID from unix domain socket"; + return ""; } + + // Open /proc//environ for that process + std::stringstream ss; + ss << "/proc/" << uc.pid << "/environ"; + std::string fn = ss.str(); + + const int envfd = open(fn.c_str(), O_RDONLY); + if (!envfd) + { + LOG_ERR << errno << ": Could not open environ block for process on other end of unix domain socket PID=" << uc.pid; + return ""; + } + + // Read environ block + char envblock[0x7fff]; + const ssize_t bytes_read = read(envfd, envblock, 0x7fff); //0x7fff bytes is an operating system size limit for this block + close(envfd); + + // Find the REMOTE_ADDR entry. Envrion block delimited by \0 + for (char *upto = envblock, *last = envblock; upto - envblock < bytes_read; ++upto) + { + if (*upto == '\0') + { + if (upto - last > 12 && strncmp(last, "REMOTE_ADDR=", 12) == 0) + return std::string((const char *)(last + 12)); + last = upto + 1; + } + } + + LOG_ERR << "Could not find REMOTE_ADDR variable in /proc/" << uc.pid << "/environ"; + return ""; } - LOG_ERR << "Could not find REMOTE_ADDR variable in /proc/" << uc.pid << "/environ"; - return ""; -} + void comm_server::stop() + { + should_stop_listening = true; + watchdog_thread.join(); -void comm_server::stop() -{ - should_stop_listening = true; - watchdog_thread.join(); - - if (websocketd_pid > 0) - kill(websocketd_pid, SIGINT); // Kill websocketd. -} + if (websocketd_pid > 0) + util::kill_process(websocketd_pid, false); // Kill websocketd. + } } // namespace comm diff --git a/src/conf.cpp b/src/conf.cpp index b3a9b66d..7a9633f6 100644 --- a/src/conf.cpp +++ b/src/conf.cpp @@ -79,8 +79,7 @@ int create_contract() boost::filesystem::create_directories(ctx.config_dir); boost::filesystem::create_directories(ctx.hist_dir); - boost::filesystem::create_directories(ctx.state_dir); - boost::filesystem::create_directories(ctx.state_hist_dir); + boost::filesystem::create_directories(ctx.state_rw_dir); //Create config file with default settings. @@ -136,9 +135,9 @@ void set_contract_dir_paths(std::string exepath, std::string basedir) basedir = util::realpath(basedir); ctx.exe_dir = boost::filesystem::path(util::realpath(exepath)).parent_path().string(); - ctx.statemon_exe_path = ctx.exe_dir + "/" + "hpstatemon"; ctx.websocketd_exe_path = ctx.exe_dir + "/" + "websocketd"; ctx.websocat_exe_path = ctx.exe_dir + "/" + "websocat"; + ctx.hpfs_exe_path = ctx.exe_dir + "/" + "hpfs"; ctx.contract_dir = basedir; ctx.config_dir = basedir + "/cfg"; @@ -147,7 +146,7 @@ void set_contract_dir_paths(std::string exepath, std::string basedir) ctx.tls_cert_file = ctx.config_dir + "/tlscert.pem"; ctx.hist_dir = basedir + "/hist"; ctx.state_dir = basedir + "/state"; - ctx.state_hist_dir = basedir + "/statehist"; + ctx.state_rw_dir = ctx.state_dir + "/rw"; ctx.log_dir = basedir + "/log"; } @@ -507,12 +506,11 @@ int validate_config() */ int validate_contract_dir_paths() { - const std::string paths[7] = { + const std::string paths[6] = { ctx.contract_dir, ctx.config_file, ctx.hist_dir, ctx.state_dir, - ctx.state_hist_dir, ctx.tls_key_file, ctx.tls_cert_file}; diff --git a/src/conf.hpp b/src/conf.hpp index e6fe21b2..84066c6f 100644 --- a/src/conf.hpp +++ b/src/conf.hpp @@ -27,14 +27,14 @@ struct contract_ctx { std::string command; // The CLI command issued to launch HotPocket std::string exe_dir; // Hot Pocket executable dir. - std::string statemon_exe_path; // State monitor executable file path. std::string websocketd_exe_path; // Websocketd executable file path. std::string websocat_exe_path; // Websocketd executable file path. + std::string hpfs_exe_path; // hpfs executable file path. std::string contract_dir; // Contract base directory full path std::string hist_dir; // Contract ledger history dir full path - std::string state_dir; // Contract executing state dir full path (This is the fuse mount point) - std::string state_hist_dir; // Contract state history dir full path + std::string state_dir; // Contract state maintenence path (hpfs path) + std::string state_rw_dir; // Contract executation read/write state path. std::string log_dir; // Contract log dir full path std::string config_dir; // Contract config dir full path std::string config_file; // Full path to the contract config file diff --git a/src/cons/cons.cpp b/src/cons/cons.cpp index e9edae0a..eb8ea0d3 100644 --- a/src/cons/cons.cpp +++ b/src/cons/cons.cpp @@ -10,11 +10,11 @@ #include "../hplog.hpp" #include "../crypto.hpp" #include "../proc.hpp" +#include "../hpfs/h32.hpp" +#include "../hpfs/hpfs.hpp" #include "ledger_handler.hpp" #include "state_handler.hpp" #include "cons.hpp" -#include "../statefs/state_common.hpp" -#include "../statefs/state_store.hpp" namespace p2pmsg = fbschema::p2pmsg; namespace jusrmsg = jsonschema::usrmsg; @@ -22,939 +22,907 @@ namespace jusrmsg = jsonschema::usrmsg; namespace cons { -/** + /** * Voting thresholds for consensus stages. */ -constexpr float STAGE1_THRESHOLD = 0.5; -constexpr float STAGE2_THRESHOLD = 0.65; -constexpr float STAGE3_THRESHOLD = 0.8; -constexpr float MAJORITY_THRESHOLD = 0.8; + constexpr float STAGE1_THRESHOLD = 0.5; + constexpr float STAGE2_THRESHOLD = 0.65; + constexpr float STAGE3_THRESHOLD = 0.8; + constexpr float MAJORITY_THRESHOLD = 0.8; -consensus_context ctx; + consensus_context ctx; -int init() -{ - //set start stage - ctx.stage = 0; + bool init_success = false; - //load lcl details from lcl history. - ledger_history ldr_hist = load_ledger(); - ctx.led_seq_no = ldr_hist.led_seq_no; - ctx.lcl = ldr_hist.lcl; - ctx.ledger_cache.swap(ldr_hist.cache); - - hasher::B2H root_hash = hasher::B2H_empty; - if (statefs::compute_hash_tree(root_hash, true) == -1) - return -1; - - LOG_INFO << "Initial state: " << root_hash; - - std::string str_root_hash(reinterpret_cast(&root_hash), hasher::HASH_SIZE); - str_root_hash.swap(ctx.curr_hash_state); - - if (!ctx.ledger_cache.empty()) + int init() { - ctx.prev_hash_state = ctx.ledger_cache.rbegin()->second.state; - } - else - { - ctx.prev_hash_state = ctx.curr_hash_state; + //set start stage + ctx.stage = 0; + + //load lcl details from lcl history. + ledger_history ldr_hist = load_ledger(); + ctx.led_seq_no = ldr_hist.led_seq_no; + ctx.lcl = ldr_hist.lcl; + ctx.ledger_cache.swap(ldr_hist.cache); + + if (hpfs::get_root_hash(ctx.curr_state_hash) == -1) + return -1; + + LOG_INFO << "Initial state: " << ctx.curr_state_hash; + + // We allocate 1/5 of the round time to each stage expect stage 3. For stage 3 we allocate 2/5. + // Stage 3 is allocated an extra stage_time unit becayse a node needs enough time to + // catch up from lcl/state desync. + ctx.stage_time = conf::cfg.roundtime / 5; + ctx.stage_reset_wait_threshold = conf::cfg.roundtime / 10; + + init_success = true; + return 0; } - ctx.state_syncing_thread = std::thread(&run_state_sync_iterator); - - // We allocate 1/5 of the round time to each stage expect stage 3. For stage 3 we allocate 2/5. - // Stage 3 is allocated an extra stage_time unit becayse a node needs enough time to - // catch up from lcl/state desync. - ctx.stage_time = conf::cfg.roundtime / 5; - ctx.stage_reset_wait_threshold = conf::cfg.roundtime / 10; - - return 0; -} - -/** + /** * Cleanup any resources. */ -void deinit() -{ - ctx.is_shutting_down = true; - ctx.state_syncing_thread.join(); -} - -void consensus() -{ - // A consensus round consists of 4 stages (0,1,2,3). - // For a given stage, this function may get visited multiple times due to time-wait conditions. - - uint64_t stage_start = 0; - if (!wait_and_proceed_stage(stage_start)) - return; // This means the stage has been reset. - - // Get the latest current time. - ctx.time_now = stage_start; - std::list collected_proposals; - - // Throughout consensus, we move over the incoming proposals collected via the network so far into - // the candidate proposal set (move and append). This is to have a private working set for the consensus - // and avoid threading conflicts with network incoming proposals. + void deinit() { - std::lock_guard lock(p2p::ctx.collected_msgs.proposals_mutex); - collected_proposals.splice(collected_proposals.end(), p2p::ctx.collected_msgs.proposals); - } + if (init_success) + { - //Copy collected propsals to candidate set of proposals. - //Add propsals of new nodes and replace proposals from old nodes to reflect current status of nodes. - for (const auto &proposal : collected_proposals) - { - auto prop_itr = ctx.candidate_proposals.find(proposal.pubkey); - if (prop_itr != ctx.candidate_proposals.end()) - { - ctx.candidate_proposals.erase(prop_itr); - ctx.candidate_proposals.emplace(proposal.pubkey, std::move(proposal)); - } - else - { - ctx.candidate_proposals.emplace(proposal.pubkey, std::move(proposal)); } } - // Throughout consensus, we move over the incoming npl messages collected via the network so far into - // the candidate npl message set (move and append). This is to have a private working set for the consensus - // and avoid threading conflicts with network incoming npl messages. - { - std::lock_guard lock(p2p::ctx.collected_msgs.npl_messages_mutex); - for (const auto &npl : p2p::ctx.collected_msgs.npl_messages) - { - const fbschema::p2pmsg::Container *container = fbschema::p2pmsg::GetContainer(npl.data()); - // Only the npl messages with a valid lcl will be passed down to the contract. lcl should match the previous round's lcl - if (fbschema::flatbuff_bytes_to_sv(container->lcl()) != ctx.lcl) - continue; - ctx.candidate_npl_messages.push_back(std::move(npl)); + int run_consensus() + { + while (true) + { + if (consensus() == -1) + return -1; } - p2p::ctx.collected_msgs.npl_messages.clear(); + + return 0; } - LOG_DBG << "Started stage " << std::to_string(ctx.stage); - - if (ctx.stage == 0) // Stage 0 means begining of a consensus round. + int consensus() { - // Broadcast non-unl proposals (NUP) containing inputs from locally connected users. - broadcast_nonunl_proposal(); - //util::sleep(conf::cfg.roundtime / 10); + // A consensus round consists of 4 stages (0,1,2,3). + // For a given stage, this function may get visited multiple times due to time-wait conditions. - // Verify and transfer user inputs from incoming NUPs onto consensus candidate data. - verify_and_populate_candidate_user_inputs(); + uint64_t stage_start = 0; + if (!wait_and_proceed_stage(stage_start)) + return 0; // This means the stage has been reset. - // In stage 0 we create a novel proposal and broadcast it. - const p2p::proposal stg_prop = create_stage0_proposal(); - broadcast_proposal(stg_prop); - } - else // Stage 1, 2, 3 - { - purify_candidate_proposals(); + // Get the latest current time. + ctx.time_now = stage_start; + std::list collected_proposals; - // Initialize vote counters - vote_counter votes; - - // check if we're ahead/behind of consensus lcl - bool is_lcl_desync, should_request_history; - std::string majority_lcl; - check_lcl_votes(is_lcl_desync, should_request_history, majority_lcl, votes); - - if (is_lcl_desync) + // Throughout consensus, we move over the incoming proposals collected via the network so far into + // the candidate proposal set (move and append). This is to have a private working set for the consensus + // and avoid threading conflicts with network incoming proposals. { - ctx.is_lcl_syncing = true; + std::lock_guard lock(p2p::ctx.collected_msgs.proposals_mutex); + collected_proposals.splice(collected_proposals.end(), p2p::ctx.collected_msgs.proposals); + } - if (should_request_history) + //Copy collected propsals to candidate set of proposals. + //Add propsals of new nodes and replace proposals from old nodes to reflect current status of nodes. + for (const auto &proposal : collected_proposals) + { + auto prop_itr = ctx.candidate_proposals.find(proposal.pubkey); + if (prop_itr != ctx.candidate_proposals.end()) { - LOG_INFO << "Syncing lcl. Curr lcl:" << cons::ctx.lcl.substr(0, 15) << " majority:" << majority_lcl.substr(0, 15); + ctx.candidate_proposals.erase(prop_itr); + ctx.candidate_proposals.emplace(proposal.pubkey, std::move(proposal)); + } + else + { + ctx.candidate_proposals.emplace(proposal.pubkey, std::move(proposal)); + } + } + // Throughout consensus, we move over the incoming npl messages collected via the network so far into + // the candidate npl message set (move and append). This is to have a private working set for the consensus + // and avoid threading conflicts with network incoming npl messages. + { + std::lock_guard lock(p2p::ctx.collected_msgs.npl_messages_mutex); + for (const auto &npl : p2p::ctx.collected_msgs.npl_messages) + { + const fbschema::p2pmsg::Container *container = fbschema::p2pmsg::GetContainer(npl.data()); + // Only the npl messages with a valid lcl will be passed down to the contract. lcl should match the previous round's lcl + if (fbschema::flatbuff_bytes_to_sv(container->lcl()) != ctx.lcl) + continue; - // TODO: If we are in a lcl fork condition try to rollback state with the help of - // state_restore to rollback state checkpoints before requesting new state. + ctx.candidate_npl_messages.push_back(std::move(npl)); + } + p2p::ctx.collected_msgs.npl_messages.clear(); + } - // Handle minority going forward when boostrapping cluster. - // Here we are mimicking invalid min ledger scenario. - if (majority_lcl == GENESIS_LEDGER) + LOG_DBG << "Started stage " << std::to_string(ctx.stage); + + if (ctx.stage == 0) // Stage 0 means begining of a consensus round. + { + // Broadcast non-unl proposals (NUP) containing inputs from locally connected users. + broadcast_nonunl_proposal(); + //util::sleep(conf::cfg.roundtime / 10); + + // Verify and transfer user inputs from incoming NUPs onto consensus candidate data. + verify_and_populate_candidate_user_inputs(); + + // In stage 0 we create a novel proposal and broadcast it. + const p2p::proposal stg_prop = create_stage0_proposal(); + broadcast_proposal(stg_prop); + } + else // Stage 1, 2, 3 + { + purify_candidate_proposals(); + + // Initialize vote counters + vote_counter votes; + + // check if we're ahead/behind of consensus lcl + bool is_lcl_desync, should_request_history; + std::string majority_lcl; + check_lcl_votes(is_lcl_desync, should_request_history, majority_lcl, votes); + + if (is_lcl_desync) + { + ctx.is_lcl_syncing = true; + + if (should_request_history) { - ctx.last_requested_lcl = majority_lcl; - p2p::history_response res; - res.error = p2p::LEDGER_RESPONSE_ERROR::INVALID_MIN_LEDGER; - handle_ledger_history_response(std::move(res)); + LOG_INFO << "Syncing lcl. Curr lcl:" << cons::ctx.lcl.substr(0, 15) << " majority:" << majority_lcl.substr(0, 15); + + // TODO: If we are in a lcl fork condition try to rollback state with the help of + // state_restore to rollback state checkpoints before requesting new state. + + // Handle minority going forward when boostrapping cluster. + // Here we are mimicking invalid min ledger scenario. + if (majority_lcl == GENESIS_LEDGER) + { + ctx.last_requested_lcl = majority_lcl; + p2p::history_response res; + res.error = p2p::LEDGER_RESPONSE_ERROR::INVALID_MIN_LEDGER; + handle_ledger_history_response(std::move(res)); + } + else + { + //create history request message and request history from a random peer. + send_ledger_history_request(ctx.lcl, majority_lcl); + } } - else + } + else + { + const bool lcl_syncing_just_finished = ctx.is_lcl_syncing; + ctx.is_lcl_syncing = false; + + if (lcl_syncing_just_finished) + ; //TODO: Check and compare majotiry state and start state sync. + bool is_state_syncing = false; + + if (!is_state_syncing) { - //create history request message and request history from a random peer. - send_ledger_history_request(ctx.lcl, majority_lcl); + conf::change_operating_mode(conf::OPERATING_MODE::PROPOSER); + + // In stage 1, 2, 3 we vote for incoming proposals and promote winning votes based on thresholds. + const p2p::proposal stg_prop = create_stage123_proposal(votes); + + broadcast_proposal(stg_prop); + + if (ctx.stage == 3) + { + if (apply_ledger(stg_prop) == -1) + return -1; + + // node has finished a consensus round (all 4 stages). + LOG_INFO << "****Stage 3 consensus reached**** (lcl:" << ctx.lcl.substr(0, 15) + << " state:" << ctx.curr_state_hash << ")"; + } } } } - else - { - const bool lcl_syncing_just_finished = ctx.is_lcl_syncing; - ctx.is_lcl_syncing = false; - if (lcl_syncing_just_finished || ctx.stage == 1 || (ctx.stage == 3 && ctx.is_state_syncing)) - check_state(votes); - - if (!ctx.is_state_syncing) - { - conf::change_operating_mode(conf::OPERATING_MODE::PROPOSER); - - // In stage 1, 2, 3 we vote for incoming proposals and promote winning votes based on thresholds. - const p2p::proposal stg_prop = create_stage123_proposal(votes); - - broadcast_proposal(stg_prop); - - if (ctx.stage == 3) - { - apply_ledger(stg_prop); - - // node has finished a consensus round (all 4 stages). - LOG_INFO << "****Stage 3 consensus reached**** (lcl:" << ctx.lcl.substr(0, 15) - << " state:" << *reinterpret_cast(cons::ctx.curr_hash_state.c_str()) << ")"; - } - } - } + // Node has finished a consensus stage. Transition to next stage. + ctx.stage = (ctx.stage + 1) % 4; + return 0; } - // Node has finished a consensus stage. Transition to next stage. - ctx.stage = (ctx.stage + 1) % 4; -} - -/** + /** * Cleanup any outdated proposals from the candidate set. */ -void purify_candidate_proposals() -{ - auto itr = ctx.candidate_proposals.begin(); - while (itr != ctx.candidate_proposals.end()) + void purify_candidate_proposals() { - const p2p::proposal &cp = itr->second; - - // only consider recent proposals and proposals from previous stage and current stage. - if ((ctx.time_now - cp.timestamp < conf::cfg.roundtime * 4) && cp.stage >= (ctx.stage - 1)) + auto itr = ctx.candidate_proposals.begin(); + while (itr != ctx.candidate_proposals.end()) { - ++itr; + const p2p::proposal &cp = itr->second; - bool self = cp.pubkey == conf::cfg.pubkey; - LOG_DBG << "Proposal [stage" << std::to_string(cp.stage) - << "] users:" << cp.users.size() - << " hinp:" << cp.hash_inputs.size() - << " hout:" << cp.hash_outputs.size() - << " ts:" << std::to_string(cp.time) - << " lcl:" << cp.lcl.substr(0, 15) - << " state:" << *reinterpret_cast(cp.curr_hash_state.c_str()) - << " self:" << self; - } - else - { - ctx.candidate_proposals.erase(itr++); + // only consider recent proposals and proposals from previous stage and current stage. + if ((ctx.time_now - cp.timestamp < conf::cfg.roundtime * 4) && cp.stage >= (ctx.stage - 1)) + { + ++itr; + + bool self = cp.pubkey == conf::cfg.pubkey; + LOG_DBG << "Proposal [stage" << std::to_string(cp.stage) + << "] users:" << cp.users.size() + << " hinp:" << cp.hash_inputs.size() + << " hout:" << cp.hash_outputs.size() + << " ts:" << std::to_string(cp.time) + << " lcl:" << cp.lcl.substr(0, 15) + << " state:" << cp.curr_state_hash + << " self:" << self; + } + else + { + ctx.candidate_proposals.erase(itr++); + } } } -} -/** + /** * Syncrhonise the stage/round time for fixed intervals and reset the stage. * @return True if consensus can proceed in the current round. False if stage is reset. */ -bool wait_and_proceed_stage(uint64_t &stage_start) -{ - // Here, nodes try to synchronise nodes stages using network clock. - // We devide universal time to windows of equal size of roundtime. Each round must be synced with the - // start of a window. - - const uint64_t now = util::get_epoch_milliseconds(); - - // Rrounds are divided into windows of roundtime. - // This gets the start time of current round window. Stage 0 must start in the next window. - const uint64_t current_round_start = (((uint64_t)(now / conf::cfg.roundtime)) * conf::cfg.roundtime); - - if (ctx.stage == 0) + bool wait_and_proceed_stage(uint64_t &stage_start) { - // Stage 0 must start in the next round window. - stage_start = current_round_start + conf::cfg.roundtime; - const int64_t to_wait = stage_start - now; + // Here, nodes try to synchronise nodes stages using network clock. + // We devide universal time to windows of equal size of roundtime. Each round must be synced with the + // start of a window. - LOG_DBG << "Waiting " << std::to_string(to_wait) << "ms for next round stage 0"; - util::sleep(to_wait); - return true; - } - else - { - stage_start = current_round_start + (ctx.stage * ctx.stage_time); + const uint64_t now = util::get_epoch_milliseconds(); - // Compute stage time wait. - // Node wait between stages to collect enough proposals from previous stages from other nodes. - const int64_t to_wait = stage_start - now; + // Rrounds are divided into windows of roundtime. + // This gets the start time of current round window. Stage 0 must start in the next window. + const uint64_t current_round_start = (((uint64_t)(now / conf::cfg.roundtime)) * conf::cfg.roundtime); - // If a node doesn't have enough time (eg. due to network delay) to recieve/send reliable stage proposals for next stage, - // it will continue particapating in this round, otherwise will join in next round. - if (to_wait < ctx.stage_reset_wait_threshold) //todo: self claculating/adjusting network delay + if (ctx.stage == 0) { - LOG_DBG << "Missed stage " << std::to_string(ctx.stage) << " window. Resetting to stage 0"; - ctx.stage = 0; - return false; - } - else - { - LOG_DBG << "Waiting " << std::to_string(to_wait) << "ms for stage " << std::to_string(ctx.stage); + // Stage 0 must start in the next round window. + stage_start = current_round_start + conf::cfg.roundtime; + const int64_t to_wait = stage_start - now; + + LOG_DBG << "Waiting " << std::to_string(to_wait) << "ms for next round stage 0"; util::sleep(to_wait); return true; } - } -} + else + { + stage_start = current_round_start + (ctx.stage * ctx.stage_time); -/** + // Compute stage time wait. + // Node wait between stages to collect enough proposals from previous stages from other nodes. + const int64_t to_wait = stage_start - now; + + // If a node doesn't have enough time (eg. due to network delay) to recieve/send reliable stage proposals for next stage, + // it will continue particapating in this round, otherwise will join in next round. + if (to_wait < ctx.stage_reset_wait_threshold) //todo: self claculating/adjusting network delay + { + LOG_DBG << "Missed stage " << std::to_string(ctx.stage) << " window. Resetting to stage 0"; + ctx.stage = 0; + return false; + } + else + { + LOG_DBG << "Waiting " << std::to_string(to_wait) << "ms for stage " << std::to_string(ctx.stage); + util::sleep(to_wait); + return true; + } + } + } + + /** * Broadcasts any inputs from locally connected users via an NUP. * @return 0 for successful broadcast. -1 for failure. */ -void broadcast_nonunl_proposal() -{ - std::lock_guard lock(p2p::ctx.collected_msgs.nonunl_proposals_mutex); - - if (usr::ctx.users.empty()) - return; - - // Construct NUP. - p2p::nonunl_proposal nup; - - for (auto &[sid, user] : usr::ctx.users) + void broadcast_nonunl_proposal() { - std::list usermsgs; - usermsgs.splice(usermsgs.end(), user.submitted_inputs); + std::lock_guard lock(p2p::ctx.collected_msgs.nonunl_proposals_mutex); - // We should create an entry for each user pubkey, even if the user has no inputs. This is - // because this data map will be used to track connected users as well in addition to inputs. - nup.user_messages.try_emplace(user.pubkey, std::move(usermsgs)); + if (usr::ctx.users.empty()) + return; + + // Construct NUP. + p2p::nonunl_proposal nup; + + for (auto &[sid, user] : usr::ctx.users) + { + std::list usermsgs; + usermsgs.splice(usermsgs.end(), user.submitted_inputs); + + // We should create an entry for each user pubkey, even if the user has no inputs. This is + // because this data map will be used to track connected users as well in addition to inputs. + nup.user_messages.try_emplace(user.pubkey, std::move(usermsgs)); + } + + flatbuffers::FlatBufferBuilder fbuf(1024); + p2pmsg::create_msg_from_nonunl_proposal(fbuf, nup); + p2p::broadcast_message(fbuf, true); + + LOG_DBG << "NUP sent." + << " users:" << nup.user_messages.size(); } - flatbuffers::FlatBufferBuilder fbuf(1024); - p2pmsg::create_msg_from_nonunl_proposal(fbuf, nup); - p2p::broadcast_message(fbuf, true); - - LOG_DBG << "NUP sent." - << " users:" << nup.user_messages.size(); -} - -/** + /** * Verifies the user signatures and populate non-expired user inputs from collected * non-unl proposals (if any) into consensus candidate data. */ -void verify_and_populate_candidate_user_inputs() -{ - // Lock the user sessions. - std::lock_guard users_lock(usr::ctx.users_mutex); - - // Lock the list so any network activity is blocked. - std::lock_guard nups_lock(p2p::ctx.collected_msgs.nonunl_proposals_mutex); - for (const p2p::nonunl_proposal &p : p2p::ctx.collected_msgs.nonunl_proposals) + void verify_and_populate_candidate_user_inputs() { - for (const auto &[pubkey, umsgs] : p.user_messages) + // Lock the user sessions. + std::lock_guard users_lock(usr::ctx.users_mutex); + + // Lock the list so any network activity is blocked. + std::lock_guard nups_lock(p2p::ctx.collected_msgs.nonunl_proposals_mutex); + for (const p2p::nonunl_proposal &p : p2p::ctx.collected_msgs.nonunl_proposals) { - // Locate this user's socket session in case we need to send any status messages regarding user inputs. - const comm::comm_session *session = usr::get_session_by_pubkey(pubkey); - - // Populate user list with this user's pubkey. - ctx.candidate_users.emplace(pubkey); - - // Keep track of total input length to verify against remaining balance. - // We only process inputs in the submitted order that can be satisfied with the remaining account balance. - size_t total_input_len = 0; - bool appbill_balance_exceeded = false; - - for (const usr::user_submitted_message &umsg : umsgs) + for (const auto &[pubkey, umsgs] : p.user_messages) { - const char *reject_reason = NULL; - const std::string sig_hash = crypto::get_hash(umsg.sig); + // Locate this user's socket session in case we need to send any status messages regarding user inputs. + const comm::comm_session *session = usr::get_session_by_pubkey(pubkey); - // Check for duplicate messages using hash of the signature. - if (ctx.recent_userinput_hashes.try_emplace(sig_hash)) + // Populate user list with this user's pubkey. + ctx.candidate_users.emplace(pubkey); + + // Keep track of total input length to verify against remaining balance. + // We only process inputs in the submitted order that can be satisfied with the remaining account balance. + size_t total_input_len = 0; + bool appbill_balance_exceeded = false; + + for (const usr::user_submitted_message &umsg : umsgs) { - // Verify the signature of the message content. - if (crypto::verify(umsg.content, umsg.sig, pubkey) == 0) + const char *reject_reason = NULL; + const std::string sig_hash = crypto::get_hash(umsg.sig); + + // Check for duplicate messages using hash of the signature. + if (ctx.recent_userinput_hashes.try_emplace(sig_hash)) { - std::string nonce; - std::string input; - uint64_t maxledgerseqno; - jusrmsg::extract_input_container(nonce, input, maxledgerseqno, umsg.content); - - // Ignore the input if our ledger has passed the input TTL. - if (maxledgerseqno > ctx.led_seq_no) + // Verify the signature of the message content. + if (crypto::verify(umsg.content, umsg.sig, pubkey) == 0) { - if (!appbill_balance_exceeded) - { - // Hash is prefixed with the nonce to support user-defined sort order. - std::string hash = std::move(nonce); - // Append the hash of the message signature to get the final hash. - hash.append(sig_hash); + std::string nonce; + std::string input; + uint64_t maxledgerseqno; + jusrmsg::extract_input_container(nonce, input, maxledgerseqno, umsg.content); - // Keep checking the subtotal of inputs extracted so far with the appbill account balance. - total_input_len += input.length(); - if (verify_appbill_check(pubkey, total_input_len)) + // Ignore the input if our ledger has passed the input TTL. + if (maxledgerseqno > ctx.led_seq_no) + { + if (!appbill_balance_exceeded) { - ctx.candidate_user_inputs.try_emplace( - hash, - candidate_user_input(pubkey, std::move(input), maxledgerseqno)); + // Hash is prefixed with the nonce to support user-defined sort order. + std::string hash = std::move(nonce); + // Append the hash of the message signature to get the final hash. + hash.append(sig_hash); + + // Keep checking the subtotal of inputs extracted so far with the appbill account balance. + total_input_len += input.length(); + if (verify_appbill_check(pubkey, total_input_len)) + { + ctx.candidate_user_inputs.try_emplace( + hash, + candidate_user_input(pubkey, std::move(input), maxledgerseqno)); + } + else + { + // Abandon processing further inputs from this user when we find out + // an input cannot be processed with the account balance. + appbill_balance_exceeded = true; + reject_reason = jusrmsg::REASON_APPBILL_BALANCE_EXCEEDED; + } } else { - // Abandon processing further inputs from this user when we find out - // an input cannot be processed with the account balance. - appbill_balance_exceeded = true; reject_reason = jusrmsg::REASON_APPBILL_BALANCE_EXCEEDED; } } else { - reject_reason = jusrmsg::REASON_APPBILL_BALANCE_EXCEEDED; + LOG_DBG << "User message bad max ledger seq expired."; + reject_reason = jusrmsg::REASON_MAX_LEDGER_EXPIRED; } } else { - LOG_DBG << "User message bad max ledger seq expired."; - reject_reason = jusrmsg::REASON_MAX_LEDGER_EXPIRED; + LOG_DBG << "User message bad signature."; + reject_reason = jusrmsg::REASON_BAD_SIG; } } else { - LOG_DBG << "User message bad signature."; - reject_reason = jusrmsg::REASON_BAD_SIG; + LOG_DBG << "Duplicate user message."; + reject_reason = jusrmsg::REASON_DUPLICATE_MSG; } - } - else - { - LOG_DBG << "Duplicate user message."; - reject_reason = jusrmsg::REASON_DUPLICATE_MSG; - } - // Send the request status result if this user is connected to us. - if (session != NULL) - { - usr::send_request_status_result(*session, - reject_reason == NULL ? jusrmsg::STATUS_ACCEPTED : jusrmsg::STATUS_REJECTED, - reject_reason == NULL ? "" : reject_reason, - jusrmsg::MSGTYPE_CONTRACT_INPUT, - jusrmsg::origin_data_for_contract_input(umsg.sig)); + // Send the request status result if this user is connected to us. + if (session != NULL) + { + usr::send_request_status_result(*session, + reject_reason == NULL ? jusrmsg::STATUS_ACCEPTED : jusrmsg::STATUS_REJECTED, + reject_reason == NULL ? "" : reject_reason, + jusrmsg::MSGTYPE_CONTRACT_INPUT, + jusrmsg::origin_data_for_contract_input(umsg.sig)); + } } } } + p2p::ctx.collected_msgs.nonunl_proposals.clear(); } - p2p::ctx.collected_msgs.nonunl_proposals.clear(); -} -/** + /** * Executes the appbill and verifies whether the user has enough account balance to process the provided input. * @param pubkey User binary pubkey. * @param input_len Total bytes length of user input. * @return Whether the user is allowed to process the input or not. */ -bool verify_appbill_check(std::string_view pubkey, const size_t input_len) -{ - // If appbill not enabled always green light the input. - if (conf::cfg.appbill.empty()) - return true; - - // execute appbill in --check mode to verify this user can submit a packet/connection to the network - // todo: this can be made more efficient, appbill --check can process 7 at a time - - // Fill appbill args - const int len = conf::cfg.runtime_appbill_args.size() + 4; - char *execv_args[len]; - for (int i = 0; i < conf::cfg.runtime_appbill_args.size(); i++) - execv_args[i] = conf::cfg.runtime_appbill_args[i].data(); - char option[] = "--check"; - execv_args[len - 4] = option; - // add the hex encoded public key as the last parameter - std::string hexpubkey; - util::bin2hex(hexpubkey, reinterpret_cast(pubkey.data()), pubkey.size()); - std::string inputsize = std::to_string(input_len); - execv_args[len - 3] = hexpubkey.data(); - execv_args[len - 2] = inputsize.data(); - execv_args[len - 1] = NULL; - - int pid = fork(); - if (pid == 0) + bool verify_appbill_check(std::string_view pubkey, const size_t input_len) { - // before execution chdir into a valid the latest state data directory that contains an appbill.table - chdir(statefs::current_ctx.data_dir.c_str()); - int ret = execv(execv_args[0], execv_args); - LOG_ERR << "Appbill process execv failed: " << ret; - return false; - } - else - { - // app bill in check mode takes a very short period of time to execute, typically 1ms - // so we will blocking wait for it here - int status = 0; - waitpid(pid, &status, 0); //todo: check error conditions here - status = WEXITSTATUS(status); - if (status != 128 && status != 0) - { - // this user's key passed appbill + // If appbill not enabled always green light the input. + if (conf::cfg.appbill.empty()) return true; + + // execute appbill in --check mode to verify this user can submit a packet/connection to the network + // todo: this can be made more efficient, appbill --check can process 7 at a time + + // Fill appbill args + const int len = conf::cfg.runtime_appbill_args.size() + 4; + char *execv_args[len]; + for (int i = 0; i < conf::cfg.runtime_appbill_args.size(); i++) + execv_args[i] = conf::cfg.runtime_appbill_args[i].data(); + char option[] = "--check"; + execv_args[len - 4] = option; + // add the hex encoded public key as the last parameter + std::string hexpubkey; + util::bin2hex(hexpubkey, reinterpret_cast(pubkey.data()), pubkey.size()); + std::string inputsize = std::to_string(input_len); + execv_args[len - 3] = hexpubkey.data(); + execv_args[len - 2] = inputsize.data(); + execv_args[len - 1] = NULL; + + int pid = fork(); + if (pid == 0) + { + // before execution chdir into a valid the latest state data directory that contains an appbill.table + chdir(conf::ctx.state_rw_dir.c_str()); + int ret = execv(execv_args[0], execv_args); + LOG_ERR << "Appbill process execv failed: " << ret; + return false; } else { - // user's key did not pass, do not add to user input candidates - LOG_DBG << "Appbill validation failed " << hexpubkey << " return code was " << status; - return false; + // app bill in check mode takes a very short period of time to execute, typically 1ms + // so we will blocking wait for it here + int status = 0; + waitpid(pid, &status, 0); //todo: check error conditions here + status = WEXITSTATUS(status); + if (status != 128 && status != 0) + { + // this user's key passed appbill + return true; + } + else + { + // user's key did not pass, do not add to user input candidates + LOG_DBG << "Appbill validation failed " << hexpubkey << " return code was " << status; + return false; + } } } -} -p2p::proposal create_stage0_proposal() -{ - // The proposal we are going to emit in stage 0. - p2p::proposal stg_prop; - stg_prop.time = ctx.time_now; - stg_prop.stage = 0; - stg_prop.lcl = ctx.lcl; - stg_prop.curr_hash_state = ctx.curr_hash_state; - - // Populate the proposal with set of candidate user pubkeys. - for (const std::string &pubkey : ctx.candidate_users) - stg_prop.users.emplace(pubkey); - - // We don't need candidate_users anymore, so clear it. It will be repopulated during next consensus round. - ctx.candidate_users.clear(); - - // Populate the proposal with hashes of user inputs. - for (const auto &[hash, cand_input] : ctx.candidate_user_inputs) - stg_prop.hash_inputs.emplace(hash); - - // Populate the proposal with hashes of user outputs. - for (const auto &[hash, cand_output] : ctx.candidate_user_outputs) - stg_prop.hash_outputs.emplace(hash); - - // todo: generate stg_prop hash and check with ctx.novel_proposal, we are sending same proposal again. - - return stg_prop; -} - -p2p::proposal create_stage123_proposal(vote_counter &votes) -{ - // The proposal to be emited at the end of this stage. - p2p::proposal stg_prop; - stg_prop.stage = ctx.stage; - - // we always vote for our current lcl and state regardless of what other peers are saying - // if there's a fork condition we will either request history and state from - // our peers or we will halt depending on level of consensus on the sides of the fork - stg_prop.lcl = ctx.lcl; - stg_prop.curr_hash_state = ctx.curr_hash_state; - - // Vote for rest of the proposal fields by looking at candidate proposals. - for (const auto &[pubkey, cp] : ctx.candidate_proposals) + p2p::proposal create_stage0_proposal() { - // Vote for times. - // Everyone votes on an arbitrary time, as long as its within the round time and not in the future. - if (ctx.time_now > cp.time && (ctx.time_now - cp.time) < conf::cfg.roundtime) - increment(votes.time, cp.time); + // The proposal we are going to emit in stage 0. + p2p::proposal stg_prop; + stg_prop.time = ctx.time_now; + stg_prop.stage = 0; + stg_prop.lcl = ctx.lcl; + stg_prop.curr_state_hash = ctx.curr_state_hash; - // Vote for user pubkeys. - for (const std::string &pubkey : cp.users) - increment(votes.users, pubkey); - - // Vote for user inputs (hashes). Only vote for the inputs that are in our candidate_inputs set. - for (const std::string &hash : cp.hash_inputs) - if (ctx.candidate_user_inputs.count(hash) > 0) - increment(votes.inputs, hash); - - // Vote for contract outputs (hashes). Only vote for the outputs that are in our candidate_outputs set. - for (const std::string &hash : cp.hash_outputs) - if (ctx.candidate_user_outputs.count(hash) > 0) - increment(votes.outputs, hash); - } - - const float_t vote_threshold = get_stage_threshold(ctx.stage); - - // todo: check if inputs being proposed by another node are actually spoofed inputs - // from a user locally connected to this node. - - // if we're at proposal stage 1 we'll accept any input and connection that has 1 or more vote. - - // Add user pubkeys which have votes over stage threshold to proposal. - for (const auto &[pubkey, numvotes] : votes.users) - if (numvotes >= vote_threshold || (ctx.stage == 1 && numvotes > 0)) + // Populate the proposal with set of candidate user pubkeys. + for (const std::string &pubkey : ctx.candidate_users) stg_prop.users.emplace(pubkey); - // Add inputs which have votes over stage threshold to proposal. - for (const auto &[hash, numvotes] : votes.inputs) - if (numvotes >= vote_threshold || (ctx.stage == 1 && numvotes > 0)) + // We don't need candidate_users anymore, so clear it. It will be repopulated during next consensus round. + ctx.candidate_users.clear(); + + // Populate the proposal with hashes of user inputs. + for (const auto &[hash, cand_input] : ctx.candidate_user_inputs) stg_prop.hash_inputs.emplace(hash); - // Add outputs which have votes over stage threshold to proposal. - for (const auto &[hash, numvotes] : votes.outputs) - if (numvotes >= vote_threshold) + // Populate the proposal with hashes of user outputs. + for (const auto &[hash, cand_output] : ctx.candidate_user_outputs) stg_prop.hash_outputs.emplace(hash); - // time is voted on a simple sorted (highest to lowest) and majority basis, since there will always be disagreement. - int32_t highest_time_vote = 0; - for (auto itr = votes.time.rbegin(); itr != votes.time.rend(); ++itr) - { - const uint64_t time = itr->first; - const int32_t numvotes = itr->second; + // todo: generate stg_prop hash and check with ctx.novel_proposal, we are sending same proposal again. - if (numvotes > highest_time_vote) - { - highest_time_vote = numvotes; - stg_prop.time = time; - } + return stg_prop; } - return stg_prop; -} + p2p::proposal create_stage123_proposal(vote_counter &votes) + { + // The proposal to be emited at the end of this stage. + p2p::proposal stg_prop; + stg_prop.stage = ctx.stage; -/** + // we always vote for our current lcl and state regardless of what other peers are saying + // if there's a fork condition we will either request history and state from + // our peers or we will halt depending on level of consensus on the sides of the fork + stg_prop.lcl = ctx.lcl; + stg_prop.curr_state_hash = ctx.curr_state_hash; + + // Vote for rest of the proposal fields by looking at candidate proposals. + for (const auto &[pubkey, cp] : ctx.candidate_proposals) + { + // Vote for times. + // Everyone votes on an arbitrary time, as long as its within the round time and not in the future. + if (ctx.time_now > cp.time && (ctx.time_now - cp.time) < conf::cfg.roundtime) + increment(votes.time, cp.time); + + // Vote for user pubkeys. + for (const std::string &pubkey : cp.users) + increment(votes.users, pubkey); + + // Vote for user inputs (hashes). Only vote for the inputs that are in our candidate_inputs set. + for (const std::string &hash : cp.hash_inputs) + if (ctx.candidate_user_inputs.count(hash) > 0) + increment(votes.inputs, hash); + + // Vote for contract outputs (hashes). Only vote for the outputs that are in our candidate_outputs set. + for (const std::string &hash : cp.hash_outputs) + if (ctx.candidate_user_outputs.count(hash) > 0) + increment(votes.outputs, hash); + } + + const float_t vote_threshold = get_stage_threshold(ctx.stage); + + // todo: check if inputs being proposed by another node are actually spoofed inputs + // from a user locally connected to this node. + + // if we're at proposal stage 1 we'll accept any input and connection that has 1 or more vote. + + // Add user pubkeys which have votes over stage threshold to proposal. + for (const auto &[pubkey, numvotes] : votes.users) + if (numvotes >= vote_threshold || (ctx.stage == 1 && numvotes > 0)) + stg_prop.users.emplace(pubkey); + + // Add inputs which have votes over stage threshold to proposal. + for (const auto &[hash, numvotes] : votes.inputs) + if (numvotes >= vote_threshold || (ctx.stage == 1 && numvotes > 0)) + stg_prop.hash_inputs.emplace(hash); + + // Add outputs which have votes over stage threshold to proposal. + for (const auto &[hash, numvotes] : votes.outputs) + if (numvotes >= vote_threshold) + stg_prop.hash_outputs.emplace(hash); + + // time is voted on a simple sorted (highest to lowest) and majority basis, since there will always be disagreement. + int32_t highest_time_vote = 0; + for (auto itr = votes.time.rbegin(); itr != votes.time.rend(); ++itr) + { + const uint64_t time = itr->first; + const int32_t numvotes = itr->second; + + if (numvotes > highest_time_vote) + { + highest_time_vote = numvotes; + stg_prop.time = time; + } + } + + return stg_prop; + } + + /** * Broadcasts the given proposal to all connected peers. * @return 0 on success. -1 if no peers to broadcast. */ -void broadcast_proposal(const p2p::proposal &p) -{ - flatbuffers::FlatBufferBuilder fbuf(1024); - p2pmsg::create_msg_from_proposal(fbuf, p); + void broadcast_proposal(const p2p::proposal &p) + { + flatbuffers::FlatBufferBuilder fbuf(1024); + p2pmsg::create_msg_from_proposal(fbuf, p); - // In observer mode, we only send out the proposal to ourselves. - if (conf::cfg.current_mode == conf::OPERATING_MODE::OBSERVER) - p2p::send_message_to_self(fbuf); - else - p2p::broadcast_message(fbuf, true); + // In observer mode, we only send out the proposal to ourselves. + if (conf::cfg.current_mode == conf::OPERATING_MODE::OBSERVER) + p2p::send_message_to_self(fbuf); + else + p2p::broadcast_message(fbuf, true); - // LOG_DBG << "Proposed [stage" << std::to_string(p.stage) - // << "] users:" << p.users.size() - // << " hinp:" << p.hash_inputs.size() - // << " hout:" << p.hash_outputs.size() - // << " ts:" << std::to_string(p.time); -} + // LOG_DBG << "Proposed [stage" << std::to_string(p.stage) + // << "] users:" << p.users.size() + // << " hinp:" << p.hash_inputs.size() + // << " hout:" << p.hash_outputs.size() + // << " ts:" << std::to_string(p.time); + } -/** + /** * Check our LCL is consistent with the proposals being made by our UNL peers lcl_votes. */ -void check_lcl_votes(bool &is_desync, bool &should_request_history, std::string &majority_lcl, vote_counter &votes) -{ - int32_t total_lcl_votes = 0; - - for (const auto &[pubkey, cp] : ctx.candidate_proposals) + void check_lcl_votes(bool &is_desync, bool &should_request_history, std::string &majority_lcl, vote_counter &votes) { - increment(votes.lcl, cp.lcl); - total_lcl_votes++; - } + int32_t total_lcl_votes = 0; - is_desync = false; - should_request_history = false; - - if (total_lcl_votes < (MAJORITY_THRESHOLD * conf::cfg.unl.size())) - { - LOG_DBG << "Not enough peers proposing to perform consensus. votes:" << std::to_string(total_lcl_votes) << " needed:" << std::to_string(MAJORITY_THRESHOLD * conf::cfg.unl.size()); - is_desync = true; - - //Not enough nodes are propsing. So Node is switching to Proposer if it's in observer mode. - conf::change_operating_mode(conf::OPERATING_MODE::PROPOSER); - - return; - } - - int32_t winning_votes = 0; - for (const auto [lcl, votes] : votes.lcl) - { - if (votes > winning_votes) + for (const auto &[pubkey, cp] : ctx.candidate_proposals) { - winning_votes = votes; - majority_lcl = lcl; + increment(votes.lcl, cp.lcl); + total_lcl_votes++; + } + + is_desync = false; + should_request_history = false; + + if (total_lcl_votes < (MAJORITY_THRESHOLD * conf::cfg.unl.size())) + { + LOG_DBG << "Not enough peers proposing to perform consensus. votes:" << std::to_string(total_lcl_votes) << " needed:" << std::to_string(MAJORITY_THRESHOLD * conf::cfg.unl.size()); + is_desync = true; + + //Not enough nodes are propsing. So Node is switching to Proposer if it's in observer mode. + conf::change_operating_mode(conf::OPERATING_MODE::PROPOSER); + + return; + } + + int32_t winning_votes = 0; + for (const auto [lcl, votes] : votes.lcl) + { + if (votes > winning_votes) + { + winning_votes = votes; + majority_lcl = lcl; + } + } + + //if winning lcl is not matched node lcl, + //that means vote is not on the consensus ledger. + //Should request history from a peer. + if (ctx.lcl != majority_lcl) + { + LOG_DBG << "We are not on the consensus ledger, requesting history from a random peer"; + is_desync = true; + + //Node is not in sync with current lcl ->switch to observer mode. + conf::change_operating_mode(conf::OPERATING_MODE::OBSERVER); + + should_request_history = true; + return; + } + + if (winning_votes < MAJORITY_THRESHOLD * ctx.candidate_proposals.size()) + { + // potential fork condition. + LOG_DBG << "No consensus on lcl. Possible fork condition. won:" << std::to_string(winning_votes) << " total:" << std::to_string(ctx.candidate_proposals.size()); + is_desync = true; + return; } } - //if winning lcl is not matched node lcl, - //that means vote is not on the consensus ledger. - //Should request history from a peer. - if (ctx.lcl != majority_lcl) - { - LOG_DBG << "We are not on the consensus ledger, requesting history from a random peer"; - is_desync = true; - - //Node is not in sync with current lcl ->switch to observer mode. - conf::change_operating_mode(conf::OPERATING_MODE::OBSERVER); - - should_request_history = true; - return; - } - - if (winning_votes < MAJORITY_THRESHOLD * ctx.candidate_proposals.size()) - { - // potential fork condition. - LOG_DBG << "No consensus on lcl. Possible fork condition. won:" << std::to_string(winning_votes) << " total:" << std::to_string(ctx.candidate_proposals.size()); - is_desync = true; - return; - } -} - -/** + /** * Returns the consensus percentage threshold for the specified stage. * @param stage The consensus stage [1, 2, 3] */ -float_t get_stage_threshold(const uint8_t stage) -{ - switch (stage) + float_t get_stage_threshold(const uint8_t stage) { - case 1: - return cons::STAGE1_THRESHOLD * conf::cfg.unl.size(); - case 2: - return cons::STAGE2_THRESHOLD * conf::cfg.unl.size(); - case 3: - return cons::STAGE3_THRESHOLD * conf::cfg.unl.size(); + switch (stage) + { + case 1: + return cons::STAGE1_THRESHOLD * conf::cfg.unl.size(); + case 2: + return cons::STAGE2_THRESHOLD * conf::cfg.unl.size(); + case 3: + return cons::STAGE3_THRESHOLD * conf::cfg.unl.size(); + } + return -1; } - return -1; -} -/** + /** * Finalize the ledger after consensus. * @param cons_prop The proposal that reached consensus. */ -void apply_ledger(const p2p::proposal &cons_prop) -{ - const std::tuple new_lcl = save_ledger(cons_prop); - ctx.led_seq_no = std::get<0>(new_lcl); - ctx.lcl = std::get<1>(new_lcl); - ctx.prev_hash_state = ctx.curr_hash_state; - - // After the current ledger seq no is updated, we remove any newly expired inputs from candidate set. + int apply_ledger(const p2p::proposal &cons_prop) { - auto itr = ctx.candidate_user_inputs.begin(); - while (itr != ctx.candidate_user_inputs.end()) + const std::tuple new_lcl = save_ledger(cons_prop); + ctx.led_seq_no = std::get<0>(new_lcl); + ctx.lcl = std::get<1>(new_lcl); + + // After the current ledger seq no is updated, we remove any newly expired inputs from candidate set. { - if (itr->second.maxledgerseqno <= ctx.led_seq_no) - ctx.candidate_user_inputs.erase(itr++); - else - ++itr; + auto itr = ctx.candidate_user_inputs.begin(); + while (itr != ctx.candidate_user_inputs.end()) + { + if (itr->second.maxledgerseqno <= ctx.led_seq_no) + ctx.candidate_user_inputs.erase(itr++); + else + ++itr; + } } + + // Send any output from the previous consensus round to locally connected users. + dispatch_user_outputs(cons_prop); + + proc::contract_bufmap_t useriobufmap; + + proc::contract_iobuf_pair nplbufpair; + nplbufpair.inputs.splice(nplbufpair.inputs.end(), ctx.candidate_npl_messages); + + feed_user_inputs_to_contract_bufmap(useriobufmap, cons_prop); + + if (run_contract_binary(cons_prop.time, useriobufmap, nplbufpair) == -1) + return -1; + + extract_user_outputs_from_contract_bufmap(useriobufmap); + broadcast_npl_output(nplbufpair.output); + return 0; } - // Send any output from the previous consensus round to locally connected users. - dispatch_user_outputs(cons_prop); - - // This will hold a list of file blocks that was updated by the contract process. - // We then feed this information to state tracking logic. - proc::contract_fblockmap_t updated_blocks; - - proc::contract_bufmap_t useriobufmap; - - proc::contract_iobuf_pair nplbufpair; - nplbufpair.inputs.splice(nplbufpair.inputs.end(), ctx.candidate_npl_messages); - - feed_user_inputs_to_contract_bufmap(useriobufmap, cons_prop); - - run_contract_binary(cons_prop.time, useriobufmap, nplbufpair, updated_blocks); - - extract_user_outputs_from_contract_bufmap(useriobufmap); - broadcast_npl_output(nplbufpair.output); -} - -/** + /** * Dispatch any consensus-reached outputs to matching users if they are connected to us locally. * @param cons_prop The proposal that achieved consensus. */ -void dispatch_user_outputs(const p2p::proposal &cons_prop) -{ - std::lock_guard lock(usr::ctx.users_mutex); - - for (const std::string &hash : cons_prop.hash_outputs) + void dispatch_user_outputs(const p2p::proposal &cons_prop) { - const auto cu_itr = ctx.candidate_user_outputs.find(hash); - const bool hashfound = (cu_itr != ctx.candidate_user_outputs.end()); - if (!hashfound) - { - LOG_ERR << "Output required but wasn't in our candidate outputs map, this will potentially cause desync."; - // todo: consider fatal - } - else - { - // Send matching outputs to locally connected users. + std::lock_guard lock(usr::ctx.users_mutex); - candidate_user_output &cand_output = cu_itr->second; - - // Find the user session by user pubkey. - const auto sess_itr = usr::ctx.sessionids.find(cand_output.userpubkey); - if (sess_itr != usr::ctx.sessionids.end()) // match found + for (const std::string &hash : cons_prop.hash_outputs) + { + const auto cu_itr = ctx.candidate_user_outputs.find(hash); + const bool hashfound = (cu_itr != ctx.candidate_user_outputs.end()); + if (!hashfound) { - const auto user_itr = usr::ctx.users.find(sess_itr->second); // sess_itr->second is the session id. - if (user_itr != usr::ctx.users.end()) // match found - { - std::string outputtosend; - outputtosend.swap(cand_output.output); - - std::string msg; - jusrmsg::create_contract_output_container(msg, outputtosend); - - const usr::connected_user &user = user_itr->second; - user.session.send(msg); - } + LOG_ERR << "Output required but wasn't in our candidate outputs map, this will potentially cause desync."; + // todo: consider fatal } + else + { + // Send matching outputs to locally connected users. - // now we can safely delete this candidate output. - ctx.candidate_user_outputs.erase(cu_itr); + candidate_user_output &cand_output = cu_itr->second; + + // Find the user session by user pubkey. + const auto sess_itr = usr::ctx.sessionids.find(cand_output.userpubkey); + if (sess_itr != usr::ctx.sessionids.end()) // match found + { + const auto user_itr = usr::ctx.users.find(sess_itr->second); // sess_itr->second is the session id. + if (user_itr != usr::ctx.users.end()) // match found + { + std::string outputtosend; + outputtosend.swap(cand_output.output); + + std::string msg; + jusrmsg::create_contract_output_container(msg, outputtosend); + + const usr::connected_user &user = user_itr->second; + user.session.send(msg); + } + } + + // now we can safely delete this candidate output. + ctx.candidate_user_outputs.erase(cu_itr); + } } } -} -/** + /** * Check state against the winning and canonical state * @param votes The voting table. */ -void check_state(vote_counter &votes) -{ - std::string majority_state; - - for (const auto &[pubkey, cp] : ctx.candidate_proposals) + void check_state(vote_counter &votes) { - increment(votes.state, cp.curr_hash_state); - } + hpfs::h32 majority_state = hpfs::h32_empty; - int32_t winning_votes = 0; - for (const auto [state, votes] : votes.state) - { - if (votes > winning_votes) + for (const auto &[pubkey, cp] : ctx.candidate_proposals) { - winning_votes = votes; - majority_state = state; + increment(votes.state, cp.curr_state_hash); + } + + int32_t winning_votes = 0; + for (const auto [state, votes] : votes.state) + { + if (votes > winning_votes) + { + winning_votes = votes; + majority_state = state; + } } } - if (ctx.is_state_syncing) - { - std::lock_guard lock(cons::ctx.state_syncing_mutex); - hasher::B2H root_hash = hasher::B2H_empty; - int ret = statefs::compute_hash_tree(root_hash); - std::string str_root_hash(reinterpret_cast(&root_hash), hasher::HASH_SIZE); - str_root_hash.swap(ctx.curr_hash_state); - } - - // We do not initiate state sync in stage 3 because the majority state is likely to get changed soon. - if (ctx.stage < 3 && majority_state != ctx.curr_hash_state) - { - if (ctx.state_sync_lcl != ctx.lcl) - { - // Switch to observer mode to avoid sending out proposals till the state is synced - conf::change_operating_mode(conf::OPERATING_MODE::OBSERVER); - - const hasher::B2H majority_state_hash = *reinterpret_cast(majority_state.c_str()); - LOG_INFO << "Syncing state. Curr state:" << *reinterpret_cast(ctx.curr_hash_state.c_str()) << " majority:" << majority_state_hash; - - start_state_sync(majority_state_hash); - - ctx.is_state_syncing = true; - ctx.state_sync_lcl = ctx.lcl; - } - } - else if (majority_state == ctx.curr_hash_state && ctx.is_state_syncing) - { - LOG_INFO << "State sync complete. state:" << *reinterpret_cast(ctx.curr_hash_state.c_str()); - - ctx.is_state_syncing = false; - ctx.state_sync_lcl.clear(); - } -} - -/** + /** * Transfers consensus-reached inputs into the provided contract buf map so it can be fed into the contract process. * @param bufmap The contract bufmap which needs to be populated with inputs. * @param cons_prop The proposal that achieved consensus. */ -void feed_user_inputs_to_contract_bufmap(proc::contract_bufmap_t &bufmap, const p2p::proposal &cons_prop) -{ - // Populate the buf map with all currently connected users regardless of whether they have inputs or not. - // This is in case the contract wanted to emit some data to a user without needing any input. - for (const std::string &pubkey : cons_prop.users) - bufmap.try_emplace(pubkey, proc::contract_iobuf_pair()); - - for (const std::string &hash : cons_prop.hash_inputs) + void feed_user_inputs_to_contract_bufmap(proc::contract_bufmap_t &bufmap, const p2p::proposal &cons_prop) { - // For each consensus input hash, we need to find the actual input content to feed the contract. - const auto itr = ctx.candidate_user_inputs.find(hash); - const bool hashfound = (itr != ctx.candidate_user_inputs.end()); - if (!hashfound) + // Populate the buf map with all currently connected users regardless of whether they have inputs or not. + // This is in case the contract wanted to emit some data to a user without needing any input. + for (const std::string &pubkey : cons_prop.users) + bufmap.try_emplace(pubkey, proc::contract_iobuf_pair()); + + for (const std::string &hash : cons_prop.hash_inputs) { - LOG_ERR << "input required but wasn't in our candidate inputs map, this will potentially cause desync."; - // TODO: consider fatal - } - else - { - // Populate the input content into the bufmap. + // For each consensus input hash, we need to find the actual input content to feed the contract. + const auto itr = ctx.candidate_user_inputs.find(hash); + const bool hashfound = (itr != ctx.candidate_user_inputs.end()); + if (!hashfound) + { + LOG_ERR << "input required but wasn't in our candidate inputs map, this will potentially cause desync."; + // TODO: consider fatal + } + else + { + // Populate the input content into the bufmap. - candidate_user_input &cand_input = itr->second; + candidate_user_input &cand_input = itr->second; - std::string inputtofeed; - inputtofeed.swap(cand_input.input); + std::string inputtofeed; + inputtofeed.swap(cand_input.input); - proc::contract_iobuf_pair &bufpair = bufmap[cand_input.userpubkey]; - bufpair.inputs.push_back(std::move(inputtofeed)); + proc::contract_iobuf_pair &bufpair = bufmap[cand_input.userpubkey]; + bufpair.inputs.push_back(std::move(inputtofeed)); - // Remove the input from the candidate set because we no longer need it. - //LOG_DBG << "candidate input deleted."; - ctx.candidate_user_inputs.erase(itr); + // Remove the input from the candidate set because we no longer need it. + //LOG_DBG << "candidate input deleted."; + ctx.candidate_user_inputs.erase(itr); + } } } -} -/** + /** * Reads any outputs the contract has produced on the provided buf map and transfers them to candidate outputs * for the next consensus round. * @param bufmap The contract bufmap containing the outputs produced by the contract. */ -void extract_user_outputs_from_contract_bufmap(proc::contract_bufmap_t &bufmap) -{ - for (auto &[pubkey, bufpair] : bufmap) + void extract_user_outputs_from_contract_bufmap(proc::contract_bufmap_t &bufmap) { - if (!bufpair.output.empty()) + for (auto &[pubkey, bufpair] : bufmap) { - std::string output; - output.swap(bufpair.output); + if (!bufpair.output.empty()) + { + std::string output; + output.swap(bufpair.output); - const std::string hash = crypto::get_hash(pubkey, output); - ctx.candidate_user_outputs.try_emplace( - std::move(hash), - candidate_user_output(pubkey, std::move(output))); + const std::string hash = crypto::get_hash(pubkey, output); + ctx.candidate_user_outputs.try_emplace( + std::move(hash), + candidate_user_output(pubkey, std::move(output))); + } } } -} -void broadcast_npl_output(std::string &output) -{ - if (!output.empty()) + void broadcast_npl_output(std::string &output) { - p2p::npl_message npl; - npl.data.swap(output); + if (!output.empty()) + { + p2p::npl_message npl; + npl.data.swap(output); - flatbuffers::FlatBufferBuilder fbuf(1024); - p2pmsg::create_msg_from_npl_output(fbuf, npl, ctx.lcl); - p2p::broadcast_message(fbuf, false); + flatbuffers::FlatBufferBuilder fbuf(1024); + p2pmsg::create_msg_from_npl_output(fbuf, npl, ctx.lcl); + p2p::broadcast_message(fbuf, false); + } } -} -/** + /** * Executes the smart contract with the specified time and provided I/O buf maps. * @param time_now The time that must be passed on to the contract. * @param useriobufmap The contract bufmap which holds user I/O buffers. */ -void run_contract_binary(const int64_t time_now, proc::contract_bufmap_t &useriobufmap, proc::contract_iobuf_pair &nplbufpair, proc::contract_fblockmap_t &state_updates) -{ - // todo:implement exchange of hpsc bufs - proc::contract_iobuf_pair hpscbufpair; - proc::exec_contract( - proc::contract_exec_args(time_now, useriobufmap, nplbufpair, hpscbufpair, state_updates)); -} + int run_contract_binary(const int64_t time_now, proc::contract_bufmap_t &useriobufmap, proc::contract_iobuf_pair &nplbufpair) + { + // todo:implement exchange of hpsc bufs + proc::contract_iobuf_pair hpscbufpair; + return proc::exec_contract( + proc::contract_exec_args(time_now, useriobufmap, nplbufpair, hpscbufpair), + ctx.curr_state_hash); + } -/** + /** * Increment voting table counter. * @param counter The counter map in which a vote should be incremented. * @param candidate The candidate whose vote should be increased by 1. */ -template -void increment(std::map &counter, const T &candidate) -{ - if (counter.count(candidate)) - counter[candidate]++; - else - counter.try_emplace(candidate, 1); -} + template + void increment(std::map &counter, const T &candidate) + { + if (counter.count(candidate)) + counter[candidate]++; + else + counter.try_emplace(candidate, 1); + } } // namespace cons diff --git a/src/cons/cons.hpp b/src/cons/cons.hpp index 71cbf13d..c84b621d 100644 --- a/src/cons/cons.hpp +++ b/src/cons/cons.hpp @@ -6,6 +6,7 @@ #include "../proc.hpp" #include "../p2p/p2p.hpp" #include "../usr/user_input.hpp" +#include "../hpfs/h32.hpp" #include "ledger_handler.hpp" #include "state_handler.hpp" @@ -73,8 +74,7 @@ struct consensus_context uint64_t time_now = 0; std::string lcl; uint64_t led_seq_no = 0; - std::string curr_hash_state; - std::string prev_hash_state; + hpfs::h32 curr_state_hash; //Map of closed ledgers(only lrdgername[sequnece_number-hash], state hash) with sequence number as map key. //contains closed ledgers from latest to latest - MAX_LEDGER_SEQUENCE. @@ -88,11 +88,6 @@ struct consensus_context uint16_t stage_time = 0; // Time allocated to a consensus stage. uint16_t stage_reset_wait_threshold = 0; // Minimum stage wait time to reset the stage. - bool is_state_syncing = false; - std::string state_sync_lcl; - std::thread state_syncing_thread; - std::mutex state_syncing_mutex; - bool is_shutting_down = false; consensus_context() @@ -108,7 +103,7 @@ struct vote_counter std::map users; std::map inputs; std::map outputs; - std::map state; + std::map state; }; extern consensus_context ctx; @@ -117,7 +112,9 @@ int init(); void deinit(); -void consensus(); +int run_consensus(); + +int consensus(); void purify_candidate_proposals(); @@ -145,7 +142,7 @@ uint64_t get_ledger_time_resolution(const uint64_t time); uint64_t get_stage_time_resolution(const uint64_t time); -void apply_ledger(const p2p::proposal &proposal); +int apply_ledger(const p2p::proposal &proposal); void dispatch_user_outputs(const p2p::proposal &cons_prop); @@ -157,7 +154,7 @@ void extract_user_outputs_from_contract_bufmap(proc::contract_bufmap_t &bufmap); void broadcast_npl_output(std::string &output); -void run_contract_binary(const int64_t time_now, proc::contract_bufmap_t &useriobufmap, proc::contract_iobuf_pair &nplbufpair, proc::contract_fblockmap_t &state_updates); +int run_contract_binary(const int64_t time_now, proc::contract_bufmap_t &useriobufmap, proc::contract_iobuf_pair &nplbufpair); template void increment(std::map &counter, const T &candidate); diff --git a/src/cons/ledger_handler.cpp b/src/cons/ledger_handler.cpp index 5d7b8081..12c14367 100644 --- a/src/cons/ledger_handler.cpp +++ b/src/cons/ledger_handler.cpp @@ -61,7 +61,7 @@ const std::tuple save_ledger(const p2p::proposal &p ledger_cache_entry c; c.lcl = file_name; - c.state = proposal.curr_hash_state; + c.state = proposal.curr_state_hash.to_string_view(); cons::ctx.ledger_cache.emplace(led_seq_no, std::move(c)); //Remove old ledgers that exceeds max sequence range. diff --git a/src/cons/state_handler.cpp b/src/cons/state_handler.cpp index 51714f8a..40cc7b98 100644 --- a/src/cons/state_handler.cpp +++ b/src/cons/state_handler.cpp @@ -5,7 +5,6 @@ #include "../p2p/p2p.hpp" #include "../pchheader.hpp" #include "../cons/cons.hpp" -#include "../statefs/state_store.hpp" #include "../hplog.hpp" #include "../util.hpp" @@ -24,7 +23,7 @@ std::list candidate_state_responses; std::list pending_requests; // List of submitted requests we are awaiting responses for, keyed by expected response hash. -std::unordered_map submitted_requests; +std::unordered_map submitted_requests; /** * Sends a state request to a random peer. @@ -33,7 +32,7 @@ std::unordered_map submit * @param block_id The requested block id. Only relevant if requesting a file block. Otherwise -1. * @param expected_hash The expected hash of the requested data. The peer will ignore the request if their hash is different. */ -void request_state_from_peer(const std::string &path, const bool is_file, const int32_t block_id, const hasher::B2H expected_hash) +void request_state_from_peer(const std::string &path, const bool is_file, const int32_t block_id, const hpfs::h32 expected_hash) { p2p::state_request sr; sr.parent_path = path; @@ -59,8 +58,8 @@ int create_state_response(flatbuffers::FlatBufferBuilder &fbuf, const p2p::state // Vector to hold the block bytes. Normally block size is constant BLOCK_SIZE (4MB), but the // last block of a file may have a smaller size. std::vector block; - if (statefs::get_block(block, sr.parent_path, sr.block_id, sr.expected_hash) == -1) - return -1; + + // TODO: get block p2p::block_response resp; resp.path = sr.parent_path; @@ -76,17 +75,19 @@ int create_state_response(flatbuffers::FlatBufferBuilder &fbuf, const p2p::state if (sr.is_file) { std::vector existing_block_hashmap; - if (statefs::get_block_hash_map(existing_block_hashmap, sr.parent_path, sr.expected_hash) == -1) - return -1; + + // TODO: get block hash list + // TODO: get file length + std::size_t file_length = 0; - fbschema::p2pmsg::create_msg_from_filehashmap_response(fbuf, sr.parent_path, existing_block_hashmap, statefs::get_file_length(sr.parent_path), sr.expected_hash, ctx.lcl); + fbschema::p2pmsg::create_msg_from_filehashmap_response(fbuf, sr.parent_path, existing_block_hashmap, file_length, sr.expected_hash, ctx.lcl); } else { // If the state request is for a directory we need to reply with the file system entries and their hashes inside that dir. std::unordered_map existing_fs_entries; - if (statefs::get_fs_entry_hashes(existing_fs_entries, sr.parent_path, sr.expected_hash) == -1) - return -1; + + // TODO: get fs entry hashes fbschema::p2pmsg::create_msg_from_fsentry_response(fbuf, sr.parent_path, existing_fs_entries, sr.expected_hash, ctx.lcl); } @@ -100,7 +101,7 @@ int create_state_response(flatbuffers::FlatBufferBuilder &fbuf, const p2p::state * @param state_hash_to_request Peer's expected state hash. If peer doesn't have this as its state hash the * request will be ignord. */ -void start_state_sync(const hasher::B2H state_hash_to_request) +void start_state_sync(const hpfs::h32 state_hash_to_request) { { std::lock_guard lock(p2p::ctx.collected_msgs.state_response_mutex); @@ -108,7 +109,6 @@ void start_state_sync(const hasher::B2H state_hash_to_request) } { - std::lock_guard lock(cons::ctx.state_syncing_mutex); candidate_state_responses.clear(); pending_requests.clear(); submitted_requests.clear(); @@ -133,8 +133,6 @@ int run_state_sync_iterator() util::sleep(SYNC_LOOP_WAIT); // TODO: Also bypass peer session handler state responses if we're not syncing. - if (!ctx.is_state_syncing) - continue; { std::lock_guard lock(p2p::ctx.collected_msgs.state_response_mutex); @@ -144,8 +142,6 @@ int run_state_sync_iterator() candidate_state_responses.splice(candidate_state_responses.end(), p2p::ctx.collected_msgs.state_response); } - std::lock_guard lock(cons::ctx.state_syncing_mutex); - for (auto &response : candidate_state_responses) { if (ctx.is_shutting_down) @@ -155,7 +151,7 @@ int run_state_sync_iterator() const fbschema::p2pmsg::State_Response_Message *resp_msg = content->message_as_State_Response_Message(); // Check whether we are actually waiting for this response's hash. If not, ignore it. - hasher::B2H response_hash = fbschema::flatbuff_bytes_to_hash(resp_msg->hash()); + hpfs::h32 response_hash = fbschema::flatbuff_bytes_to_hash(resp_msg->hash()); const auto pending_resp_itr = submitted_requests.find(response_hash); if (pending_resp_itr == submitted_requests.end()) continue; @@ -250,15 +246,17 @@ int handle_fs_entry_response(const fbschema::p2pmsg::Fs_Entry_Response *fs_entry std::string_view root_path_sv = fbschema::flatbuff_str_to_sv(fs_entry_resp->path()); std::string root_path_str(root_path_sv.data(), root_path_sv.size()); - if (!statefs::is_dir_exists(root_path_str)) - { - statefs::create_dir(root_path_str); - } - else - { - if (statefs::get_fs_entry_hashes(existing_fs_entries, std::move(root_path_str), hasher::B2H_empty) == -1) - return -1; - } + // TODO: Create state path dir if not exist. + // TODO: Get existing fs entries hash map. + // if (!statefs::is_dir_exists(root_path_str)) + // { + // statefs::create_dir(root_path_str); + // } + // else + // { + // if (statefs::get_fs_entry_hashes(existing_fs_entries, std::move(root_path_str), hpfs::h32_empty) == -1) + // return -1; + // } // Request more info on fs entries that exist on both sides but are different. for (const auto &[path, fs_entry] : existing_fs_entries) @@ -281,13 +279,13 @@ int handle_fs_entry_response(const fbschema::p2pmsg::Fs_Entry_Response *fs_entry // If there was an entry that does not exist on other side, delete it from this node. if (fs_entry.is_file) { - if (statefs::delete_file(path) == -1) - return -1; + //if (statefs::delete_file(path) == -1) + // return -1; } else { - if (statefs::delete_dir(path) == -1) - return -1; + //if (statefs::delete_dir(path) == -1) + // return -1; } } } @@ -310,14 +308,14 @@ int handle_file_hashmap_response(const fbschema::p2pmsg::File_HashMap_Response * const std::string path_str(path_sv.data(), path_sv.size()); std::vector existing_block_hashmap; - if (statefs::get_block_hash_map(existing_block_hashmap, path_str, hasher::B2H_empty) == -1) - return -1; + //if (statefs::get_block_hash_map(existing_block_hashmap, path_str, hpfs::h32_empty) == -1) + // return -1; - const hasher::B2H *existing_hashes = reinterpret_cast(existing_block_hashmap.data()); - auto existing_hash_count = existing_block_hashmap.size() / hasher::HASH_SIZE; + const hpfs::h32 *existing_hashes = reinterpret_cast(existing_block_hashmap.data()); + auto existing_hash_count = existing_block_hashmap.size() / sizeof(hpfs::h32); - const hasher::B2H *resp_hashes = reinterpret_cast(file_resp->hash_map()->data()); - auto resp_hash_count = file_resp->hash_map()->size() / hasher::HASH_SIZE; + const hpfs::h32 *resp_hashes = reinterpret_cast(file_resp->hash_map()->data()); + auto resp_hash_count = file_resp->hash_map()->size() / sizeof(hpfs::h32); auto insert_itr = pending_requests.begin(); @@ -335,8 +333,8 @@ int handle_file_hashmap_response(const fbschema::p2pmsg::File_HashMap_Response * if (existing_hash_count > resp_hash_count) { - if (statefs::truncate_file(path_str, file_resp->file_length()) == -1) - return -1; + //if (statefs::truncate_file(path_str, file_resp->file_length()) == -1) + // return -1; } else if (existing_hash_count < resp_hash_count) { @@ -354,8 +352,8 @@ int handle_file_block_response(const fbschema::p2pmsg::Block_Response *block_msg { p2p::block_response block_resp = fbschema::p2pmsg::create_block_response_from_msg(*block_msg); - if (statefs::write_block(block_resp.path, block_resp.block_id, block_resp.data.data(), block_resp.data.size()) == -1) - return -1; + //if (statefs::write_block(block_resp.path, block_resp.block_id, block_resp.data.data(), block_resp.data.size()) == -1) + // return -1; return 0; } diff --git a/src/cons/state_handler.hpp b/src/cons/state_handler.hpp index 01955255..47963fd3 100644 --- a/src/cons/state_handler.hpp +++ b/src/cons/state_handler.hpp @@ -4,7 +4,7 @@ #include "../pchheader.hpp" #include "../p2p/p2p.hpp" #include "../fbschema/p2pmsg_content_generated.h" -#include "../statefs/hasher.hpp" +#include "../hpfs/h32.hpp" namespace cons { @@ -22,7 +22,7 @@ struct backlog_item BACKLOG_ITEM_TYPE type; std::string path; int32_t block_id = -1; // Only relevant if type=BLOCK - hasher::B2H expected_hash; + hpfs::h32 expected_hash; // No. of cycles that this item has been waiting in pending state. // Used by pending_responses list to increase wait count. @@ -33,9 +33,9 @@ extern std::list candidate_state_responses; int create_state_response(flatbuffers::FlatBufferBuilder &fbuf, const p2p::state_request &sr); -void request_state_from_peer(const std::string &path, const bool is_file, const int32_t block_id, const hasher::B2H expected_hash); +void request_state_from_peer(const std::string &path, const bool is_file, const int32_t block_id, const hpfs::h32 expected_hash); -void start_state_sync(const hasher::B2H state_hash_to_request); +void start_state_sync(const hpfs::h32 state_hash_to_request); int run_state_sync_iterator(); diff --git a/src/fbschema/common_helpers.cpp b/src/fbschema/common_helpers.cpp index 51a6e73e..0008f5bf 100644 --- a/src/fbschema/common_helpers.cpp +++ b/src/fbschema/common_helpers.cpp @@ -3,124 +3,124 @@ namespace fbschema { -//---Conversion helpers from flatbuffers data types to std data types---// + //---Conversion helpers from flatbuffers data types to std data types---// -/** + /** * Returns string_view from flat buffer data pointer and length. */ -std::string_view flatbuff_bytes_to_sv(const uint8_t *data, const flatbuffers::uoffset_t length) -{ - const char *signature_content_str = reinterpret_cast(data); - return std::string_view(signature_content_str, length); -} + std::string_view flatbuff_bytes_to_sv(const uint8_t *data, const flatbuffers::uoffset_t length) + { + const char *signature_content_str = reinterpret_cast(data); + return std::string_view(signature_content_str, length); + } -/** + /** * Returns string_view from Flat Buffer vector of bytes. */ -std::string_view flatbuff_bytes_to_sv(const flatbuffers::Vector *buffer) -{ - return flatbuff_bytes_to_sv(buffer->Data(), buffer->size()); -} + std::string_view flatbuff_bytes_to_sv(const flatbuffers::Vector *buffer) + { + return flatbuff_bytes_to_sv(buffer->Data(), buffer->size()); + } -/** + /** * Returns return string_view from Flat Buffer string. */ -std::string_view flatbuff_str_to_sv(const flatbuffers::String *buffer) -{ - return flatbuff_bytes_to_sv(buffer->Data(), buffer->size()); -} + std::string_view flatbuff_str_to_sv(const flatbuffers::String *buffer) + { + return flatbuff_bytes_to_sv(buffer->Data(), buffer->size()); + } -/** + /** * Returns hash from Flat Buffer vector of bytes. */ -hasher::B2H flatbuff_bytes_to_hash(const flatbuffers::Vector *buffer) -{ - return *reinterpret_cast(buffer->data()); -} + hpfs::h32 flatbuff_bytes_to_hash(const flatbuffers::Vector *buffer) + { + return *reinterpret_cast(buffer->data()); + } -/** + /** * Returns set from Flatbuffer vector of ByteArrays. */ -const std::set flatbuf_bytearrayvector_to_stringlist(const flatbuffers::Vector> *fbvec) -{ - std::set set; - for (auto el : *fbvec) - set.emplace(std::string(flatbuff_bytes_to_sv(el->array()))); - return set; -} + const std::set flatbuf_bytearrayvector_to_stringlist(const flatbuffers::Vector> *fbvec) + { + std::set set; + for (auto el : *fbvec) + set.emplace(std::string(flatbuff_bytes_to_sv(el->array()))); + return set; + } -/** + /** * Returns a map from Flatbuffer vector of key value pairs. */ -const std::unordered_map -flatbuf_pairvector_to_stringmap(const flatbuffers::Vector> *fbvec) -{ - std::unordered_map map; - map.reserve(fbvec->size()); - for (auto el : *fbvec) - map.emplace(flatbuff_bytes_to_sv(el->key()), flatbuff_bytes_to_sv(el->value())); - return map; -} + const std::unordered_map + flatbuf_pairvector_to_stringmap(const flatbuffers::Vector> *fbvec) + { + std::unordered_map map; + map.reserve(fbvec->size()); + for (auto el : *fbvec) + map.emplace(flatbuff_bytes_to_sv(el->key()), flatbuff_bytes_to_sv(el->value())); + return map; + } -//---Conversion helpers from std data types to flatbuffers data types---// -//---These are used in constructing Flatbuffer messages using builders---// + //---Conversion helpers from std data types to flatbuffers data types---// + //---These are used in constructing Flatbuffer messages using builders---// -/** + /** * Returns Flatbuffer bytes vector from string_view. */ -const flatbuffers::Offset> -sv_to_flatbuff_bytes(flatbuffers::FlatBufferBuilder &builder, std::string_view sv) -{ - return builder.CreateVector(reinterpret_cast(sv.data()), sv.size()); -} + const flatbuffers::Offset> + sv_to_flatbuff_bytes(flatbuffers::FlatBufferBuilder &builder, std::string_view sv) + { + return builder.CreateVector(reinterpret_cast(sv.data()), sv.size()); + } -/** + /** * Returns Flatbuffer string from string_view. */ -const flatbuffers::Offset -sv_to_flatbuff_str(flatbuffers::FlatBufferBuilder &builder, std::string_view sv) -{ - return builder.CreateString(sv); -} + const flatbuffers::Offset + sv_to_flatbuff_str(flatbuffers::FlatBufferBuilder &builder, std::string_view sv) + { + return builder.CreateString(sv); + } -/** + /** * Returns Flatbuffer bytes vector from hash. */ -const flatbuffers::Offset> -hash_to_flatbuff_bytes(flatbuffers::FlatBufferBuilder &builder, const hasher::B2H hash) -{ - return builder.CreateVector(reinterpret_cast(&hash), hasher::HASH_SIZE); -} + const flatbuffers::Offset> + hash_to_flatbuff_bytes(flatbuffers::FlatBufferBuilder &builder, const hpfs::h32 hash) + { + return builder.CreateVector(reinterpret_cast(&hash), sizeof(hpfs::h32)); + } -/** + /** * Returns Flatbuffer vector of ByteArrays from given set of strings. */ -const flatbuffers::Offset>> -stringlist_to_flatbuf_bytearrayvector(flatbuffers::FlatBufferBuilder &builder, const std::set &set) -{ - std::vector> fbvec; - fbvec.reserve(set.size()); - for (std::string_view str : set) - fbvec.push_back(CreateByteArray(builder, sv_to_flatbuff_bytes(builder, str))); - return builder.CreateVector(fbvec); -} + const flatbuffers::Offset>> + stringlist_to_flatbuf_bytearrayvector(flatbuffers::FlatBufferBuilder &builder, const std::set &set) + { + std::vector> fbvec; + fbvec.reserve(set.size()); + for (std::string_view str : set) + fbvec.push_back(CreateByteArray(builder, sv_to_flatbuff_bytes(builder, str))); + return builder.CreateVector(fbvec); + } -/** + /** * Returns Flatbuffer vector of key value pairs from given map. */ -const flatbuffers::Offset>> -stringmap_to_flatbuf_bytepairvector(flatbuffers::FlatBufferBuilder &builder, const std::unordered_map &map) -{ - std::vector> fbvec; - fbvec.reserve(map.size()); - for (auto const &[key, value] : map) + const flatbuffers::Offset>> + stringmap_to_flatbuf_bytepairvector(flatbuffers::FlatBufferBuilder &builder, const std::unordered_map &map) { - fbvec.push_back(CreateBytesKeyValuePair( - builder, - sv_to_flatbuff_bytes(builder, key), - sv_to_flatbuff_bytes(builder, value))); + std::vector> fbvec; + fbvec.reserve(map.size()); + for (auto const &[key, value] : map) + { + fbvec.push_back(CreateBytesKeyValuePair( + builder, + sv_to_flatbuff_bytes(builder, key), + sv_to_flatbuff_bytes(builder, value))); + } + return builder.CreateVector(fbvec); } - return builder.CreateVector(fbvec); -} } // namespace fbschema \ No newline at end of file diff --git a/src/fbschema/common_helpers.hpp b/src/fbschema/common_helpers.hpp index dd305cd5..1505f6c3 100644 --- a/src/fbschema/common_helpers.hpp +++ b/src/fbschema/common_helpers.hpp @@ -2,8 +2,8 @@ #define _HP_FBSCHEMA_COMMON_HELPERS_ #include "../pchheader.hpp" +#include "../hpfs/h32.hpp" #include "common_schema_generated.h" -#include "../statefs/hasher.hpp" namespace fbschema { @@ -19,7 +19,7 @@ std::string_view flatbuff_bytes_to_sv(const flatbuffers::Vector *buffer std::string_view flatbuff_str_to_sv(const flatbuffers::String *buffer); -hasher::B2H flatbuff_bytes_to_hash(const flatbuffers::Vector *buffer); +hpfs::h32 flatbuff_bytes_to_hash(const flatbuffers::Vector *buffer); const std::set flatbuf_bytearrayvector_to_stringlist(const flatbuffers::Vector> *fbvec); @@ -36,7 +36,7 @@ const flatbuffers::Offset sv_to_flatbuff_str(flatbuffers::FlatBufferBuilder &builder, std::string_view sv); const flatbuffers::Offset> -hash_to_flatbuff_bytes(flatbuffers::FlatBufferBuilder &builder, hasher::B2H hash); +hash_to_flatbuff_bytes(flatbuffers::FlatBufferBuilder &builder, hpfs::h32 hash); const flatbuffers::Offset>> stringlist_to_flatbuf_bytearrayvector(flatbuffers::FlatBufferBuilder &builder, const std::set &set); diff --git a/src/fbschema/ledger_helpers.cpp b/src/fbschema/ledger_helpers.cpp index 700a73a6..90f814c3 100644 --- a/src/fbschema/ledger_helpers.cpp +++ b/src/fbschema/ledger_helpers.cpp @@ -19,7 +19,7 @@ const std::string_view create_ledger_from_proposal(flatbuffers::FlatBufferBuilde seq_no, p.time, sv_to_flatbuff_bytes(builder, p.lcl), - sv_to_flatbuff_bytes(builder, p.curr_hash_state), + sv_to_flatbuff_bytes(builder, p.curr_state_hash.to_string_view()), stringlist_to_flatbuf_bytearrayvector(builder, p.users), stringlist_to_flatbuf_bytearrayvector(builder, p.hash_inputs), stringlist_to_flatbuf_bytearrayvector(builder, p.hash_outputs)); diff --git a/src/fbschema/p2pmsg_helpers.cpp b/src/fbschema/p2pmsg_helpers.cpp index 358eea04..765fd1a8 100644 --- a/src/fbschema/p2pmsg_helpers.cpp +++ b/src/fbschema/p2pmsg_helpers.cpp @@ -215,7 +215,7 @@ const p2p::proposal create_proposal_from_msg(const Proposal_Message &msg, const p.time = msg.time(); p.stage = msg.stage(); p.lcl = flatbuff_bytes_to_sv(lcl); - p.curr_hash_state = flatbuff_bytes_to_sv(msg.curr_state_hash()); + p.curr_state_hash = flatbuff_bytes_to_sv(msg.curr_state_hash()); if (msg.users()) p.users = flatbuf_bytearrayvector_to_stringlist(msg.users()); @@ -365,7 +365,7 @@ void create_msg_from_proposal(flatbuffers::FlatBufferBuilder &container_builder, stringlist_to_flatbuf_bytearrayvector(builder, p.users), stringlist_to_flatbuf_bytearrayvector(builder, p.hash_inputs), stringlist_to_flatbuf_bytearrayvector(builder, p.hash_outputs), - sv_to_flatbuff_bytes(builder, p.curr_hash_state)); + sv_to_flatbuff_bytes(builder, p.curr_state_hash.to_string_view())); const flatbuffers::Offset message = CreateContent(builder, Message_Proposal_Message, proposal.Union()); builder.Finish(message); // Finished building message content to get serialised content. @@ -477,7 +477,7 @@ void create_msg_from_state_request(flatbuffers::FlatBufferBuilder &container_bui * @param expected_hash The exptected hash of the requested path. * @param lcl Lcl to be include in the container msg. */ -void create_msg_from_fsentry_response(flatbuffers::FlatBufferBuilder &container_builder, const std::string_view path, std::unordered_map &fs_entries, hasher::B2H expected_hash, std::string_view lcl) +void create_msg_from_fsentry_response(flatbuffers::FlatBufferBuilder &container_builder, const std::string_view path, std::unordered_map &fs_entries, hpfs::h32 expected_hash, std::string_view lcl) { flatbuffers::FlatBufferBuilder builder(1024); @@ -507,7 +507,7 @@ void create_msg_from_fsentry_response(flatbuffers::FlatBufferBuilder &container_ * @param hashmap Hashmap of the file * @param lcl Lcl to be include in the container msg. */ -void create_msg_from_filehashmap_response(flatbuffers::FlatBufferBuilder &container_builder, std::string_view path, std::vector &hashmap, std::size_t file_length, hasher::B2H expected_hash, std::string_view lcl) +void create_msg_from_filehashmap_response(flatbuffers::FlatBufferBuilder &container_builder, std::string_view path, std::vector &hashmap, std::size_t file_length, hpfs::h32 expected_hash, std::string_view lcl) { // todo:get a average propsal message size and allocate content builder based on that. flatbuffers::FlatBufferBuilder builder(1024); diff --git a/src/fbschema/p2pmsg_helpers.hpp b/src/fbschema/p2pmsg_helpers.hpp index 45986a18..9e789418 100644 --- a/src/fbschema/p2pmsg_helpers.hpp +++ b/src/fbschema/p2pmsg_helpers.hpp @@ -5,7 +5,7 @@ #include "p2pmsg_container_generated.h" #include "p2pmsg_content_generated.h" #include "../p2p/p2p.hpp" -#include "../statefs/hasher.hpp" +#include "../hpfs/h32.hpp" namespace fbschema::p2pmsg { @@ -57,9 +57,9 @@ void create_msg_from_npl_output(flatbuffers::FlatBufferBuilder &container_builde void create_msg_from_state_request(flatbuffers::FlatBufferBuilder &container_builder, const p2p::state_request &hr, std::string_view lcl); void create_msg_from_fsentry_response(flatbuffers::FlatBufferBuilder &container_builder, const std::string_view path, - std::unordered_map &fs_entries, hasher::B2H expected_hash, std::string_view lcl); + std::unordered_map &fs_entries, hpfs::h32 expected_hash, std::string_view lcl); -void create_msg_from_filehashmap_response(flatbuffers::FlatBufferBuilder &container_builder, std::string_view path, std::vector &hashmap, std::size_t file_length, hasher::B2H expected_hash, std::string_view lcl); +void create_msg_from_filehashmap_response(flatbuffers::FlatBufferBuilder &container_builder, std::string_view path, std::vector &hashmap, std::size_t file_length, hpfs::h32 expected_hash, std::string_view lcl); void create_msg_from_block_response(flatbuffers::FlatBufferBuilder &container_builder, p2p::block_response &block_resp, std::string_view lcl); diff --git a/src/hpfs/h32.cpp b/src/hpfs/h32.cpp new file mode 100644 index 00000000..78c07e1c --- /dev/null +++ b/src/hpfs/h32.cpp @@ -0,0 +1,74 @@ +#include "h32.hpp" + +/** + * Based on https://github.com/codetsunami/file-ptracer/blob/master/merkle.cpp + */ +namespace hpfs +{ + /** + * Helper functions for working with 32 byte hash type h32. + */ + + h32 h32_empty; + + bool h32::operator==(const h32 rhs) const + { + return this->data[0] == rhs.data[0] && this->data[1] == rhs.data[1] && this->data[2] == rhs.data[2] && this->data[3] == rhs.data[3]; + } + + bool h32::operator!=(const h32 rhs) const + { + return this->data[0] != rhs.data[0] || this->data[1] != rhs.data[1] || this->data[2] != rhs.data[2] || this->data[3] != rhs.data[3]; + } + + void h32::operator^=(const h32 rhs) + { + this->data[0] ^= rhs.data[0]; + this->data[1] ^= rhs.data[1]; + this->data[2] ^= rhs.data[2]; + this->data[3] ^= rhs.data[3]; + } + + std::string_view h32::to_string_view() const + { + return std::string_view(reinterpret_cast(this), sizeof(hpfs::h32)); + } + + h32 &h32::operator=(std::string_view sv) + { + memcpy(this->data, sv.data(), sizeof(h32)); + return *this; + } + + // Comparison operator for std::map key support. + bool h32::operator<(const h32 rhs) const + { + // Here we do not actually care about true comparison value. + // We just need the comparison to return consistent result based on + // a fixed criteria. + return this->data[0] < rhs.data[0]; + } + + std::ostream &operator<<(std::ostream &output, const h32 &h) + { + const uint8_t *buf = reinterpret_cast(&h); + for (int i = 0; i < 8; i++) + output << std::hex << std::setfill('0') << std::setw(2) << (int)buf[i]; + + return output; + } + + // Helper class to support std::map/std::unordered_map custom hashing function. + // This is needed to use B2H as the std map container key. + size_t h32_std_key_hasher::operator()(const h32 h) const + { + // Compute individual hash values. http://stackoverflow.com/a/1646913/126995 + size_t res = 17; + res = res * 31 + std::hash()(h.data[0]); + res = res * 31 + std::hash()(h.data[1]); + res = res * 31 + std::hash()(h.data[2]); + res = res * 31 + std::hash()(h.data[3]); + return res; + } + +} // namespace hpfs \ No newline at end of file diff --git a/src/hpfs/h32.hpp b/src/hpfs/h32.hpp new file mode 100644 index 00000000..8a75edaf --- /dev/null +++ b/src/hpfs/h32.hpp @@ -0,0 +1,36 @@ +#ifndef _HP_HPFS_H32_ +#define _HP_HPFS_H32_ + +#include "../pchheader.hpp" + +namespace hpfs +{ + + // blake2b hash is 32 bytes which we store as 4 quad words + // Originally from https://github.com/codetsunami/file-ptracer/blob/master/merkle.cpp + struct h32 + { + uint64_t data[4]; + + bool operator==(const h32 rhs) const; + bool operator!=(const h32 rhs) const; + void operator^=(const h32 rhs); + std::string_view to_string_view() const; + h32 &operator=(std::string_view sv); + bool operator<(const h32 rhs) const; + }; + extern h32 h32_empty; + + std::ostream &operator<<(std::ostream &output, const h32 &h); + + // Helper class to support std::map/std::unordered_map custom hashing function. + // This is needed to use B2H as the std map container key. + class h32_std_key_hasher + { + public: + size_t operator()(const h32 h) const; + }; + +} // namespace hpfs + +#endif \ No newline at end of file diff --git a/src/hpfs/hpfs.cpp b/src/hpfs/hpfs.cpp new file mode 100644 index 00000000..20752804 --- /dev/null +++ b/src/hpfs/hpfs.cpp @@ -0,0 +1,178 @@ +#include "hpfs.hpp" +#include "h32.hpp" +#include "../conf.hpp" +#include "../hplog.hpp" +#include "../util.hpp" + +namespace hpfs +{ + pid_t merge_pid = 0; + + bool init_success = false; + + int init() + { + LOG_INFO << "Starting hpfs merge process..."; + if (start_merge_process() == -1) + return -1; + + LOG_INFO << "Started hpfs merge process. pid:" << merge_pid; + init_success = true; + return 0; + } + + void deinit() + { + if (init_success) + { + LOG_INFO << "Stopping hpfs merge process... pid:" << merge_pid; + if (merge_pid > 0 && util::kill_process(merge_pid, true) == 0) + LOG_INFO << "Stopped hpfs merge process."; + } + } + + int start_merge_process() + { + const pid_t pid = fork(); + + if (pid > 0) + { + // HotPocket process. + // Check if process is still running. + util::sleep(20); + if (kill(pid, 0) == -1) + return -1; + + merge_pid = pid; + } + else if (pid == 0) + { + // hpfs process. + // Fill process args. + char *execv_args[] = { + conf::ctx.hpfs_exe_path.data(), + (char *)"merge", + conf::ctx.state_dir.data(), + NULL}; + + const int ret = execv(execv_args[0], execv_args); + LOG_ERR << errno << ": hpfs merge process execv failed."; + exit(1); + } + else + { + LOG_ERR << errno << ": fork() failed when starting hpfs merge process."; + return -1; + } + + return 0; + } + + int start_fs_session(pid_t &session_pid, std::string &mount_dir, + const char *mode, const bool hash_map_enabled) + { + const pid_t pid = fork(); + + if (pid > 0) + { + // HotPocket process. + + // If the mound dir is not specified, assign a mount dir based on hpfs process id. + if (mount_dir.empty()) + mount_dir = std::string(conf::ctx.state_dir) + .append("/") + .append(std::to_string(pid)); + + // Wait until hpfs is initialized properly. + bool hpfs_initialized = false; + uint8_t retry_count = 0; + do + { + util::sleep(20); + + // Check if process is still running. + if (kill(pid, 0) == -1) + break; + + // If hpfs is initialized, the inode no. of the mounted root dir is always 1. + struct stat st; + hpfs_initialized = (stat(mount_dir.c_str(), &st) == 0 && st.st_ino == 1); + + } while (!hpfs_initialized && ++retry_count < 100); + + // Kill the process if hpfs couldn't be initialized after the wait period. + if (!hpfs_initialized) + { + LOG_ERR << "Couldn't initialize hpfs session."; + util::kill_process(pid, true); + return -1; + } + + session_pid = pid; + } + else if (pid == 0) + { + // hpfs process. + + // If the mound dir is not specified, assign a mount dir based on hpfs process id. + const pid_t self_pid = getpid(); + if (mount_dir.empty()) + mount_dir = std::string(conf::ctx.state_dir) + .append("/") + .append(std::to_string(self_pid)); + + // Fill process args. + char *execv_args[] = { + conf::ctx.hpfs_exe_path.data(), + (char *)mode, // hpfs mode: rw | ro + conf::ctx.state_dir.data(), + mount_dir.data(), + (char *)(hash_map_enabled ? "hmap=true" : "hmap-false"), + NULL}; + + const int ret = execv(execv_args[0], execv_args); + LOG_ERR << errno << ": hpfs session process execv failed."; + exit(1); + } + else + { + LOG_ERR << errno << ": fork() failed when starting hpfs session process."; + return -1; + } + + return 0; + } + + int get_root_hash(h32 &hash) + { + pid_t pid; + std::string mount_dir; + if (start_fs_session(pid, mount_dir, "ro", true) == -1) + return -1; + + int res = get_hash(hash, mount_dir, "/"); + util::kill_process(pid, true); + + return res; + } + + int get_hash(h32 &hash, const std::string_view mount_dir, const std::string_view vpath) + { + std::string path = std::string(mount_dir).append(vpath).append("::hpfs.hmap.hash"); + int fd = open(path.c_str(), O_RDONLY); + if (fd == -1) + { + LOG_ERR << errno << ": Error opening hash file."; + return -1; + } + int res = read(fd, &hash, sizeof(h32)); + close(fd); + if (res == -1) + { + LOG_ERR << errno << ": Error reading hash file."; + return -1; + } + return 0; + } + +} // namespace hpfs \ No newline at end of file diff --git a/src/hpfs/hpfs.hpp b/src/hpfs/hpfs.hpp new file mode 100644 index 00000000..41361b84 --- /dev/null +++ b/src/hpfs/hpfs.hpp @@ -0,0 +1,18 @@ +#ifndef _HP_HPFS_HPFS_ +#define _HP_HPFS_HPFS_ + +#include "../pchheader.hpp" +#include "h32.hpp" + +namespace hpfs +{ + int init(); + void deinit(); + int start_merge_process(); + int start_fs_session(pid_t &session_pid, std::string &mount_dir, + const char *mode, const bool hash_map_enabled); + int get_root_hash(h32 &hash); + int get_hash(h32 &hash, const std::string_view mount_dir, const std::string_view vpath); +} // namespace hpfs + +#endif \ No newline at end of file diff --git a/src/main.cpp b/src/main.cpp index 58f0e463..bc718766 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -11,7 +11,7 @@ #include "usr/usr.hpp" #include "p2p/p2p.hpp" #include "cons/cons.hpp" -#include "statefs/state_common.hpp" +#include "hpfs/hpfs.hpp" /** * Parses CLI args and extracts hot pocket command and parameters given. @@ -68,6 +68,7 @@ void deinit() p2p::deinit(); cons::deinit(); proc::deinit(); + hpfs::deinit(); hplog::deinit(); } @@ -82,18 +83,18 @@ void signal_handler(int signum) namespace boost { -inline void assertion_failed_msg(char const *expr, char const *msg, char const *function, char const * /*file*/, long /*line*/) -{ - LOG_ERR << "Expression '" << expr << "' is false in function '" << function << "': " << (msg ? msg : "<...>") << ".\n" - << "Backtrace:\n" - << boost::stacktrace::stacktrace() << '\n'; - std::abort(); -} + inline void assertion_failed_msg(char const *expr, char const *msg, char const *function, char const * /*file*/, long /*line*/) + { + LOG_ERR << "Expression '" << expr << "' is false in function '" << function << "': " << (msg ? msg : "<...>") << ".\n" + << "Backtrace:\n" + << boost::stacktrace::stacktrace() << '\n'; + std::abort(); + } -inline void assertion_failed(char const *expr, char const *function, char const *file, long line) -{ - ::boost::assertion_failed_msg(expr, 0 /*nullptr*/, function, file, line); -} + inline void assertion_failed(char const *expr, char const *function, char const *file, long line) + { + ::boost::assertion_failed_msg(expr, 0 /*nullptr*/, function, file, line); + } } // namespace boost /** @@ -187,18 +188,22 @@ int main(int argc, char **argv) LOG_INFO << "Operating mode: " << (conf::cfg.startup_mode == conf::OPERATING_MODE::OBSERVER ? "Observer" : "Proposer"); - statefs::init(conf::ctx.state_hist_dir); - - if (p2p::init() != 0 || usr::init() != 0 || cons::init() != 0) + if (hpfs::init() != 0 || p2p::init() != 0 || usr::init() != 0 || cons::init() != 0) + { + deinit(); return -1; + } // After initializing primary subsystems, register the SIGINT handler. signal(SIGINT, signal_handler); - while (true) - cons::consensus(); + if (cons::run_consensus() == -1) + { + LOG_ERR << "Error occured in consensus."; + deinit(); + return -1; + } - // Free resources. deinit(); } } diff --git a/src/p2p/p2p.cpp b/src/p2p/p2p.cpp index dc349a76..e11af096 100644 --- a/src/p2p/p2p.cpp +++ b/src/p2p/p2p.cpp @@ -11,213 +11,220 @@ namespace p2p { -// Holds global connected-peers and related objects. -connected_context ctx; + // Holds global connected-peers and related objects. + connected_context ctx; -/** + bool init_success = false; + + /** * Initializes the p2p subsystem. Must be called once during application startup. * @return 0 for successful initialization. -1 for failure. */ -int init() -{ - //Entry point for p2p which will start peer connections to other nodes - return start_peer_connections(); -} + int init() + { + //Entry point for p2p which will start peer connections to other nodes + if (start_peer_connections() == -1) + return -1; -/** + init_success = true; + return 0; + } + + /** * Cleanup any running processes. */ -void deinit() -{ - ctx.listener.stop(); -} - -int start_peer_connections() -{ - const uint64_t metric_thresholds[] = {conf::cfg.peermaxcpm, conf::cfg.peermaxdupmpm, conf::cfg.peermaxbadsigpm, conf::cfg.peermaxbadmpm}; - if (ctx.listener.start( - conf::cfg.peerport, ".sock-peer", comm::SESSION_TYPE::PEER, true, false, metric_thresholds, conf::cfg.peers, conf::cfg.peermaxsize) == -1) - return -1; - - LOG_INFO << "Started listening for peer connections on " << std::to_string(conf::cfg.peerport); - return 0; -} - -int resolve_peer_challenge(comm::comm_session &session, const peer_challenge_response &challenge_resp) -{ - - // Compare the response challenge string with the original issued challenge. - if (session.issued_challenge != challenge_resp.challenge) + void deinit() { - LOG_DBG << "Peer challenge response, challenge invalid."; - return -1; + if (init_success) + ctx.listener.stop(); } - // Verify the challenge signature. - if (crypto::verify( - challenge_resp.challenge, - challenge_resp.signature, - challenge_resp.pubkey) != 0) + int start_peer_connections() { - LOG_DBG << "Peer challenge response signature verification failed."; - return -1; - } + const uint64_t metric_thresholds[] = {conf::cfg.peermaxcpm, conf::cfg.peermaxdupmpm, conf::cfg.peermaxbadsigpm, conf::cfg.peermaxbadmpm}; + if (ctx.listener.start( + conf::cfg.peerport, ".sock-peer", comm::SESSION_TYPE::PEER, true, false, metric_thresholds, conf::cfg.peers, conf::cfg.peermaxsize) == -1) + return -1; - // Converting the binary pub key into hexa decimal string this will be used as the key in storing peer sessions - std::string pubkeyhex; - util::bin2hex(pubkeyhex, reinterpret_cast(challenge_resp.pubkey.data()), challenge_resp.pubkey.length()); - - const int res = challenge_resp.pubkey.compare(conf::cfg.pubkey); - - // If pub key is same as our (self) pub key, then this is the loopback connection to ourselves. - // Hence we must keep the connection but only one of two sessions must be added to peer_connections. - // If pub key is greater than our id (< 0), then we should give priority to any existing inbound connection - // from the same peer and drop the outbound connection. - // If pub key is lower than our id (> 0), then we should give priority to any existing outbound connection - // from the same peer and drop the inbound connection. - - std::lock_guard lock(ctx.peer_connections_mutex); - - const auto iter = p2p::ctx.peer_connections.find(pubkeyhex); - if (iter == p2p::ctx.peer_connections.end()) - { - // Add the new connection straight away, if we haven't seen it before. - session.uniqueid.swap(pubkeyhex); - session.challenge_status = comm::CHALLENGE_VERIFIED; - p2p::ctx.peer_connections.try_emplace(session.uniqueid, &session); + LOG_INFO << "Started listening for peer connections on " << std::to_string(conf::cfg.peerport); return 0; } - else if (res == 0) // New connection is self (There can be two sessions for self (inbound/outbound)) + + int resolve_peer_challenge(comm::comm_session &session, const peer_challenge_response &challenge_resp) { - session.is_self = true; - session.uniqueid.swap(pubkeyhex); - session.challenge_status = comm::CHALLENGE_VERIFIED; - return 0; - } - else // New connection is not self but with same pub key. - { - comm::comm_session &ex_session = *iter->second; - // We don't allow duplicate connections to the same peer to same direction. - if (ex_session.is_inbound != session.is_inbound) + + // Compare the response challenge string with the original issued challenge. + if (session.issued_challenge != challenge_resp.challenge) { - // Decide whether we need to replace existing session with new session. - const bool replace_needed = ((res < 0 && !ex_session.is_inbound) || (res > 0 && ex_session.is_inbound)); - if (replace_needed) - { - // If we happen to replace a peer session with known IP, transfer required details to the new session. - if (session.known_ipport.first.empty()) - session.known_ipport.swap(ex_session.known_ipport); - session.uniqueid.swap(pubkeyhex); - session.challenge_status = comm::CHALLENGE_VERIFIED; - - ex_session.close(false); - p2p::ctx.peer_connections.erase(iter); // remove existing session. - p2p::ctx.peer_connections.try_emplace(session.uniqueid, &session); // add new session. - - LOG_DBG << "Replacing existing connection [" << session.uniqueid << "]"; - return 0; - } - else if (ex_session.known_ipport.first.empty() || !session.known_ipport.first.empty()) - { - // If we have any known ip-port info from the new session, transfer them to the existing session. - ex_session.known_ipport.swap(session.known_ipport); - } + LOG_DBG << "Peer challenge response, challenge invalid."; + return -1; } - // Reaching this point means we don't need the new session. - LOG_DBG << "Rejecting new peer connection because existing connection takes priority [" << pubkeyhex << "]"; - return -1; - } -} + // Verify the challenge signature. + if (crypto::verify( + challenge_resp.challenge, + challenge_resp.signature, + challenge_resp.pubkey) != 0) + { + LOG_DBG << "Peer challenge response signature verification failed."; + return -1; + } -/** + // Converting the binary pub key into hexa decimal string this will be used as the key in storing peer sessions + std::string pubkeyhex; + util::bin2hex(pubkeyhex, reinterpret_cast(challenge_resp.pubkey.data()), challenge_resp.pubkey.length()); + + const int res = challenge_resp.pubkey.compare(conf::cfg.pubkey); + + // If pub key is same as our (self) pub key, then this is the loopback connection to ourselves. + // Hence we must keep the connection but only one of two sessions must be added to peer_connections. + // If pub key is greater than our id (< 0), then we should give priority to any existing inbound connection + // from the same peer and drop the outbound connection. + // If pub key is lower than our id (> 0), then we should give priority to any existing outbound connection + // from the same peer and drop the inbound connection. + + std::lock_guard lock(ctx.peer_connections_mutex); + + const auto iter = p2p::ctx.peer_connections.find(pubkeyhex); + if (iter == p2p::ctx.peer_connections.end()) + { + // Add the new connection straight away, if we haven't seen it before. + session.uniqueid.swap(pubkeyhex); + session.challenge_status = comm::CHALLENGE_VERIFIED; + p2p::ctx.peer_connections.try_emplace(session.uniqueid, &session); + return 0; + } + else if (res == 0) // New connection is self (There can be two sessions for self (inbound/outbound)) + { + session.is_self = true; + session.uniqueid.swap(pubkeyhex); + session.challenge_status = comm::CHALLENGE_VERIFIED; + return 0; + } + else // New connection is not self but with same pub key. + { + comm::comm_session &ex_session = *iter->second; + // We don't allow duplicate connections to the same peer to same direction. + if (ex_session.is_inbound != session.is_inbound) + { + // Decide whether we need to replace existing session with new session. + const bool replace_needed = ((res < 0 && !ex_session.is_inbound) || (res > 0 && ex_session.is_inbound)); + if (replace_needed) + { + // If we happen to replace a peer session with known IP, transfer required details to the new session. + if (session.known_ipport.first.empty()) + session.known_ipport.swap(ex_session.known_ipport); + session.uniqueid.swap(pubkeyhex); + session.challenge_status = comm::CHALLENGE_VERIFIED; + + ex_session.close(false); + p2p::ctx.peer_connections.erase(iter); // remove existing session. + p2p::ctx.peer_connections.try_emplace(session.uniqueid, &session); // add new session. + + LOG_DBG << "Replacing existing connection [" << session.uniqueid << "]"; + return 0; + } + else if (ex_session.known_ipport.first.empty() || !session.known_ipport.first.empty()) + { + // If we have any known ip-port info from the new session, transfer them to the existing session. + ex_session.known_ipport.swap(session.known_ipport); + } + } + + // Reaching this point means we don't need the new session. + LOG_DBG << "Rejecting new peer connection because existing connection takes priority [" << pubkeyhex << "]"; + return -1; + } + } + + /** * Broadcasts the given message to all currently connected outbound peers. * @param msg Peer outbound message to be broadcasted. * @param send_to_self Whether to also send the message to self (this node). */ -void broadcast_message(const flatbuffers::FlatBufferBuilder &fbuf, const bool send_to_self) -{ - if (ctx.peer_connections.size() == 0) + void broadcast_message(const flatbuffers::FlatBufferBuilder &fbuf, const bool send_to_self) { - LOG_DBG << "No peers to broadcast (not even self). Waiting until at least one peer connects."; - while (ctx.peer_connections.size() == 0) - util::sleep(100); + if (ctx.peer_connections.size() == 0) + { + LOG_DBG << "No peers to broadcast (not even self). Waiting until at least one peer connects."; + while (ctx.peer_connections.size() == 0) + util::sleep(100); + } + + //Broadcast while locking the peer_connections. + std::lock_guard lock(ctx.peer_connections_mutex); + + for (const auto &[k, session] : ctx.peer_connections) + { + if (!send_to_self && session->is_self) + continue; + + std::string_view msg = std::string_view( + reinterpret_cast(fbuf.GetBufferPointer()), fbuf.GetSize()); + session->send(msg); + } } - //Broadcast while locking the peer_connections. - std::lock_guard lock(ctx.peer_connections_mutex); - - for (const auto &[k, session] : ctx.peer_connections) - { - if (!send_to_self && session->is_self) - continue; - - std::string_view msg = std::string_view( - reinterpret_cast(fbuf.GetBufferPointer()), fbuf.GetSize()); - session->send(msg); - } -} - -/** + /** * Sends the given message to self (this node). * @param msg Peer outbound message to be sent to self. */ -void send_message_to_self(const flatbuffers::FlatBufferBuilder &fbuf) -{ - //Send while locking the peer_connections. - std::lock_guard lock(p2p::ctx.peer_connections_mutex); - - // Find the peer session connected to self. - const auto peer_itr = ctx.peer_connections.find(conf::cfg.pubkeyhex); - if (peer_itr != ctx.peer_connections.end()) + void send_message_to_self(const flatbuffers::FlatBufferBuilder &fbuf) { - std::string_view msg = std::string_view( - reinterpret_cast(fbuf.GetBufferPointer()), fbuf.GetSize()); + //Send while locking the peer_connections. + std::lock_guard lock(p2p::ctx.peer_connections_mutex); - const comm::comm_session *session = peer_itr->second; - session->send(msg); - } -} - -/** - * Sends the given message to a random peer (except self). - * @param msg Peer outbound message to be sent to peer. - */ -void send_message_to_random_peer(const flatbuffers::FlatBufferBuilder &fbuf) -{ - //Send while locking the peer_connections. - std::lock_guard lock(p2p::ctx.peer_connections_mutex); - - const size_t connected_peers = ctx.peer_connections.size(); - if (connected_peers == 0) - { - LOG_DBG << "No peers to send (not even self)."; - return; - } - else if (connected_peers == 1 && ctx.peer_connections.begin()->second->is_self) - { - LOG_DBG << "Only self is connected."; - return; - } - - while (true) - { - // Initialize random number generator with current timestamp. - const int random_peer_index = (rand() % connected_peers); // select a random peer index. - auto it = ctx.peer_connections.begin(); - std::advance(it, random_peer_index); //move iterator to point to random selected peer. - - //send message to selected peer. - const comm::comm_session *session = it->second; - if (!session->is_self) // Exclude self peer. + // Find the peer session connected to self. + const auto peer_itr = ctx.peer_connections.find(conf::cfg.pubkeyhex); + if (peer_itr != ctx.peer_connections.end()) { std::string_view msg = std::string_view( reinterpret_cast(fbuf.GetBufferPointer()), fbuf.GetSize()); + const comm::comm_session *session = peer_itr->second; session->send(msg); - break; } } -} + + /** + * Sends the given message to a random peer (except self). + * @param msg Peer outbound message to be sent to peer. + */ + void send_message_to_random_peer(const flatbuffers::FlatBufferBuilder &fbuf) + { + //Send while locking the peer_connections. + std::lock_guard lock(p2p::ctx.peer_connections_mutex); + + const size_t connected_peers = ctx.peer_connections.size(); + if (connected_peers == 0) + { + LOG_DBG << "No peers to send (not even self)."; + return; + } + else if (connected_peers == 1 && ctx.peer_connections.begin()->second->is_self) + { + LOG_DBG << "Only self is connected."; + return; + } + + while (true) + { + // Initialize random number generator with current timestamp. + const int random_peer_index = (rand() % connected_peers); // select a random peer index. + auto it = ctx.peer_connections.begin(); + std::advance(it, random_peer_index); //move iterator to point to random selected peer. + + //send message to selected peer. + const comm::comm_session *session = it->second; + if (!session->is_self) // Exclude self peer. + { + std::string_view msg = std::string_view( + reinterpret_cast(fbuf.GetBufferPointer()), fbuf.GetSize()); + + session->send(msg); + break; + } + } + } } // namespace p2p \ No newline at end of file diff --git a/src/p2p/p2p.hpp b/src/p2p/p2p.hpp index f601d415..81ba3f09 100644 --- a/src/p2p/p2p.hpp +++ b/src/p2p/p2p.hpp @@ -7,7 +7,7 @@ #include "../comm/comm_session.hpp" #include "../usr/user_input.hpp" #include "peer_session_handler.hpp" -#include "../statefs/hasher.hpp" +#include "../hpfs/h32.hpp" #include "../conf.hpp" namespace p2p @@ -20,7 +20,7 @@ struct proposal uint64_t time; uint8_t stage; std::string lcl; - std::string curr_hash_state; + hpfs::h32 curr_state_hash; std::set users; std::set hash_inputs; std::set hash_outputs; @@ -75,14 +75,14 @@ struct state_request std::string parent_path; // The requested file or dir path. bool is_file; // Whether the path is a file or dir. int32_t block_id; // Block id of the file if we are requesting for file block. Otherwise -1. - hasher::B2H expected_hash; // The expected hash of the requested result. + hpfs::h32 expected_hash; // The expected hash of the requested result. }; // Represents state file system entry. struct state_fs_hash_entry { bool is_file; // Whether this is a file or dir. - hasher::B2H hash; // Hash of the file or dir. + hpfs::h32 hash; // Hash of the file or dir. }; // Represents a file block data resposne. @@ -91,7 +91,7 @@ struct block_response std::string path; // Path of the file. uint32_t block_id; // Id of the block where the data belongs to. std::string_view data; // The block data. - hasher::B2H hash; // Hash of the bloc data. + hpfs::h32 hash; // Hash of the bloc data. }; struct message_collection diff --git a/src/pchheader.hpp b/src/pchheader.hpp index 01090149..8926a262 100644 --- a/src/pchheader.hpp +++ b/src/pchheader.hpp @@ -28,6 +28,7 @@ #include #include #include +#include #include #include #include diff --git a/src/proc.cpp b/src/proc.cpp index 7c32889c..ba897e56 100644 --- a/src/proc.cpp +++ b/src/proc.cpp @@ -4,226 +4,178 @@ #include "fbschema/common_helpers.hpp" #include "fbschema/p2pmsg_container_generated.h" #include "fbschema/p2pmsg_content_generated.h" -#include "statefs/hasher.hpp" -#include "statefs/state_common.hpp" -#include "statefs/hashtree_builder.hpp" #include "proc.hpp" -#include "cons/cons.hpp" +#include "hpfs/hpfs.hpp" namespace proc { -// Enum used to differenciate pipe fds maintained for SC I/O pipes. -enum FDTYPE -{ - // Used by Smart Contract to read input sent by Hot Pocket - SCREAD = 0, - // Used by Hot Pocket to write input to the smart contract. - HPWRITE = 1, - // Used by Hot Pocket to read output from the smart contract. - HPREAD = 2, - // Used by Smart Contract to write output back to Hot Pocket. - SCWRITE = 3 -}; + // Enum used to differenciate pipe fds maintained for SC I/O pipes. + enum FDTYPE + { + // Used by Smart Contract to read input sent by Hot Pocket + SCREAD = 0, + // Used by Hot Pocket to write input to the smart contract. + HPWRITE = 1, + // Used by Hot Pocket to read output from the smart contract. + HPREAD = 2, + // Used by Smart Contract to write output back to Hot Pocket. + SCWRITE = 3 + }; -// Map of user pipe fds (map key: user public key) -contract_fdmap_t userfds; + // Map of user pipe fds (map key: user public key) + contract_fdmap_t userfds; -// Pipe fds for NPL <--> messages. -std::vector nplfds; + // Pipe fds for NPL <--> messages. + std::vector nplfds; -// Pipe fds for HP <--> messages. -std::vector hpscfds; + // Pipe fds for HP <--> messages. + std::vector hpscfds; -// Holds the contract process id (if currently executing). -pid_t contract_pid; + // Holds the contract process id (if currently executing). + pid_t contract_pid; -// Holds the state monitor process id (if currently executing). -pid_t statemon_pid; + // Holds the hpfs rw process id (if currently executing). + pid_t hpfs_pid; -const char *FINDMNT_COMMAND = "findmnt --noheadings "; + const char *FINDMNT_COMMAND = "findmnt --noheadings "; -/** + /** * Executes the contract process and passes the specified arguments. * @return 0 on successful process creation. -1 on failure or contract process is already running. */ -int exec_contract(const contract_exec_args &args) -{ - // Setup io pipes and feed all inputs to them. - create_iopipes_for_fdmap(userfds, args.userbufs); - create_iopipes(nplfds); - create_iopipes(hpscfds); - - if (feed_inputs(args) != 0) - return -1; - - // Start the state monitor before starting the contract process. - if (start_state_monitor() != 0) - return -1; - - const pid_t pid = fork(); - if (pid > 0) + int exec_contract(const contract_exec_args &args, hpfs::h32 &state_hash) { - // HotPocket process. - contract_pid = pid; + // Setup io pipes and feed all inputs to them. + create_iopipes_for_fdmap(userfds, args.userbufs); + create_iopipes(nplfds); + create_iopipes(hpscfds); - // Close all fds unused by HP process. - close_unused_fds(true); + if (feed_inputs(args) != 0) + return -1; - // Wait for child process (contract process) to complete execution. - const int presult = await_process_execution(contract_pid); - LOG_DBG << "Contract process ended."; + // Start the hpfs rw session before starting the contract process. + if (start_hpfs_rw_session() != 0) + return -1; - contract_pid = 0; - if (presult != 0) + const pid_t pid = fork(); + if (pid > 0) { - LOG_ERR << "Contract process exited with non-normal status code: " << presult; + // HotPocket process. + contract_pid = pid; + + // Close all fds unused by HP process. + close_unused_fds(true); + + // Wait for child process (contract process) to complete execution. + const int presult = await_process_execution(contract_pid); + LOG_DBG << "Contract process ended."; + + contract_pid = 0; + if (presult != 0) + { + LOG_ERR << "Contract process exited with non-normal status code: " << presult; + return -1; + } + + if (stop_hpfs_rw_session(state_hash) != 0) + return -1; + + // After contract execution, collect contract outputs. + if (fetch_outputs(args) != 0) + return -1; + } + else if (pid == 0) + { + // Contract process. + // Set up the process environment and overlay the contract binary program with execv(). + + // Close all fds unused by SC process. + close_unused_fds(false); + + // Write the contract input message from HotPocket to the stdin (0) of the contract process. + write_contract_args(args); + + LOG_DBG << "Starting contract process..."; + + const bool using_appbill = !conf::cfg.appbill.empty(); + int len = conf::cfg.runtime_binexec_args.size() + 1; + if (using_appbill) + len += conf::cfg.runtime_appbill_args.size(); + + // Fill process args. + char *execv_args[len]; + int j = 0; + if (using_appbill) + for (int i = 0; i < conf::cfg.runtime_appbill_args.size(); i++, j++) + execv_args[i] = conf::cfg.runtime_appbill_args[i].data(); + + for (int i = 0; i < conf::cfg.runtime_binexec_args.size(); i++, j++) + execv_args[j] = conf::cfg.runtime_binexec_args[i].data(); + execv_args[len - 1] = NULL; + + chdir(conf::ctx.state_rw_dir.c_str()); + + int ret = execv(execv_args[0], execv_args); + LOG_ERR << errno << ": Contract process execv failed."; + exit(1); + } + else + { + LOG_ERR << "fork() failed when starting contract process."; return -1; } - if (stop_state_monitor() != 0) - return -1; - - // After contract execution, collect contract outputs. - if (fetch_outputs(args) != 0) - return -1; - } - else if (pid == 0) - { - // Contract process. - // Set up the process environment and overlay the contract binary program with execv(). - - // Close all fds unused by SC process. - close_unused_fds(false); - - // Write the contract input message from HotPocket to the stdin (0) of the contract process. - write_contract_args(args); - - LOG_DBG << "Starting contract process..."; - - const bool using_appbill = !conf::cfg.appbill.empty(); - int len = conf::cfg.runtime_binexec_args.size() + 1; - if (using_appbill) - len += conf::cfg.runtime_appbill_args.size(); - - // Fill process args. - char *execv_args[len]; - int j = 0; - if (using_appbill) - for (int i = 0; i < conf::cfg.runtime_appbill_args.size(); i++, j++) - execv_args[i] = conf::cfg.runtime_appbill_args[i].data(); - - for (int i = 0; i < conf::cfg.runtime_binexec_args.size(); i++, j++) - execv_args[j] = conf::cfg.runtime_binexec_args[i].data(); - execv_args[len - 1] = NULL; - - int ret = execv(execv_args[0], execv_args); - LOG_ERR << errno << ": Contract process execv failed."; - exit(1); - } - else - { - LOG_ERR << "fork() failed when starting contract process."; - return -1; + return 0; } - return 0; -} - -/** - * Blocks the calling thread until the specified process compelted exeution (if running). + /** + * Blocks the calling thread until the specified process completed exeution (if running). * @return 0 if process exited normally or exit code of process if abnormally exited. */ -int await_process_execution(pid_t pid) -{ - if (pid > 0) + int await_process_execution(pid_t pid) { - int scstatus; - waitpid(pid, &scstatus, 0); - if (!WIFEXITED(scstatus)) - return WEXITSTATUS(scstatus); - } - return 0; -} - -/** - * Mounts the fuse file system at the contract state dir by starting the state monitor process. - * State monitor will automatically create a state history checkpoint as well. - */ -int start_state_monitor() -{ - pid_t pid = fork(); - if (pid > 0) - { - // HP process. - statemon_pid = pid; - - // Give enough time for the state monitor to start. - // We wait until Fuse filesystem is mounted for max number of retries. - uint16_t retry_count = 0; - std::string findmnt_command = FINDMNT_COMMAND + conf::ctx.state_dir; - while (retry_count < 50) + if (pid > 0) { - util::sleep(10); - int ret = system(findmnt_command.c_str()); - if (WEXITSTATUS(ret) == 0) // Success. Fuse fs has been mounted. - return 0; - retry_count++; + int scstatus; + waitpid(pid, &scstatus, 0); + if (!WIFEXITED(scstatus)) + return WEXITSTATUS(scstatus); } - - // We waited enough time for Fuse fs to be mounted but no luck. - return -1; + return 0; } - else if (pid == 0) - { - // State monitor process. - LOG_DBG << "Starting state monitor..."; - // Fill process args. - char *execv_args[4]; - execv_args[0] = conf::ctx.statemon_exe_path.data(); - execv_args[1] = conf::ctx.state_hist_dir.data(); - execv_args[2] = conf::ctx.state_dir.data(); - execv_args[3] = NULL; - - int ret = execv(execv_args[0], execv_args); - LOG_ERR << errno << ": State monitor execv failed."; - exit(1); - } - else if (pid < 0) - { - LOG_ERR << "fork() failed when starting state monitor."; - return -1; - } -} - -/** - * Terminate the state monitor and update the latest state hash tree. + /** + * Starts the hpfs read/write state filesystem. */ -int stop_state_monitor() -{ - kill(statemon_pid, SIGINT); + int start_hpfs_rw_session() + { + LOG_DBG << "Starting hpfs rw session..."; + if (hpfs::start_fs_session(hpfs_pid, conf::ctx.state_rw_dir, "rw", true) == -1) + return -1; - // Wait for state monitor process to complete execution after the SIGINT. - const int presult = await_process_execution(statemon_pid); - LOG_DBG << "State monitor stopped."; + LOG_DBG << "hpfs rw session started. pid:" << hpfs_pid; + } - statemon_pid = 0; + /** + * Stops the hpfs state filesystem. + */ + int stop_hpfs_rw_session(hpfs::h32 &state_hash) + { + // Read the root hash. + if (hpfs::get_hash(state_hash, conf::ctx.state_rw_dir, "/") == -1) + return -1; - if (presult != 0) - LOG_ERR << "State monitor process exited with non-normal status code: " << presult; + LOG_DBG << "Stopping hpfs rw session... pid:" << hpfs_pid; + if (util::kill_process(hpfs_pid, true) == -1) + return -1; - // Update the hash tree. - hasher::B2H statehash = hasher::B2H_empty; - statefs::hashtree_builder htree_builder(statefs::get_state_dir_context()); - if (htree_builder.generate(statehash) != 0) - return -1; + hpfs_pid = 0; + LOG_DBG << "hpfs rw session stopped."; + return 0; + } - cons::ctx.curr_hash_state = std::string(reinterpret_cast(&statehash), hasher::HASH_SIZE); - return 0; -} - -/** + /** * Writes the contract args (JSON) into the stdin of the contract process. * Args format: * { @@ -236,200 +188,200 @@ int stop_state_monitor() * "unl":[ "pkhex", ... ] * } */ -int write_contract_args(const contract_exec_args &args) -{ - // Populate the json string with contract args. - // We don't use a JSON parser here because it's lightweight to contrstuct the - // json string manually. - - std::ostringstream os; - os << "{\"version\":\"" << util::HP_VERSION - << "\",\"pubkey\":\"" << conf::cfg.pubkeyhex - << "\",\"ts\":" << args.timestamp - << ",\"hpfd\":[" << hpscfds[FDTYPE::SCREAD] << "," << hpscfds[FDTYPE::SCWRITE] - << "],\"usrfd\":{"; - - fdmap_json_to_stream(userfds, os); - - os << "},\"nplfd\":[" << nplfds[FDTYPE::SCREAD] << "," << nplfds[FDTYPE::SCWRITE] - << "],\"unl\":["; - - for (auto nodepk = conf::cfg.unl.begin(); nodepk != conf::cfg.unl.end(); nodepk++) + int write_contract_args(const contract_exec_args &args) { - if (nodepk != conf::cfg.unl.begin()) - os << ","; // Trailing comma separator for previous element. + // Populate the json string with contract args. + // We don't use a JSON parser here because it's lightweight to contrstuct the + // json string manually. - // Convert binary nodepk into hex. - std::string pubkeyhex; - util::bin2hex( - pubkeyhex, - reinterpret_cast((*nodepk).data()), - (*nodepk).length()); + std::ostringstream os; + os << "{\"version\":\"" << util::HP_VERSION + << "\",\"pubkey\":\"" << conf::cfg.pubkeyhex + << "\",\"ts\":" << args.timestamp + << ",\"hpfd\":[" << hpscfds[FDTYPE::SCREAD] << "," << hpscfds[FDTYPE::SCWRITE] + << "],\"usrfd\":{"; - os << "\"" << pubkeyhex << "\""; + fdmap_json_to_stream(userfds, os); + + os << "},\"nplfd\":[" << nplfds[FDTYPE::SCREAD] << "," << nplfds[FDTYPE::SCWRITE] + << "],\"unl\":["; + + for (auto nodepk = conf::cfg.unl.begin(); nodepk != conf::cfg.unl.end(); nodepk++) + { + if (nodepk != conf::cfg.unl.begin()) + os << ","; // Trailing comma separator for previous element. + + // Convert binary nodepk into hex. + std::string pubkeyhex; + util::bin2hex( + pubkeyhex, + reinterpret_cast((*nodepk).data()), + (*nodepk).length()); + + os << "\"" << pubkeyhex << "\""; + } + + os << "]}"; + + // Get the json string that should be written to contract input pipe. + const std::string json = os.str(); + + // Establish contract input pipe. + int stdinpipe[2]; + if (pipe(stdinpipe) != 0) + { + LOG_ERR << "Failed to create pipe to the contract process."; + return -1; + } + + // Redirect pipe read-end to the contract std input so the + // contract process can read from our pipe. + dup2(stdinpipe[0], STDIN_FILENO); + close(stdinpipe[0]); + + // Write the json message and close write fd. + if (write(stdinpipe[1], json.data(), json.size()) == -1) + { + LOG_ERR << "Failed to write to stdin of contract process."; + return -1; + } + close(stdinpipe[1]); + + return 0; } - os << "]}"; - - // Get the json string that should be written to contract input pipe. - const std::string json = os.str(); - - // Establish contract input pipe. - int stdinpipe[2]; - if (pipe(stdinpipe) != 0) + int feed_inputs(const contract_exec_args &args) { - LOG_ERR << "Failed to create pipe to the contract process."; - return -1; + // Write any hp or npl input messages to hp->sc and npl->sc pipe. + if (write_contract_hp_npl_inputs(args) != 0) + { + return -1; + } + + // Write any verified (consensus-reached) user inputs to user pipes. + if (write_contract_fdmap_inputs(userfds, args.userbufs) != 0) + { + cleanup_fdmap(userfds); + LOG_ERR << "Failed to write user inputs to contract."; + return -1; + } + + return 0; } - // Redirect pipe read-end to the contract std input so the - // contract process can read from our pipe. - dup2(stdinpipe[0], STDIN_FILENO); - close(stdinpipe[0]); - - // Write the json message and close write fd. - if (write(stdinpipe[1], json.data(), json.size()) == -1) + int fetch_outputs(const contract_exec_args &args) { - LOG_ERR << "Failed to write to stdin of contract process."; - return -1; - } - close(stdinpipe[1]); + if (read_contract_hp_npl_outputs(args) != 0) + { + return -1; + } - return 0; -} + if (read_contract_fdmap_outputs(userfds, args.userbufs) != 0) + { + LOG_ERR << "Error reading User output from the contract."; + return -1; + } -int feed_inputs(const contract_exec_args &args) -{ - // Write any hp or npl input messages to hp->sc and npl->sc pipe. - if (write_contract_hp_npl_inputs(args) != 0) - { - return -1; + nplfds.clear(); + userfds.clear(); + return 0; } - // Write any verified (consensus-reached) user inputs to user pipes. - if (write_contract_fdmap_inputs(userfds, args.userbufs) != 0) - { - cleanup_fdmap(userfds); - LOG_ERR << "Failed to write user inputs to contract."; - return -1; - } - - return 0; -} - -int fetch_outputs(const contract_exec_args &args) -{ - if (read_contract_hp_npl_outputs(args) != 0) - { - return -1; - } - - if (read_contract_fdmap_outputs(userfds, args.userbufs) != 0) - { - LOG_ERR << "Error reading User output from the contract."; - return -1; - } - - nplfds.clear(); - userfds.clear(); - return 0; -} - -/** + /** * Writes any hp input messages to the contract. */ -int write_contract_hp_npl_inputs(const contract_exec_args &args) -{ - if (write_iopipe(hpscfds, args.hpscbufs.inputs) != 0) + int write_contract_hp_npl_inputs(const contract_exec_args &args) { - LOG_ERR << "Error writing HP inputs to SC"; - return -1; + if (write_iopipe(hpscfds, args.hpscbufs.inputs) != 0) + { + LOG_ERR << "Error writing HP inputs to SC"; + return -1; + } + + if (write_npl_iopipe(nplfds, args.nplbuff.inputs) != 0) + { + LOG_ERR << "Error writing NPL inputs to SC"; + return -1; + } + + return 0; } - if (write_npl_iopipe(nplfds, args.nplbuff.inputs) != 0) - { - LOG_ERR << "Error writing NPL inputs to SC"; - return -1; - } - - return 0; -} - -/** + /** * Read all HP output messages produced by the contract process and store them in * the buffer for later processing. * * @return 0 on success. -1 on failure. */ -int read_contract_hp_npl_outputs(const contract_exec_args &args) -{ - // Clear the input buffers because we are sure the contract has finished reading from - // that mapped memory portion. - args.hpscbufs.inputs.clear(); - - if (read_iopipe(hpscfds, args.hpscbufs.output) != 0) // hpscbufs.second is the output buffer. + int read_contract_hp_npl_outputs(const contract_exec_args &args) { - LOG_ERR << "Error reading HP output from the contract."; - return -1; + // Clear the input buffers because we are sure the contract has finished reading from + // that mapped memory portion. + args.hpscbufs.inputs.clear(); + + if (read_iopipe(hpscfds, args.hpscbufs.output) != 0) // hpscbufs.second is the output buffer. + { + LOG_ERR << "Error reading HP output from the contract."; + return -1; + } + + if (read_iopipe(nplfds, args.nplbuff.output) != 0) // hpscbufs.second is the output buffer. + { + LOG_ERR << "Error reading NPL output from the contract."; + return -1; + } + + return 0; } - if (read_iopipe(nplfds, args.nplbuff.output) != 0) // hpscbufs.second is the output buffer. - { - LOG_ERR << "Error reading NPL output from the contract."; - return -1; - } - - return 0; -} - -/** + /** * Common helper function to write json output of fdmap to given ostream. * @param fdmap Any pubkey->fdlist map. (eg. userfds, nplfds) * @param os An output stream. */ -void fdmap_json_to_stream(const contract_fdmap_t &fdmap, std::ostringstream &os) -{ - for (auto itr = fdmap.begin(); itr != fdmap.end(); itr++) + void fdmap_json_to_stream(const contract_fdmap_t &fdmap, std::ostringstream &os) { - if (itr != fdmap.begin()) - os << ","; // Trailing comma separator for previous element. + for (auto itr = fdmap.begin(); itr != fdmap.end(); itr++) + { + if (itr != fdmap.begin()) + os << ","; // Trailing comma separator for previous element. - // Get the hex pubkey. - std::string_view pubkey = itr->first; // Pubkey in binary format. - std::string pubkeyhex; - util::bin2hex( - pubkeyhex, - reinterpret_cast(pubkey.data()), - pubkey.length()); + // Get the hex pubkey. + std::string_view pubkey = itr->first; // Pubkey in binary format. + std::string pubkeyhex; + util::bin2hex( + pubkeyhex, + reinterpret_cast(pubkey.data()), + pubkey.length()); - // Write hex pubkey and fds. - os << "\"" << pubkeyhex << "\":[" - << itr->second[FDTYPE::SCREAD] << "," - << itr->second[FDTYPE::SCWRITE] << "]"; + // Write hex pubkey and fds. + os << "\"" << pubkeyhex << "\":[" + << itr->second[FDTYPE::SCREAD] << "," + << itr->second[FDTYPE::SCWRITE] << "]"; + } } -} -/** + /** * Creates io pipes for all pubkeys specified in bufmap. * @param fdmap A map which has public key and a vector as fd list for that public key. * @param bufmap A map which has a public key and input/output buffer lists for that public key. * @return 0 on success. -1 on failure. */ -int create_iopipes_for_fdmap(contract_fdmap_t &fdmap, contract_bufmap_t &bufmap) -{ - for (auto &[pubkey, buflist] : bufmap) + int create_iopipes_for_fdmap(contract_fdmap_t &fdmap, contract_bufmap_t &bufmap) { - std::vector fds = std::vector(); - if (create_iopipes(fds) != 0) - return -1; + for (auto &[pubkey, buflist] : bufmap) + { + std::vector fds = std::vector(); + if (create_iopipes(fds) != 0) + return -1; - fdmap.emplace(pubkey, std::move(fds)); + fdmap.emplace(pubkey, std::move(fds)); + } + + return 0; } - return 0; -} - -/** + /** * Common function to create the pipes and write buffer inputs to the fdmap. * We take mutable parameters since the internal entries in the maps will be * modified (eg. fd close, buffer clear). @@ -438,19 +390,19 @@ int create_iopipes_for_fdmap(contract_fdmap_t &fdmap, contract_bufmap_t &bufmap) * @param bufmap A map which has a public key and input/output buffer lists for that public key. * @return 0 on success. -1 on failure. */ -int write_contract_fdmap_inputs(contract_fdmap_t &fdmap, contract_bufmap_t &bufmap) -{ - // Loop through input buffers for each pubkey. - for (auto &[pubkey, buflist] : bufmap) + int write_contract_fdmap_inputs(contract_fdmap_t &fdmap, contract_bufmap_t &bufmap) { - if (write_iopipe(fdmap[pubkey], buflist.inputs) != 0) - return -1; + // Loop through input buffers for each pubkey. + for (auto &[pubkey, buflist] : bufmap) + { + if (write_iopipe(fdmap[pubkey], buflist.inputs) != 0) + return -1; + } + + return 0; } - return 0; -} - -/** + /** * Common function to read all outputs produced by the contract process and store them in * output buffers for later processing. * @@ -458,274 +410,274 @@ int write_contract_fdmap_inputs(contract_fdmap_t &fdmap, contract_bufmap_t &bufm * @param bufmap A map which has a public key and input/output buffer pair for that public key. * @return 0 on success. -1 on failure. */ -int read_contract_fdmap_outputs(contract_fdmap_t &fdmap, contract_bufmap_t &bufmap) -{ - for (auto &[pubkey, bufpair] : bufmap) + int read_contract_fdmap_outputs(contract_fdmap_t &fdmap, contract_bufmap_t &bufmap) { - // Clear the input buffer because we are sure the contract has finished reading from - // the inputs' mapped memory portion. - bufpair.inputs.clear(); + for (auto &[pubkey, bufpair] : bufmap) + { + // Clear the input buffer because we are sure the contract has finished reading from + // the inputs' mapped memory portion. + bufpair.inputs.clear(); - // Get fds for the pubkey. - std::vector &fds = fdmap[pubkey]; + // Get fds for the pubkey. + std::vector &fds = fdmap[pubkey]; - if (read_iopipe(fds, bufpair.output) != 0) // bufpair.second is the output buffer. - return -1; + if (read_iopipe(fds, bufpair.output) != 0) // bufpair.second is the output buffer. + return -1; + } + + return 0; } - return 0; -} - -/** + /** * Common function to close any open fds in the map after an error. * @param fdmap Any pubkey->fdlist map. (eg. userfds, nplfds) */ -void cleanup_fdmap(contract_fdmap_t &fdmap) -{ - for (auto &[pubkey, fds] : fdmap) + void cleanup_fdmap(contract_fdmap_t &fdmap) { - for (int i = 0; i < 4; i++) + for (auto &[pubkey, fds] : fdmap) { - if (fds[i] > 0) - close(fds[i]); - fds[i] = 0; + for (int i = 0; i < 4; i++) + { + if (fds[i] > 0) + close(fds[i]); + fds[i] = 0; + } } } -} -/** + /** * Common function to create a pair of pipes (Hp->SC, SC->HP). * @param fds Vector to populate fd list. * @param inputbuffer Buffer to write into the HP write fd. */ -int create_iopipes(std::vector &fds) -{ - int inpipe[2]; - if (pipe(inpipe) != 0) - return -1; - - int outpipe[2]; - if (pipe(outpipe) != 0) + int create_iopipes(std::vector &fds) { - // Close the earlier created pipe. - close(inpipe[0]); - close(inpipe[1]); - return -1; + int inpipe[2]; + if (pipe(inpipe) != 0) + return -1; + + int outpipe[2]; + if (pipe(outpipe) != 0) + { + // Close the earlier created pipe. + close(inpipe[0]); + close(inpipe[1]); + return -1; + } + + // If both pipes got created, assign them to the fd vector. + fds.clear(); + fds.push_back(inpipe[0]); //SCREAD + fds.push_back(inpipe[1]); //HPWRITE + fds.push_back(outpipe[0]); //HPREAD + fds.push_back(outpipe[1]); //SCWRITE + + return 0; } - // If both pipes got created, assign them to the fd vector. - fds.clear(); - fds.push_back(inpipe[0]); //SCREAD - fds.push_back(inpipe[1]); //HPWRITE - fds.push_back(outpipe[0]); //HPREAD - fds.push_back(outpipe[1]); //SCWRITE - - return 0; -} - -/** + /** * Common function to write the given input buffer into the write fd from the HP side. * @param fds Vector of fd list. * @param inputs Buffer to write into the HP write fd. */ -int write_iopipe(std::vector &fds, std::list &inputs) -{ - // Write the inputs (if any) into the contract and close the writefd. - - const int writefd = fds[FDTYPE::HPWRITE]; - bool vmsplice_error = false; - - if (!inputs.empty()) + int write_iopipe(std::vector &fds, std::list &inputs) { - // Prepare the input memory segments to map with vmsplice. - size_t i = 0; - iovec memsegs[inputs.size()]; - for (std::string &input : inputs) + // Write the inputs (if any) into the contract and close the writefd. + + const int writefd = fds[FDTYPE::HPWRITE]; + bool vmsplice_error = false; + + if (!inputs.empty()) { - memsegs[i].iov_base = input.data(); - memsegs[i].iov_len = input.length(); - i++; + // Prepare the input memory segments to map with vmsplice. + size_t i = 0; + iovec memsegs[inputs.size()]; + for (std::string &input : inputs) + { + memsegs[i].iov_base = input.data(); + memsegs[i].iov_len = input.length(); + i++; + } + + // We use vmsplice to map (zero-copy) the inputs into the fd. + if (vmsplice(writefd, memsegs, inputs.size(), 0) == -1) + vmsplice_error = true; + + // It's important that we DO NOT clear the input buffer string until the contract + // process has actually read from the fd. Because the OS is just mapping our + // input buffer memory portion into the fd, if we clear it now, the contract process + // will get invaid bytes when reading the fd. } - // We use vmsplice to map (zero-copy) the inputs into the fd. - if (vmsplice(writefd, memsegs, inputs.size(), 0) == -1) - vmsplice_error = true; + // Close the writefd since we no longer need it. + close(writefd); + fds[FDTYPE::HPWRITE] = 0; - // It's important that we DO NOT clear the input buffer string until the contract - // process has actually read from the fd. Because the OS is just mapping our - // input buffer memory portion into the fd, if we clear it now, the contract process - // will get invaid bytes when reading the fd. + return vmsplice_error ? -1 : 0; } - // Close the writefd since we no longer need it. - close(writefd); - fds[FDTYPE::HPWRITE] = 0; - - return vmsplice_error ? -1 : 0; -} - -/** + /** * Write the given input buffer into the write fd from the HP side. * @param fds Vector of fd list. * @param inputs Buffer to write into the HP write fd. */ -int write_npl_iopipe(std::vector &fds, std::list &inputs) -{ - /** + int write_npl_iopipe(std::vector &fds, std::list &inputs) + { + /** * npl inputs are feed into the contract in a binary protocol. It follows the following pattern * |**NPL version (1 byte)**|**Reserved (1 byte)**|**Length of the message (2 bytes)**|**Public key (4 bytes)**|**Npl message data**| * Length of the message is calculated without including public key length */ - const int writefd = fds[FDTYPE::HPWRITE]; - bool vmsplice_error = false; - if (!inputs.empty()) - { - int8_t total_memsegs = inputs.size() * 3; - iovec memsegs[total_memsegs]; - size_t i = 0; - for (auto &input : inputs) + const int writefd = fds[FDTYPE::HPWRITE]; + bool vmsplice_error = false; + if (!inputs.empty()) { - int8_t pre_header_index = i * 3; - int8_t pubkey_index = pre_header_index + 1; - int8_t msg_index = pre_header_index + 2; + int8_t total_memsegs = inputs.size() * 3; + iovec memsegs[total_memsegs]; + size_t i = 0; + for (auto &input : inputs) + { + int8_t pre_header_index = i * 3; + int8_t pubkey_index = pre_header_index + 1; + int8_t msg_index = pre_header_index + 2; - // First binary representation of version, reserve and message length is constructed and feed it into - // memory segment. Then the public key and at last the message data + // First binary representation of version, reserve and message length is constructed and feed it into + // memory segment. Then the public key and at last the message data - // At the moment no data is inserted as reserve - uint8_t reserve = 0; + // At the moment no data is inserted as reserve + uint8_t reserve = 0; - //Get message container - const fbschema::p2pmsg::Container *container = fbschema::p2pmsg::GetContainer(input.data()); - const flatbuffers::Vector *container_content = container->content(); + //Get message container + const fbschema::p2pmsg::Container *container = fbschema::p2pmsg::GetContainer(input.data()); + const flatbuffers::Vector *container_content = container->content(); - uint16_t msg_length = container_content->size(); + uint16_t msg_length = container_content->size(); - /** + /** * Pre header is constructed using bit shifting. This will generate a bit pattern as explain in the example below * version = 00000001 * reserve = 00000000 * msg_length = 0000000010001101 * pre_header = 00000001000000000000000010001101 */ - uint32_t pre_header = util::MIN_NPL_INPUT_VERSION; - pre_header = pre_header << 8; - pre_header += reserve; + uint32_t pre_header = util::MIN_NPL_INPUT_VERSION; + pre_header = pre_header << 8; + pre_header += reserve; - pre_header = pre_header << 16; - pre_header += msg_length; - memsegs[pre_header_index].iov_base = &pre_header; - memsegs[pre_header_index].iov_len = 4; + pre_header = pre_header << 16; + pre_header += msg_length; + memsegs[pre_header_index].iov_base = &pre_header; + memsegs[pre_header_index].iov_len = 4; - std::string_view msg_pubkey = fbschema::flatbuff_bytes_to_sv(container->pubkey()); - memsegs[pubkey_index].iov_base = reinterpret_cast(const_cast(msg_pubkey.data())); - memsegs[pubkey_index].iov_len = msg_pubkey.size(); + std::string_view msg_pubkey = fbschema::flatbuff_bytes_to_sv(container->pubkey()); + memsegs[pubkey_index].iov_base = reinterpret_cast(const_cast(msg_pubkey.data())); + memsegs[pubkey_index].iov_len = msg_pubkey.size(); - memsegs[msg_index].iov_base = reinterpret_cast(const_cast(container_content->Data())); - memsegs[msg_index].iov_len = container_content->size(); + memsegs[msg_index].iov_base = reinterpret_cast(const_cast(container_content->Data())); + memsegs[msg_index].iov_len = container_content->size(); - i++; + i++; + } + + if (vmsplice(writefd, memsegs, total_memsegs, 0) == -1) + vmsplice_error = true; } + // It's important that we DO NOT clear the input buffer string until the contract + // process has actually read from the fd. Because the OS is just mapping our + // input buffer memory portion into the fd, if we clear it now, the contract process + // will get invaid bytes when reading the fd. - if (vmsplice(writefd, memsegs, total_memsegs, 0) == -1) - vmsplice_error = true; + // Close the writefd since we no longer need it. + close(writefd); + fds[FDTYPE::HPWRITE] = 0; + + return vmsplice_error ? -1 : 0; } - // It's important that we DO NOT clear the input buffer string until the contract - // process has actually read from the fd. Because the OS is just mapping our - // input buffer memory portion into the fd, if we clear it now, the contract process - // will get invaid bytes when reading the fd. - // Close the writefd since we no longer need it. - close(writefd); - fds[FDTYPE::HPWRITE] = 0; - - return vmsplice_error ? -1 : 0; -} - -/** + /** * Common function to read and close SC output from the pipe and populate the output list. * @param fds Vector representing the pipes fd list. * @param output The buffer to place the read output. */ -int read_iopipe(std::vector &fds, std::string &output) -{ - // Read any data that have been written by the contract process - // from the output pipe and store in the output buffer. - // Outputs will be read by the consensus process later when it wishes so. - - const int readfd = fds[FDTYPE::HPREAD]; - int bytes_available = 0; - ioctl(readfd, FIONREAD, &bytes_available); - bool vmsplice_error = false; - - if (bytes_available > 0) + int read_iopipe(std::vector &fds, std::string &output) { - output.resize(bytes_available); + // Read any data that have been written by the contract process + // from the output pipe and store in the output buffer. + // Outputs will be read by the consensus process later when it wishes so. - // Populate the user output buffer with new data from the pipe. - // We use vmsplice to map (zero-copy) the output from the fd into output bbuffer. - iovec memsegs[1]; - memsegs[0].iov_base = output.data(); - memsegs[0].iov_len = bytes_available; + const int readfd = fds[FDTYPE::HPREAD]; + int bytes_available = 0; + ioctl(readfd, FIONREAD, &bytes_available); + bool vmsplice_error = false; - if (vmsplice(readfd, memsegs, 1, 0) == -1) - vmsplice_error = true; + if (bytes_available > 0) + { + output.resize(bytes_available); + + // Populate the user output buffer with new data from the pipe. + // We use vmsplice to map (zero-copy) the output from the fd into output bbuffer. + iovec memsegs[1]; + memsegs[0].iov_base = output.data(); + memsegs[0].iov_len = bytes_available; + + if (vmsplice(readfd, memsegs, 1, 0) == -1) + vmsplice_error = true; + } + + // Close readfd fd on HP process side because we are done with contract process I/O. + close(readfd); + fds[FDTYPE::HPREAD] = 0; + + return vmsplice_error ? -1 : 0; } - // Close readfd fd on HP process side because we are done with contract process I/O. - close(readfd); - fds[FDTYPE::HPREAD] = 0; + void close_unused_fds(const bool is_hp) + { + close_unused_vectorfds(is_hp, hpscfds); - return vmsplice_error ? -1 : 0; -} + close_unused_vectorfds(is_hp, nplfds); -void close_unused_fds(const bool is_hp) -{ - close_unused_vectorfds(is_hp, hpscfds); + // Loop through user fds. + for (auto &[pubkey, fds] : userfds) + close_unused_vectorfds(is_hp, fds); + } - close_unused_vectorfds(is_hp, nplfds); - - // Loop through user fds. - for (auto &[pubkey, fds] : userfds) - close_unused_vectorfds(is_hp, fds); -} - -/** + /** * Common function for closing unused fds based on which process this gets called from. * @param is_hp Specify 'true' when calling from HP process. 'false' from SC process. * @param fds Vector of fds to close. */ -void close_unused_vectorfds(const bool is_hp, std::vector &fds) -{ - if (is_hp) + void close_unused_vectorfds(const bool is_hp, std::vector &fds) { - // Close unused fds in Hot Pocket process. - close(fds[FDTYPE::SCREAD]); - fds[FDTYPE::SCREAD] = 0; - close(fds[FDTYPE::SCWRITE]); - fds[FDTYPE::SCWRITE] = 0; - } - else - { - // Close unused fds in smart contract process. - close(fds[FDTYPE::HPREAD]); - fds[FDTYPE::HPREAD] = 0; + if (is_hp) + { + // Close unused fds in Hot Pocket process. + close(fds[FDTYPE::SCREAD]); + fds[FDTYPE::SCREAD] = 0; + close(fds[FDTYPE::SCWRITE]); + fds[FDTYPE::SCWRITE] = 0; + } + else + { + // Close unused fds in smart contract process. + close(fds[FDTYPE::HPREAD]); + fds[FDTYPE::HPREAD] = 0; - // HPWRITE fd has aleady been closed by HP process after writing - // inputs (before the fork). + // HPWRITE fd has aleady been closed by HP process after writing + // inputs (before the fork). + } } -} -/** + /** * Cleanup any running processes. */ -void deinit() -{ - if (contract_pid > 0) - kill(contract_pid, SIGINT); + void deinit() + { + if (contract_pid > 0) + util::kill_process(contract_pid, true); - if (statemon_pid > 0) - kill(statemon_pid, SIGINT); -} + if (hpfs_pid > 0) + util::kill_process(hpfs_pid, true); + } } // namespace proc diff --git a/src/proc.hpp b/src/proc.hpp index b92e2c41..66c34fb4 100644 --- a/src/proc.hpp +++ b/src/proc.hpp @@ -3,6 +3,7 @@ #include "pchheader.hpp" #include "usr/usr.hpp" +#include "hpfs/h32.hpp" #include "util.hpp" /** @@ -33,11 +34,6 @@ typedef std::unordered_map> contract_fdmap_t; // This is used to keep track of input/output buffers for a given public key (eg. user, npl) typedef std::unordered_map contract_bufmap_t; -// Common typedef for a map of updated blocks of state files by the contract process. -// This is used as a hint in updating the state merkle tree. -// filename->modified blocks -typedef std::unordered_map> contract_fblockmap_t; - /** * Holds information that should be passed into the contract process. */ @@ -54,10 +50,6 @@ struct contract_exec_args // Pair of HP<->SC JSON message buffers (mainly used for control messages). // Input buffers for HP->SC messages, Output buffers for SC->HP messages. contract_iobuf_pair &hpscbufs; - - // The map of state files that was updated with updated block ids. - // Each block id N represents Nth 4MB block of the file. - contract_fblockmap_t &state_updates; // Current HotPocket timestamp. const int64_t timestamp; @@ -66,18 +58,16 @@ struct contract_exec_args int64_t timestamp, contract_bufmap_t &userbufs, contract_iobuf_pair &nplbuff, - contract_iobuf_pair &hpscbufs, - contract_fblockmap_t &state_updates) : + contract_iobuf_pair &hpscbufs) : userbufs(userbufs), nplbuff(nplbuff), hpscbufs(hpscbufs), - state_updates(state_updates), timestamp(timestamp) { } }; -int exec_contract(const contract_exec_args &args); +int exec_contract(const contract_exec_args &args, hpfs::h32 &state_hash); void deinit(); @@ -85,9 +75,9 @@ void deinit(); int await_process_execution(pid_t pid); -int start_state_monitor(); +int start_hpfs_rw_session(); -int stop_state_monitor(); +int stop_hpfs_rw_session(hpfs::h32 &state_hash); int write_contract_args(const contract_exec_args &args); diff --git a/src/statefs/hasher.cpp b/src/statefs/hasher.cpp deleted file mode 100644 index d9c0a1f3..00000000 --- a/src/statefs/hasher.cpp +++ /dev/null @@ -1,82 +0,0 @@ -#include "hasher.hpp" - -/** - * Contains hashing functions and helpers used to manipulate block hashes used in state management. - * This could also be used throughout rest of the application as well. However for now we are only - * using this for state management code base only. - * - * Based on https://github.com/codetsunami/file-ptracer/blob/master/merkle.cpp - */ -namespace hasher -{ - -// Represents empty/default B2H hash value. -B2H B2H_empty = hasher::B2H_empty; - -/** - * Helper functions for working with 32 byte hash type B2H. - */ - -bool B2H::operator==(const B2H rhs) const -{ - return this->data[0] == rhs.data[0] && this->data[1] == rhs.data[1] && this->data[2] == rhs.data[2] && this->data[3] == rhs.data[3]; -} - -bool B2H::operator!=(const B2H rhs) const -{ - return this->data[0] != rhs.data[0] || this->data[1] != rhs.data[1] || this->data[2] != rhs.data[2] || this->data[3] != rhs.data[3]; -} - -void B2H::operator^=(const B2H rhs) -{ - this->data[0] ^= rhs.data[0]; - this->data[1] ^= rhs.data[1]; - this->data[2] ^= rhs.data[2]; - this->data[3] ^= rhs.data[3]; -} - -std::ostream &operator<<(std::ostream &output, const B2H &h) -{ - output << h.data[0];// << h.data[1] << h.data[2] << h.data[3]; - return output; -} - -std::stringstream &operator<<(std::stringstream &output, const B2H &h) -{ - output << std::hex << h; - return output; -} - -// The actual hash function, note that the B2H datatype is always passed by value being only 4 quadwords. -// This function accepts two buffers to hash together in order to support common use case in state handling. -B2H hash(const void *buf1, const size_t buf1len, const void *buf2, const size_t buf2len) -{ - crypto_generichash_blake2b_state state; - crypto_generichash_blake2b_init(&state, NULL, 0, HASH_SIZE); - - crypto_generichash_blake2b_update(&state, - reinterpret_cast(buf1), buf1len); - crypto_generichash_blake2b_update(&state, - reinterpret_cast(buf2), buf2len); - B2H ret; - crypto_generichash_blake2b_final( - &state, - reinterpret_cast(&ret), - HASH_SIZE); - return ret; -} - -// Helper class to support std::map/std::unordered_map custom hashing function. -// This is needed to use B2H as the std map container key. -size_t B2H_std_key_hasher::operator()(const hasher::B2H h) const -{ - // Compute individual hash values. http://stackoverflow.com/a/1646913/126995 - size_t res = 17; - res = res * 31 + std::hash()(h.data[0]); - res = res * 31 + std::hash()(h.data[1]); - res = res * 31 + std::hash()(h.data[2]); - res = res * 31 + std::hash()(h.data[3]); - return res; -} - -} // namespace hasher \ No newline at end of file diff --git a/src/statefs/hasher.hpp b/src/statefs/hasher.hpp deleted file mode 100644 index 26006cf1..00000000 --- a/src/statefs/hasher.hpp +++ /dev/null @@ -1,40 +0,0 @@ -#ifndef _HASHER_ -#define _HASHER_ - -#include "../pchheader.hpp" - -namespace hasher -{ - -// Hash length (32 bytes) -constexpr size_t HASH_SIZE = crypto_generichash_blake2b_BYTES; - -// blake2b hash is 32 bytes which we store as 4 quad words -// Originally from https://github.com/codetsunami/file-ptracer/blob/master/merkle.cpp -struct B2H -{ - uint64_t data[4]; - - bool operator==(const B2H rhs) const; - bool operator!=(const B2H rhs) const; - void operator^=(const B2H rhs); -}; - -extern B2H B2H_empty; - -std::ostream &operator<<(std::ostream &output, const B2H &h); -std::stringstream &operator<<(std::stringstream &output, const B2H &h); - -B2H hash(const void *buf1, const size_t buf1len, const void *buf2, const size_t buf2len); - -// Helper class to support std::map/std::unordered_map custom hashing function. -// This is needed to use B2H as the std map container key. -class B2H_std_key_hasher -{ -public: - size_t operator()(const hasher::B2H h) const; -}; - -} // namespace hasher - -#endif \ No newline at end of file diff --git a/src/statefs/hashmap_builder.cpp b/src/statefs/hashmap_builder.cpp deleted file mode 100644 index 6a2bb4d8..00000000 --- a/src/statefs/hashmap_builder.cpp +++ /dev/null @@ -1,505 +0,0 @@ -#include "../pchheader.hpp" -#include "../hplog.hpp" -#include "state_common.hpp" -#include "hashmap_builder.hpp" -#include "hasher.hpp" - -namespace statefs -{ - -/** - * Hashmap builder class is responsible for updating file hash based on the modified blocks of a file. - */ - -hashmap_builder::hashmap_builder(const state_dir_context &ctx) : ctx(ctx) -{ -} - -/** - * Generates/updates the block hash map for a file and updates the parent dir hash accordingly as well. - * @param parent_dir_hash Hash of the parent directory. This will be updated of the file hash was updated. - * @param filepath Full path to the actual state file. - * @param file_relpath The relative path to the state file from the state data directory. - * @param changed_blocks Index of changed blocks and the new hashes to be used as a hint. - * @return 0 on success. -1 on failure. - */ -int hashmap_builder::generate_hashmap_for_file(hasher::B2H &parent_dir_hash, const std::string &filepath, const std::string &file_relpath, const std::map &changed_blocks) -{ - // We attempt to avoid a full rebuild of the block hash map file when possible. - // For this optimisation, both the block hash map (.bhmap) file and the - // delta block index must exist. - - // Block index may be provided as an argument. If it is empty we attempt to read from the - // .bindex file from the state checkpoint delta. - - // If the block index exists, we generate/update the hashmap file with the aid of that. - // Block index file contains the updated blockids. If not, we simply rehash all the blocks. - - // Open the actual data file and calculate the block count. - int orifd = open(filepath.data(), O_RDONLY); - if (orifd == -1) - { - LOG_ERR << errno << ": Open failed " << filepath; - return -1; - } - const off_t file_length = lseek(orifd, 0, SEEK_END); - const uint32_t block_count = ceil((double)file_length / (double)BLOCK_SIZE); - - // Attempt to read the existing block hash map file. - std::string bhmap_file; - std::vector bhmap_data; - if (read_block_hashmap(bhmap_data, bhmap_file, file_relpath) == -1) - { - close(orifd); - return -1; - } - - hasher::B2H old_file_hash = hasher::B2H_empty; - if (!bhmap_data.empty()) - memcpy(&old_file_hash, bhmap_data.data(), hasher::HASH_SIZE); - - // Array to contain the updated block hashes. Slot 0 is for the root hash. - // Allocating hash array on the heap to avoid filling limited stack space. - std::unique_ptr hashes = std::make_unique(1 + block_count); - const size_t hashes_size = (1 + block_count) * hasher::HASH_SIZE; - - if (changed_blocks.empty()) - { - // Attempt to read the delta block index file. - std::map bindex; - uint32_t original_block_count; - if (get_delta_block_index(bindex, original_block_count, file_relpath) == -1) - { - close(orifd); - return -1; - } - - if (update_hashes_with_backup_block_hints(hashes.get(), hashes_size, file_relpath, orifd, block_count, original_block_count, bindex, bhmap_data) == -1) - { - close(orifd); - return -1; - } - } - else - { - if (update_hashes_with_changed_block_hints(hashes.get(), hashes_size, file_relpath, orifd, block_count, changed_blocks, bhmap_data) == -1) - { - close(orifd); - return -1; - } - } - - close(orifd); - - if (write_block_hashmap(bhmap_file, hashes.get(), hashes_size) == -1) - return -1; - - if (update_hashtree_entry(parent_dir_hash, !bhmap_data.empty(), old_file_hash, hashes[0], bhmap_file, file_relpath) == -1) - return -1; - - return 0; -} - -/** - * Reads the block hash map of a given data file into the provided vector. - * @param bhmap_data Vector to copy the block hash map contents. - * @param bhmap_file The full path to the block hash map file pointed to by the relative path. - * @param relpath The relative path of the actual data file. - * @return 0 on success. -1 on failure. - */ -int hashmap_builder::read_block_hashmap(std::vector &bhmap_data, std::string &bhmap_file, const std::string &relpath) -{ - bhmap_file.reserve(ctx.block_hashmap_dir.length() + relpath.length() + BLOCK_HASHMAP_EXT_LEN); - bhmap_file.append(ctx.block_hashmap_dir).append(relpath).append(BLOCK_HASHMAP_EXT); - - if (boost::filesystem::exists(bhmap_file)) - { - int hmapfd = open(bhmap_file.c_str(), O_RDONLY); - if (hmapfd == -1) - { - LOG_ERR << errno << ": Open failed " << bhmap_file; - return -1; - } - - off_t size = lseek(hmapfd, 0, SEEK_END); - bhmap_data.resize(size); - - if (pread(hmapfd, bhmap_data.data(), size, 0) == -1) - { - LOG_ERR << errno << ": Read failed " << bhmap_file; - close(hmapfd); - return -1; - } - - close(hmapfd); - } - else - { - // Create directory tree if not exist so we are able to create the hashmap files. - boost::filesystem::path hmapsubdir = boost::filesystem::path(bhmap_file).parent_path(); - if (created_bhmapsubdirs.count(hmapsubdir.string()) == 0) - { - boost::filesystem::create_directories(hmapsubdir); - created_bhmapsubdirs.emplace(hmapsubdir.string()); - } - } - - return 0; -} - -/** - * Reads the delta block index of a file. - * @param idxmap Map to copy the block index contents (block id --> hash). - * @param total_block_count Reference to hold the total block count of the original data file. - * @param file_relpath Relative path to the data file. - * @return 0 on success. -1 on failure. - */ -int hashmap_builder::get_delta_block_index(std::map &idxmap, uint32_t &total_block_count, const std::string &file_relpath) -{ - std::string bindex_file; - bindex_file.reserve(ctx.delta_dir.length() + file_relpath.length() + BLOCK_INDEX_EXT_LEN); - bindex_file.append(ctx.delta_dir).append(file_relpath).append(BLOCK_INDEX_EXT); - - if (boost::filesystem::exists(bindex_file)) - { - std::ifstream in_file(bindex_file, std::ios::binary | std::ios::ate); - std::streamsize idx_size = in_file.tellg(); - in_file.seekg(0, std::ios::beg); - - // Read the block index file into a vector. - std::vector bindex(idx_size); - if (in_file.read(bindex.data(), idx_size)) - { - // First 8 bytes contain the original file length. - off_t orifilelen; - memcpy(&orifilelen, bindex.data(), 8); - total_block_count = ceil((double)orifilelen / (double)BLOCK_SIZE); - - // Skip the first 8 bytes and loop through index entries. - for (uint32_t idx_offset = 8; idx_offset < bindex.size();) - { - // Read the block no. (4 bytes) of where this block is from in the original file. - uint32_t block_no = 0; - memcpy(&block_no, bindex.data() + idx_offset, 4); - idx_offset += 12; // Skip the cached block offset (8 bytes) - - // Read the block hash (32 bytes). - hasher::B2H hash; - memcpy(&hash, bindex.data() + idx_offset, 32); - idx_offset += 32; - - idxmap.try_emplace(block_no, hash); - } - } - else - { - LOG_ERR << errno << ": Read failed " << bindex_file; - return -1; - } - - in_file.close(); - } - - return 0; -} - -/** - * Updates the hash map with the use of delta backup block ids. - * @param hashes Pointer to the hash array to copy the block hashes after the update. - * @param hashes_size Byte length of the hashes array. - * @param relpath Relative path of the data file. - * @param orifd An open file descriptor to the data file. - * @param block_count Block count of the updated file. - * @param original_block_count Original block count before the update. - * @param bindex Delta backup block index map. - * @param bhmap_data Contents of the existing block hash map. - * @return 0 on success. -1 on failure. - */ -int hashmap_builder::update_hashes_with_backup_block_hints( - hasher::B2H *hashes, const off_t hashes_size, const std::string &relpath, const int orifd, const uint32_t block_count, - const uint32_t original_block_count, const std::map &bindex, const std::vector &bhmap_data) -{ - uint32_t nohint_blockstart = 0; - - // If both existing delta block index and block hash map is available, we can just overlay the - // changed block hashes (mentioned in the delta block index) on top of the old block hashes. - // This would prevent unncessarily hashing lot of blocks. - if (!bhmap_data.empty() && !bindex.empty()) - { - // Load old hashes. - memcpy(hashes, bhmap_data.data(), hashes_size < bhmap_data.size() ? hashes_size : bhmap_data.size()); - - // Refer to the block index and rehash the changed blocks. - for (const auto [block_id, old_hash] : bindex) - { - // If the block_id from the block index is no longer there, that means the current file is - // shorter than the previous version. So we can stop hashing at this point. - if (block_id >= block_count) - break; - - if (compute_blockhash(hashes[block_id + 1], block_id, orifd, relpath) == -1) - return -1; - } - - // If the current file has more blocks than the previous version, we need to hash those - // additional blocks as well. - if (block_count > original_block_count) - nohint_blockstart = original_block_count; - else - nohint_blockstart = block_count; // No additional blocks remaining. - } - - //Hash any additional blocks that has to be hashed without the guidance of block index. - for (uint32_t block_id = nohint_blockstart; block_id < block_count; block_id++) - { - if (compute_blockhash(hashes[block_id + 1], block_id, orifd, relpath) == -1) - return -1; - } - - // Calculate the new file hash: filehash = HASH(filename + XOR(block hashes)) - hasher::B2H filehash = hasher::B2H_empty; - for (int i = 1; i <= block_count; i++) - filehash ^= hashes[i]; - - // Rehash the file hash with filename included. - const std::string filename = boost::filesystem::path(relpath.data()).filename().string(); - filehash = hasher::hash(filename.c_str(), filename.length(), &filehash, hasher::HASH_SIZE); - - hashes[0] = filehash; - return 0; -} - -/** - * Updates the hash map with the use of list of updated block ids. - * @param hashes Pointer to the hash array to copy the block hashes after the update. - * @param hashes_size Byte length of the hashes array. - * @param relpath Relative path of the data file. - * @param orifd An open file descriptor to the data file. - * @param block_count Block count of the updated file. - * @param bindex Map of updated block ids and new hashes. - * @param bhmap_data Contents of the existing block hash map. - * @return 0 on success. -1 on failure. - */ -int hashmap_builder::update_hashes_with_changed_block_hints( - hasher::B2H *hashes, const off_t hashes_size, const std::string &relpath, const int orifd, - const uint32_t block_count, const std::map &bindex, const std::vector &bhmap_data) -{ - // If both existing delta block index and block hash map is available, we can just overlay the - // changed block hashes (mentioned in the delta block index) on top of the old block hashes. - // This would prevent unncessarily hashing lot of blocks. - - if (!bindex.empty()) - { - // Load old hashes if exists. - if (!bhmap_data.empty()) - memcpy(hashes, bhmap_data.data(), hashes_size < bhmap_data.size() ? hashes_size : bhmap_data.size()); - - // Refer to the block index and overlay the new hash into the hashes array. - for (const auto [block_id, new_hash] : bindex) - hashes[block_id + 1] = new_hash; - - // If the block hash map didn't existed, we need to calculate and fill the unchanged block hashes from the actual file. - if (bhmap_data.empty()) - { - for (uint32_t block_id = 0; block_id < block_count; block_id++) - { - if (bindex.count(block_id) == 0 && compute_blockhash(hashes[block_id + 1], block_id, orifd, relpath) == -1) - return -1; - } - } - } - else - { - // If we don't have the changed block index, we have to hash the entire file blocks again. - for (uint32_t block_id = 0; block_id < block_count; block_id++) - { - if (compute_blockhash(hashes[block_id + 1], block_id, orifd, relpath) == -1) - return -1; - } - } - - // Calculate the new file hash: filehash = HASH(filename + XOR(block hashes)) - hasher::B2H filehash = hasher::B2H_empty; - for (int i = 1; i <= block_count; i++) - filehash ^= hashes[i]; - - // Rehash the file hash with filename included. - const std::string filename = boost::filesystem::path(relpath.data()).filename().string(); - filehash = hasher::hash(filename.c_str(), filename.length(), &filehash, hasher::HASH_SIZE); - - hashes[0] = filehash; - return 0; -} - -/** - * Calculates the hash of the specified block id of a file. - * @param hash Reference to assign the calculated hash. - * @param block_id Id of the block to be hashed. - * @param filefd Open file descriptor for the state data file. - * @param relpath Relative path of the state data file. - * @return 0 on success. -1 on failure. - */ -int hashmap_builder::compute_blockhash(hasher::B2H &hash, const uint32_t block_id, const int filefd, const std::string &relpath) -{ - // Allocating block buffer on the heap to avoid filling limited stack space. - std::unique_ptr block_buf = std::make_unique(BLOCK_SIZE); - const off_t block_offset = BLOCK_SIZE * block_id; - size_t bytes_read = pread(filefd, block_buf.get(), BLOCK_SIZE, block_offset); - if (bytes_read == -1) - { - LOG_ERR << errno << ": Read failed " << relpath; - return -1; - } - - hash = hasher::hash(&block_offset, 8, block_buf.get(), bytes_read); - return 0; -} - -/** - * Saves the block hash map into the relevant .bhmap file. - * @param bhmap_file Full path to the block hash map file. - * @param hashes Pointer to the hashes array containing the root hash and block hashes. - * @param hashes_size Byte length of the hashes array. - * @return 0 on success. -1 on failure. - */ -int hashmap_builder::write_block_hashmap(const std::string &bhmap_file, const hasher::B2H *hashes, const off_t hashes_size) -{ - int hmapfd = open(bhmap_file.c_str(), O_RDWR | O_TRUNC | O_CREAT, FILE_PERMS); - if (hmapfd == -1) - { - LOG_ERR << errno << ": Open failed " << bhmap_file; - return -1; - } - - // Write the updated hash list into the block hash map file. - if (pwrite(hmapfd, hashes, hashes_size, 0) == -1) - { - LOG_ERR << errno << ": Write failed " << bhmap_file; - close(hmapfd); - return -1; - } - - if (ftruncate(hmapfd, hashes_size) == -1) - { - LOG_ERR << errno << ": Truncate failed " << bhmap_file; - close(hmapfd); - return -1; - } - - close(hmapfd); -} - -/** - * Updates a file hash and adjust parent dir hash of the hash tree. - * @param parent_dir_hash Current hash of the parent dir. This will be assigned the new hash after the update. - * @param old_bhmap_exists Whether the block hash map file already exists or not. - * @param old_file_hash Old file hash. (0000 if this is a new file) - * @param new_file_hash New file hash. - * @param bhmap_file Full path to the block hash map file. - * @param relpath Relative path to the state data file. - * @return 0 on success. -1 on failure. - */ -int hashmap_builder::update_hashtree_entry(hasher::B2H &parent_dir_hash, const bool old_bhmap_exists, const hasher::B2H old_file_hash, - const hasher::B2H new_file_hash, const std::string &bhmap_file, const std::string &relpath) -{ - std::string hardlink_dir(ctx.hashtree_dir); - const std::string relpath_dir = boost::filesystem::path(relpath).parent_path().string(); - - hardlink_dir.append(relpath_dir); - if (relpath_dir != "/") - hardlink_dir.append("/"); - - std::stringstream new_hl_path; - new_hl_path << hardlink_dir << new_file_hash << ".rh"; - - // TODO: Even though we maintain hardlinks named after the file hash, we don't actually utilize them elsewhere. - // The intention is to be able to get a hash listing of the entire directory. Such ability is useful to serve state - // requests. However since state requests need the file name along with the hash we have to resort to iterating each - // .bhmap file and reading the file hash from first 32 bytes. - - if (old_bhmap_exists) - { - // Rename the existing hard link if old block hash map existed. - // We thereby assume the old hard link also existed. - std::stringstream oldhlpath; - oldhlpath << hardlink_dir << old_file_hash << ".rh"; - if (rename(oldhlpath.str().c_str(), new_hl_path.str().c_str()) == -1) - return -1; - - // Subtract the old root hash and add the new root hash from the parent dir hash. - parent_dir_hash ^= old_file_hash; - parent_dir_hash ^= new_file_hash; - } - else - { - // Create a new hard link with new root hash as the name. - if (link(bhmap_file.c_str(), new_hl_path.str().c_str()) == -1) - return -1; - - // Add the new root hash to parent hash. - parent_dir_hash ^= new_file_hash; - } - - return 0; -} - -/** - * Removes an existing block hash map file. Caled when deleting a state data file. - * @param parent_dir_hash Current hash of the parent dir. This will be assigned the new hash after the update. - * @param Full path to the block hash map file. - * @return 0 on success. -1 on failure. - */ -int hashmap_builder::remove_hashmap_file(hasher::B2H &parent_dir_hash, const std::string &bhmap_file) -{ - if (boost::filesystem::exists(bhmap_file)) - { - int hmapfd = open(bhmap_file.data(), O_RDONLY); - if (hmapfd == -1) - { - LOG_ERR << errno << ": Open failed " << bhmap_file; - return -1; - } - - hasher::B2H filehash; - if (read(hmapfd, &filehash, hasher::HASH_SIZE) == -1) - { - LOG_ERR << errno << ": Read failed " << bhmap_file; - close(hmapfd); - return -1; - } - - // Delete the .bhmap file. - if (remove(bhmap_file.c_str()) == -1) - { - LOG_ERR << errno << ": Delete failed " << bhmap_file; - close(hmapfd); - return -1; - } - - // Delete the hardlink of the .bhmap file. - std::string hardlink_dir(ctx.hashtree_dir); - const std::string relpath = get_relpath(bhmap_file, ctx.block_hashmap_dir); - const std::string relpath_dir = boost::filesystem::path(relpath).parent_path().string(); - - hardlink_dir.append(relpath_dir); - if (relpath_dir != "/") - hardlink_dir.append("/"); - - std::stringstream hlpath; - hlpath << hardlink_dir << filehash << ".rh"; - if (remove(hlpath.str().c_str()) == -1) - { - LOG_ERR << errno << ": Delete failed for hard link " << filehash << " of " << bhmap_file; - close(hmapfd); - return -1; - } - - // XOR parent dir hash with file hash so the file hash gets removed from parent dir hash. - parent_dir_hash ^= filehash; - close(hmapfd); - } - - return 0; -} - -} // namespace statefs \ No newline at end of file diff --git a/src/statefs/hashmap_builder.hpp b/src/statefs/hashmap_builder.hpp deleted file mode 100644 index e554746f..00000000 --- a/src/statefs/hashmap_builder.hpp +++ /dev/null @@ -1,38 +0,0 @@ -#ifndef _HP_STATEFS_HASHMAP_BUILDER_ -#define _HP_STATEFS_HASHMAP_BUILDER_ - -#include "../pchheader.hpp" -#include "hasher.hpp" -#include "state_common.hpp" - -namespace statefs -{ - -class hashmap_builder -{ -private: - const state_dir_context ctx; - // List of new block hash map sub directories created during the session. - std::unordered_set created_bhmapsubdirs; - - int read_block_hashmap(std::vector &bhmap_data, std::string &hmapfile, const std::string &relpath); - int get_delta_block_index(std::map &idxmap, uint32_t &total_block_count, const std::string &file_relpath); - int update_hashes_with_backup_block_hints( - hasher::B2H *hashes, const off_t hashes_size, const std::string &relpath, const int orifd, - const uint32_t block_count, const uint32_t original_block_count, const std::map &bindex, const std::vector &bhmap_data); - int update_hashes_with_changed_block_hints( - hasher::B2H *hashes, const off_t hashes_size, const std::string &relpath, const int orifd, - const uint32_t block_count, const std::map &bindex, const std::vector &bhmap_data); - int compute_blockhash(hasher::B2H &hash, const uint32_t block_id, const int filefd, const std::string &relpath); - int write_block_hashmap(const std::string &bhmap_file, const hasher::B2H *hashes, const off_t hashes_size); - int update_hashtree_entry(hasher::B2H &parent_dir_hash, const bool old_bhmap_exists, const hasher::B2H old_file_hash, const hasher::B2H new_file_hash, const std::string &bhmap_file, const std::string &relpath); - -public: - hashmap_builder(const state_dir_context &ctx); - int generate_hashmap_for_file(hasher::B2H &parent_dir_hash, const std::string &filepath, const std::string &file_relpath, const std::map &changed_blocks); - int remove_hashmap_file(hasher::B2H &parent_dir_hash, const std::string &filepath); -}; - -} // namespace statefs - -#endif diff --git a/src/statefs/hashtree_builder.cpp b/src/statefs/hashtree_builder.cpp deleted file mode 100644 index 30d23899..00000000 --- a/src/statefs/hashtree_builder.cpp +++ /dev/null @@ -1,299 +0,0 @@ -#include "../pchheader.hpp" -#include "hashtree_builder.hpp" -#include "state_restore.hpp" -#include "state_common.hpp" - -namespace statefs -{ - -hashtree_builder::hashtree_builder(const state_dir_context &ctx) : ctx(ctx), hmapbuilder(ctx) -{ - force_rebuild_all = false; - hint_mode = false; -} - -int hashtree_builder::generate(hasher::B2H &root_hash) -{ - // Load modified file path hints if available. - populate_hint_paths_from_idx_file(IDX_TOUCHED_FILES); - populate_hint_paths_from_idx_file(IDX_NEW_FILES); - hint_mode = !hint_paths.empty(); - - return traverse_and_generate(root_hash); -} - -int hashtree_builder::generate(hasher::B2H &root_hash, const bool force_all) -{ - force_rebuild_all = force_all; - if (force_rebuild_all) - { - boost::filesystem::remove_all(ctx.block_hashmap_dir); - boost::filesystem::remove_all(ctx.hashtree_dir); - - boost::filesystem::create_directories(ctx.block_hashmap_dir); - boost::filesystem::create_directories(ctx.hashtree_dir); - } - - return traverse_and_generate(root_hash); -} - -int hashtree_builder::generate(hasher::B2H &root_hash, const std::unordered_map> &touched_files) -{ - hint_mode = true; - file_block_index = touched_files; - for (const auto &[relpath, bindex] : touched_files) - insert_hint_path(relpath); - - return traverse_and_generate(root_hash); -} - -int hashtree_builder::traverse_and_generate(hasher::B2H &root_hash) -{ - // Load current root hash if exist. - const std::string dir_hash_file = ctx.hashtree_dir + "/" + DIR_HASH_FNAME; - root_hash = get_existing_dir_hash(dir_hash_file); - - traversel_rootdir = ctx.data_dir; - removal_mode = false; - if (update_hashtree(root_hash) != 0) - return -1; - - // If there are any remaining hint files directly under this directory, that means - // those files are no longer there. So we need to delete the corresponding .bhmap and rh files - // and adjust the directory hash accordingly. - if (hint_mode && !hint_paths.empty()) - { - traversel_rootdir = ctx.block_hashmap_dir; - removal_mode = true; - if (update_hashtree(root_hash) != 0) - return -1; - } - - return 0; -} - -int hashtree_builder::update_hashtree(hasher::B2H &root_hash) -{ - hintpath_map::iterator hint_dir_itr = hint_paths.end(); - if (!should_process_dir(hint_dir_itr, traversel_rootdir)) - return 0; - - if (update_hashtree_fordir(root_hash, traversel_rootdir, hint_dir_itr, true) != 0) - return -1; - - return 0; -} - -int hashtree_builder::update_hashtree_fordir(hasher::B2H &parent_dir_hash, const std::string &dirpath, const hintpath_map::iterator hint_dir_itr, const bool is_root_level) -{ - const std::string htree_dirpath = switch_base_path(dirpath, traversel_rootdir, ctx.hashtree_dir); - - // Load current dir hash if exist. - const std::string dir_hash_file = htree_dirpath + "/" + DIR_HASH_FNAME; - hasher::B2H dir_hash = get_existing_dir_hash(dir_hash_file); - - // Remember the dir hash before we mutate it. - hasher::B2H original_dir_hash = dir_hash; - - // Iterate files/subdirs inside this dir. - const boost::filesystem::directory_iterator itr_end; - for (boost::filesystem::directory_iterator itr(dirpath); itr != itr_end; itr++) - { - const bool is_dir = boost::filesystem::is_directory(itr->path()); - const std::string path_str = itr->path().string(); - - if (is_dir) - { - hintpath_map::iterator hint_subdir_itr = hint_paths.end(); - if (!should_process_dir(hint_subdir_itr, path_str)) - continue; - - if (update_hashtree_fordir(dir_hash, path_str, hint_subdir_itr, false) != 0) - return -1; - } - else - { - if (!should_process_file(hint_dir_itr, path_str)) - continue; - - if (process_file(dir_hash, path_str, htree_dirpath) != 0) - return -1; - } - } - - // If there are no more files in the hint dir, delete the hint dir entry as well. - if (hint_dir_itr != hint_paths.end() && hint_dir_itr->second.empty()) - hint_paths.erase(hint_dir_itr); - - // In removalmode, we check whether the dir is empty. If so we remove the dir as well. - if (removal_mode && boost::filesystem::is_empty(dirpath)) - { - // We remove the dirs if we are below root level only. - // Otherwise we only remove root dir.hash file. - if (!is_root_level) - { - boost::filesystem::remove_all(dirpath); - boost::filesystem::remove_all(htree_dirpath); - } - else - { - boost::filesystem::remove(dir_hash_file); - } - - // Subtract the original dir hash from the parent dir hash. - parent_dir_hash ^= original_dir_hash; - } - else if (dir_hash != original_dir_hash) - { - // If dir hash has changed, write it back to dir hash file. - if (save_dir_hash(dir_hash_file, dir_hash) == -1) - return -1; - - // Also update the parent dir hash by subtracting the old hash and adding the new hash. - parent_dir_hash ^= original_dir_hash; - parent_dir_hash ^= dir_hash; - } - else - { - parent_dir_hash = dir_hash; - } - - return 0; -} - -hasher::B2H hashtree_builder::get_existing_dir_hash(const std::string &dir_hash_file) -{ - // Load current dir hash if exist. - hasher::B2H dir_hash = hasher::B2H_empty; - int dir_hash_fd = open(dir_hash_file.c_str(), O_RDONLY); - if (dir_hash_fd > 0) - { - read(dir_hash_fd, &dir_hash, hasher::HASH_SIZE); - close(dir_hash_fd); - } - return dir_hash; -} - -int hashtree_builder::save_dir_hash(const std::string &dir_hash_file, hasher::B2H dir_hash) -{ - int dir_hash_fd = open(dir_hash_file.c_str(), O_RDWR | O_TRUNC | O_CREAT, FILE_PERMS); - if (dir_hash_fd == -1) - return -1; - - if (write(dir_hash_fd, &dir_hash, hasher::HASH_SIZE) == -1) - { - close(dir_hash_fd); - return -1; - } - - close(dir_hash_fd); - return 0; -} - -inline bool hashtree_builder::should_process_dir(hintpath_map::iterator &dir_itr, const std::string &dirpath) -{ - if (force_rebuild_all) - return true; - - return (hint_mode ? get_hinteddir_match(dir_itr, dirpath) : true); -} - -bool hashtree_builder::should_process_file(const hintpath_map::iterator hint_dir_itr, const std::string filepath) -{ - if (force_rebuild_all) - return true; - - if (hint_mode) - { - if (hint_dir_itr == hint_paths.end()) - return false; - - std::string relpath = get_relpath(filepath, traversel_rootdir); - - // If in removal mode, we are traversing .bhmap files. Hence we should truncate .bhmap extension - // before we search for the path in file hints. - if (removal_mode) - relpath = relpath.substr(0, relpath.length() - BLOCK_HASHMAP_EXT_LEN); - - std::unordered_set &hint_files = hint_dir_itr->second; - const auto hint_file_itr = hint_files.find(relpath); - if (hint_file_itr == hint_files.end()) - return false; - - // Erase the visiting filepath from hint files. - hint_files.erase(hint_file_itr); - } - return true; -} - -int hashtree_builder::process_file(hasher::B2H &parent_dir_hash, const std::string &filepath, const std::string &htree_dirpath) -{ - if (!removal_mode) - { - // Create directory tree if not exist so we are able to create the file root hash files (hard links). - if (created_htree_subdirs.count(htree_dirpath) == 0) - { - boost::filesystem::create_directories(htree_dirpath); - created_htree_subdirs.emplace(htree_dirpath); - } - - const std::string relpath = get_relpath(filepath, ctx.data_dir); - const std::map &changed_blocks = file_block_index[relpath]; - - if (hmapbuilder.generate_hashmap_for_file(parent_dir_hash, filepath, relpath, changed_blocks) == -1) - return -1; - } - else - { - if (hmapbuilder.remove_hashmap_file(parent_dir_hash, filepath) == -1) - return -1; - } - - return 0; -} - -void hashtree_builder::populate_hint_paths_from_idx_file(const char *const idxfile) -{ - std::ifstream in_file(std::string(ctx.delta_dir).append(idxfile)); - if (!in_file.fail()) - { - for (std::string relpath; std::getline(in_file, relpath);) - insert_hint_path(relpath); - in_file.close(); - } -} - -void hashtree_builder::insert_hint_path(const std::string &relpath) -{ - boost::filesystem::path p_relpath(relpath); - std::string parent_dir = p_relpath.parent_path().string(); - hint_paths[parent_dir].emplace(relpath); -} - -bool hashtree_builder::get_hinteddir_match(hintpath_map::iterator &match_itr, const std::string &dirpath) -{ - // First check whether there's an exact match. If not check for a partial match. - // Exact match will return the iterator. Partial match or not found will return end() iterator. - const std::string relpath = get_relpath(dirpath, traversel_rootdir); - const auto exact_match_itr = hint_paths.find(relpath); - - if (exact_match_itr != hint_paths.end()) - { - match_itr = exact_match_itr; - return true; - } - - for (auto itr = hint_paths.begin(); itr != hint_paths.end(); itr++) - { - if (strncmp(relpath.c_str(), itr->first.c_str(), relpath.length()) == 0) - { - // Partial match found. - match_itr = hint_paths.end(); - return true; - } - } - - return false; // Not found at all. -} - -} // namespace statefs \ No newline at end of file diff --git a/src/statefs/hashtree_builder.hpp b/src/statefs/hashtree_builder.hpp deleted file mode 100644 index ef2d582d..00000000 --- a/src/statefs/hashtree_builder.hpp +++ /dev/null @@ -1,54 +0,0 @@ -#ifndef _HP_STATEFS_HASHTREE_BUILDER_ -#define _HP_STATEFS_HASHTREE_BUILDER_ - -#include "../pchheader.hpp" -#include "hasher.hpp" -#include "hashmap_builder.hpp" -#include "state_common.hpp" - -namespace statefs -{ - -typedef std::unordered_map> hintpath_map; - -class hashtree_builder -{ -private: - const state_dir_context ctx; - hashmap_builder hmapbuilder; - - // Hint path map with parent dir as key and list of file paths under each parent dir. - hintpath_map hint_paths; - bool force_rebuild_all; - bool hint_mode; - bool removal_mode; - std::string traversel_rootdir; - std::unordered_map> file_block_index; - - // List of new root hash map sub directories created during the session. - std::unordered_set created_htree_subdirs; - - int traverse_and_generate(hasher::B2H &root_hash); - int update_hashtree(hasher::B2H &root_hash); - int update_hashtree_fordir(hasher::B2H &parent_dir_hash, const std::string &relpath, const hintpath_map::iterator hint_dir_itr, const bool is_root_level); - - hasher::B2H get_existing_dir_hash(const std::string &dir_hash_file); - int save_dir_hash(const std::string &dir_hash_file, hasher::B2H dir_hash); - bool should_process_dir(hintpath_map::iterator &hint_subdir_itr, const std::string &dirpath); - bool should_process_file(const hintpath_map::iterator hint_dir_itr, const std::string filepath); - int process_file(hasher::B2H &parent_dir_hash, const std::string &filepath, const std::string &htree_dirpath); - int update_hashtree_entry(hasher::B2H &parent_dir_hash, const bool old_bhmap_exists, const hasher::B2H old_file_hash, const hasher::B2H new_file_hash, const std::string &bhmap_file, const std::string &relpath); - void populate_hint_paths_from_idx_file(const char *const idxfile); - void insert_hint_path(const std::string &relpath); - bool get_hinteddir_match(hintpath_map::iterator &match_itr, const std::string &dirpath); - -public: - hashtree_builder(const state_dir_context &ctx); - int generate(hasher::B2H &root_hash); - int generate(hasher::B2H &root_hash, const bool force_all); - int generate(hasher::B2H &root_hash, const std::unordered_map> &touched_files); -}; - -} // namespace statefs - -#endif diff --git a/src/statefs/state_common.cpp b/src/statefs/state_common.cpp deleted file mode 100644 index ba55755a..00000000 --- a/src/statefs/state_common.cpp +++ /dev/null @@ -1,61 +0,0 @@ -#include -#include -#include "state_common.hpp" - -namespace statefs -{ - -std::string state_hist_dir; -state_dir_context current_ctx; - -void init(const std::string &state_hist_dir_root) -{ - state_hist_dir = realpath(state_hist_dir_root.c_str(), NULL); - - // Initialize 0 state (current state) directory. - current_ctx = get_state_dir_context(0, true); -} - -std::string get_state_dir_root(const int16_t checkpoint_id) -{ - return state_hist_dir + "/" + std::to_string(checkpoint_id); -} - -state_dir_context get_state_dir_context(const int16_t checkpoint_id, const bool create_dirs) -{ - state_dir_context ctx; - ctx.root_dir = get_state_dir_root(checkpoint_id); - ctx.data_dir = ctx.root_dir + DATA_DIR; - ctx.block_hashmap_dir = ctx.root_dir + BHMAP_DIR; - ctx.hashtree_dir = ctx.root_dir + HTREE_DIR; - ctx.delta_dir = ctx.root_dir + DELTA_DIR; - - if (create_dirs) - { - if (!boost::filesystem::exists(ctx.data_dir)) - boost::filesystem::create_directories(ctx.data_dir); - if (!boost::filesystem::exists(ctx.block_hashmap_dir)) - boost::filesystem::create_directories(ctx.block_hashmap_dir); - if (!boost::filesystem::exists(ctx.hashtree_dir)) - boost::filesystem::create_directories(ctx.hashtree_dir); - if (!boost::filesystem::exists(ctx.delta_dir)) - boost::filesystem::create_directories(ctx.delta_dir); - } - - return ctx; -} - -std::string get_relpath(const std::string &fullpath, const std::string &base_path) -{ - std::string relpath = fullpath.substr(base_path.length(), fullpath.length() - base_path.length()); - if (relpath.empty()) - relpath = "/"; - return relpath; -} - -std::string switch_base_path(const std::string &fullpath, const std::string &from_base_path, const std::string &to_base_path) -{ - return to_base_path + get_relpath(fullpath, from_base_path); -} - -} // namespace statefs \ No newline at end of file diff --git a/src/statefs/state_common.hpp b/src/statefs/state_common.hpp deleted file mode 100644 index 09e1a268..00000000 --- a/src/statefs/state_common.hpp +++ /dev/null @@ -1,67 +0,0 @@ -#ifndef _HP_STATEFS_STATE_COMMON_ -#define _HP_STATEFS_STATE_COMMON_ - -#include -#include -#include "hasher.hpp" - -namespace statefs -{ - -// Max number of state history checkpoints to keep. -constexpr int16_t MAX_CHECKPOINTS = 5; - -// Cache block size. -constexpr size_t BLOCK_SIZE = 4 * 1024 * 1024; // 4MB - -// Cache block index entry bytes length. -constexpr size_t BLOCK_INDEX_ENTRY_SIZE = 44; - -// Permissions used when creating block cache and index files. -constexpr int FILE_PERMS = 0644; - -const char *const BLOCK_HASHMAP_EXT = ".bhmap"; -constexpr size_t BLOCK_HASHMAP_EXT_LEN = 6; - -const char *const BLOCK_INDEX_EXT = ".bindex"; -constexpr size_t BLOCK_INDEX_EXT_LEN = 7; - -const char *const BLOCK_CACHE_EXT = ".bcache"; -constexpr size_t BLOCK_CACHE_EXT_LEN = 7; - -const char *const IDX_NEW_FILES = "/idxnew.idx"; -const char *const IDX_TOUCHED_FILES = "/idxtouched.idx"; -const char *const DIR_HASH_FNAME = "dir.hash"; - -const char *const DATA_DIR = "/data"; -const char *const BHMAP_DIR = "/bhmap"; -const char *const HTREE_DIR = "/htree"; -const char *const DELTA_DIR = "/delta"; - -/** - * Context struct to hold all state-related directory paths. - */ -struct state_dir_context -{ - std::string root_dir; // Directory holding state sub dirs. - std::string data_dir; // Directory containing smart contract data. - std::string block_hashmap_dir; // Directory containing block hash map files. - std::string hashtree_dir; // Directory containing hash tree files (dir.hash and hard links). - std::string delta_dir; // Directory containing original smart contract data. -}; - -// Container directory to contain all checkpoints. -extern std::string state_hist_dir; - -// Currently loaded state checkpoint directory context (usually checkpoint 0) -extern state_dir_context current_ctx; - -void init(const std::string &state_hist_dir_root); -std::string get_state_dir_root(const int16_t checkpoint_id); -state_dir_context get_state_dir_context(int16_t checkpoint_id = 0, bool create_dirs = false); -std::string get_relpath(const std::string &fullpath, const std::string &base_path); -std::string switch_base_path(const std::string &fullpath, const std::string &from_base_path, const std::string &to_base_path); - -} // namespace statefs - -#endif \ No newline at end of file diff --git a/src/statefs/state_monitor/fusefs.cpp b/src/statefs/state_monitor/fusefs.cpp deleted file mode 100644 index dd8584c9..00000000 --- a/src/statefs/state_monitor/fusefs.cpp +++ /dev/null @@ -1,1380 +0,0 @@ -/* - * Code copied and adopted from https://github.com/libfuse/libfuse/blob/master/example/passthrough_hp.cc - */ - -#define FUSE_USE_VERSION 35 - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -#ifndef _GNU_SOURCE -#define _GNU_SOURCE -#endif - -// C includes -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -// C++ includes -#include -#include -#include - -#include "../../pchheader.hpp" -#include "../state_common.hpp" -#include "state_monitor.hpp" - -using namespace std; - -// Uniquely identifies a file in the source directory tree. This could -// be simplified to just ino_t since we require the source directory -// not to contain any mountpoints. This hasn't been done yet in case -// we need to reconsider this constraint (but relaxing this would have -// the drawback that we can no longer re-use inode numbers, and thus -// readdir() would need to do a full lookup() in order to report the -// right inode number). -typedef std::pair SrcId; - -// Define a hash function for SrcId -namespace std -{ -template <> -struct hash -{ - size_t operator()(const SrcId &id) const - { - return hash{}(id.first) ^ hash{}(id.second); - } -}; -} // namespace std - -namespace helpers -{ - -int getfilepath(std::string &filepath, int parentfd, const char *filename) -{ - // Get parent directory path using the parentfd. - char proclnk[32]; - char parentpath[PATH_MAX]; - sprintf(proclnk, "/proc/self/fd/%d", parentfd); - ssize_t parentlen = readlink(proclnk, parentpath, PATH_MAX); - if (parentlen > 0) - { - // Concat parent dir path and filename to get the full path. - filepath.reserve(parentlen + strlen(filename) + 1); - filepath.append(parentpath, parentlen).append("/").append(filename); - return 0; - } - return -1; -} - -} // namespace helpers - -namespace fusefs -{ - -/* We are re-using pointers to our `struct sfs_inode` and `struct - sfs_dirp` elements as inodes and file handles. This means that we - must be able to store pointer a pointer in both a fuse_ino_t - variable and a uint64_t variable (used for file handles). */ -static_assert(sizeof(fuse_ino_t) >= sizeof(void *), - "void* must fit into fuse_ino_t"); -static_assert(sizeof(fuse_ino_t) >= sizeof(uint64_t), - "fuse_ino_t must be at least 64 bits"); - -/* Forward declarations */ -struct Inode; -static Inode &get_inode(fuse_ino_t ino); -static void forget_one(fuse_ino_t ino, uint64_t n); - -// Maps files in the source directory tree to inodes -typedef std::unordered_map InodeMap; - -struct Inode -{ - int fd{-1}; - bool is_symlink{false}; - dev_t src_dev{0}; - ino_t src_ino{0}; - uint64_t nlookup{0}; - std::mutex m; - - // Delete copy constructor and assignments. We could implement - // move if we need it. - Inode() = default; - Inode(const Inode &) = delete; - Inode(Inode &&inode) = delete; - Inode &operator=(Inode &&inode) = delete; - Inode &operator=(const Inode &) = delete; - - ~Inode() - { - if (fd > 0) - close(fd); - } -}; - -struct Fs -{ - // Must be acquired *after* any Inode.m locks. - std::mutex mutex; - InodeMap inodes; // protected by mutex - Inode root; - double timeout; - bool debug; - std::string source; - size_t blocksize; - dev_t src_dev; - bool nosplice; - bool nocache; -}; -static Fs fs{}; -static statefs::state_monitor statemonitor; - -#define FUSE_BUF_COPY_FLAGS \ - (fs.nosplice ? FUSE_BUF_NO_SPLICE : static_cast(0)) - -static Inode &get_inode(fuse_ino_t ino) -{ - if (ino == FUSE_ROOT_ID) - return fs.root; - - Inode *inode = reinterpret_cast(ino); - if (inode->fd == -1) - { - cerr << "INTERNAL ERROR: Unknown inode " << ino << endl; - abort(); - } - return *inode; -} - -static int get_fs_fd(fuse_ino_t ino) -{ - int fd = get_inode(ino).fd; - return fd; -} - -static void sfs_init(void *userdata, fuse_conn_info *conn) -{ - (void)userdata; - if (conn->capable & FUSE_CAP_EXPORT_SUPPORT) - conn->want |= FUSE_CAP_EXPORT_SUPPORT; - - if (fs.timeout && conn->capable & FUSE_CAP_WRITEBACK_CACHE) - conn->want |= FUSE_CAP_WRITEBACK_CACHE; - - if (conn->capable & FUSE_CAP_FLOCK_LOCKS) - conn->want |= FUSE_CAP_FLOCK_LOCKS; - - // Use splicing if supported. Since we are using writeback caching - // and readahead, individual requests should have a decent size so - // that splicing between fd's is well worth it. - if (conn->capable & FUSE_CAP_SPLICE_WRITE && !fs.nosplice) - conn->want |= FUSE_CAP_SPLICE_WRITE; - if (conn->capable & FUSE_CAP_SPLICE_READ && !fs.nosplice) - conn->want |= FUSE_CAP_SPLICE_READ; -} - -static void sfs_getattr(fuse_req_t req, fuse_ino_t ino, fuse_file_info *fi) -{ - (void)fi; - Inode &inode = get_inode(ino); - struct stat attr; - auto res = fstatat(inode.fd, "", &attr, - AT_EMPTY_PATH | AT_SYMLINK_NOFOLLOW); - if (res == -1) - { - fuse_reply_err(req, errno); - return; - } - fuse_reply_attr(req, &attr, fs.timeout); -} - -#ifdef HAVE_UTIMENSAT -static int utimensat_empty_nofollow(Inode &inode, - const struct timespec *tv) -{ - if (inode.is_symlink) - { - /* Does not work on current kernels, but may in the future: - https://marc.info/?l=linux-kernel&m=154158217810354&w=2 */ - auto res = utimensat(inode.fd, "", tv, AT_EMPTY_PATH | AT_SYMLINK_NOFOLLOW); - if (res == -1 && errno == EINVAL) - { - /* Sorry, no race free way to set times on symlink. */ - errno = EPERM; - } - return res; - } - - char procname[64]; - sprintf(procname, "/proc/self/fd/%i", inode.fd); - - return utimensat(AT_FDCWD, procname, tv, 0); -} -#endif - -static void do_setattr(fuse_req_t req, fuse_ino_t ino, struct stat *attr, - int valid, struct fuse_file_info *fi) -{ - Inode &inode = get_inode(ino); - int ifd = inode.fd; - int res; - - if (valid & FUSE_SET_ATTR_MODE) - { - if (fi) - { - res = fchmod(fi->fh, attr->st_mode); - } - else - { - char procname[64]; - sprintf(procname, "/proc/self/fd/%i", ifd); - res = chmod(procname, attr->st_mode); - } - if (res == -1) - goto out_err; - } - if (valid & (FUSE_SET_ATTR_UID | FUSE_SET_ATTR_GID)) - { - uid_t uid = (valid & FUSE_SET_ATTR_UID) ? attr->st_uid : static_cast(-1); - gid_t gid = (valid & FUSE_SET_ATTR_GID) ? attr->st_gid : static_cast(-1); - - res = fchownat(ifd, "", uid, gid, AT_EMPTY_PATH | AT_SYMLINK_NOFOLLOW); - if (res == -1) - goto out_err; - } - if (valid & FUSE_SET_ATTR_SIZE) - { - if (fi) - { - res = ftruncate(fi->fh, attr->st_size); - } - else - { - char procname[64]; - sprintf(procname, "/proc/self/fd/%i", ifd); - res = truncate(procname, attr->st_size); - } - if (res == -1) - goto out_err; - } - if (valid & (FUSE_SET_ATTR_ATIME | FUSE_SET_ATTR_MTIME)) - { - struct timespec tv[2]; - - tv[0].tv_sec = 0; - tv[1].tv_sec = 0; - tv[0].tv_nsec = UTIME_OMIT; - tv[1].tv_nsec = UTIME_OMIT; - - if (valid & FUSE_SET_ATTR_ATIME_NOW) - tv[0].tv_nsec = UTIME_NOW; - else if (valid & FUSE_SET_ATTR_ATIME) - tv[0] = attr->st_atim; - - if (valid & FUSE_SET_ATTR_MTIME_NOW) - tv[1].tv_nsec = UTIME_NOW; - else if (valid & FUSE_SET_ATTR_MTIME) - tv[1] = attr->st_mtim; - - if (fi) - res = futimens(fi->fh, tv); - else - { -#ifdef HAVE_UTIMENSAT - res = utimensat_empty_nofollow(inode, tv); -#else - res = -1; - errno = EOPNOTSUPP; -#endif - } - if (res == -1) - goto out_err; - } - return sfs_getattr(req, ino, fi); - -out_err: - fuse_reply_err(req, errno); -} - -static void sfs_setattr(fuse_req_t req, fuse_ino_t ino, struct stat *attr, - int valid, fuse_file_info *fi) -{ - // We use some conditions to detect truncate call. - if (fi != NULL && fi->fh > 0 && attr->st_size > 0) - statemonitor.ontruncate(fi->fh, attr->st_size); - - (void)ino; - do_setattr(req, ino, attr, valid, fi); -} - -static int do_lookup(fuse_ino_t parent, const char *name, - fuse_entry_param *e) -{ - if (fs.debug) - cerr << "DEBUG: lookup(): name=" << name - << ", parent=" << parent << endl; - memset(e, 0, sizeof(*e)); - e->attr_timeout = fs.timeout; - e->entry_timeout = fs.timeout; - - auto newfd = openat(get_fs_fd(parent), name, O_PATH | O_NOFOLLOW); - if (newfd == -1) - return errno; - - auto res = fstatat(newfd, "", &e->attr, AT_EMPTY_PATH | AT_SYMLINK_NOFOLLOW); - if (res == -1) - { - auto saveerr = errno; - close(newfd); - if (fs.debug) - cerr << "DEBUG: lookup(): fstatat failed" << endl; - return saveerr; - } - - if (e->attr.st_dev != fs.src_dev) - { - cerr << "WARNING: Mountpoints in the source directory tree will be hidden." << endl; - return ENOTSUP; - } - else if (e->attr.st_ino == FUSE_ROOT_ID) - { - cerr << "ERROR: Source directory tree must not include inode " - << FUSE_ROOT_ID << endl; - return EIO; - } - - SrcId id{e->attr.st_ino, e->attr.st_dev}; - unique_lock fs_lock{fs.mutex}; - Inode *inode_p; - try - { - inode_p = &fs.inodes[id]; - } - catch (std::bad_alloc &) - { - return ENOMEM; - } - e->ino = reinterpret_cast(inode_p); - Inode &inode{*inode_p}; - - if (inode.fd != -1) - { // found existing inode - fs_lock.unlock(); - if (fs.debug) - cerr << "DEBUG: lookup(): inode " << e->attr.st_ino - << " (userspace) already known." << endl; - lock_guard g{inode.m}; - inode.nlookup++; - close(newfd); - } - else - { // no existing inode - /* This is just here to make Helgrind happy. It violates the - lock ordering requirement (inode.m must be acquired before - fs.mutex), but this is of no consequence because at this - point no other thread has access to the inode mutex */ - lock_guard g{inode.m}; - inode.src_ino = e->attr.st_ino; - inode.src_dev = e->attr.st_dev; - inode.is_symlink = S_ISLNK(e->attr.st_mode); - inode.nlookup = 1; - inode.fd = newfd; - fs_lock.unlock(); - - if (fs.debug) - cerr << "DEBUG: lookup(): created userspace inode " << e->attr.st_ino - << endl; - } - - return 0; -} - -static void sfs_lookup(fuse_req_t req, fuse_ino_t parent, const char *name) -{ - fuse_entry_param e{}; - auto err = do_lookup(parent, name, &e); - if (err == ENOENT) - { - e.attr_timeout = fs.timeout; - e.entry_timeout = fs.timeout; - e.ino = e.attr.st_ino = 0; - fuse_reply_entry(req, &e); - } - else if (err) - { - if (err == ENFILE || err == EMFILE) - cerr << "ERROR: Reached maximum number of file descriptors." << endl; - fuse_reply_err(req, err); - } - else - { - fuse_reply_entry(req, &e); - } -} - -static void mknod_symlink(fuse_req_t req, fuse_ino_t parent, - const char *name, mode_t mode, dev_t rdev, - const char *link) -{ - int res; - Inode &inode_p = get_inode(parent); - auto saverr = ENOMEM; - - if (S_ISDIR(mode)) - res = mkdirat(inode_p.fd, name, mode); - else if (S_ISLNK(mode)) - res = symlinkat(link, inode_p.fd, name); - else - res = mknodat(inode_p.fd, name, mode, rdev); - saverr = errno; - if (res == -1) - goto out; - - fuse_entry_param e; - saverr = do_lookup(parent, name, &e); - if (saverr) - goto out; - - fuse_reply_entry(req, &e); - return; - -out: - if (saverr == ENFILE || saverr == EMFILE) - cerr << "ERROR: Reached maximum number of file descriptors." << endl; - fuse_reply_err(req, saverr); -} - -static void sfs_mknod(fuse_req_t req, fuse_ino_t parent, const char *name, - mode_t mode, dev_t rdev) -{ - mknod_symlink(req, parent, name, mode, rdev, nullptr); -} - -static void sfs_mkdir(fuse_req_t req, fuse_ino_t parent, const char *name, - mode_t mode) -{ - mknod_symlink(req, parent, name, S_IFDIR | mode, 0, nullptr); -} - -static void sfs_symlink(fuse_req_t req, const char *link, fuse_ino_t parent, - const char *name) -{ - mknod_symlink(req, parent, name, S_IFLNK, 0, link); -} - -static int linkat_empty_nofollow(Inode &inode, int dfd, const char *name) -{ - if (inode.is_symlink) - { - auto res = linkat(inode.fd, "", dfd, name, AT_EMPTY_PATH); - if (res == -1 && (errno == ENOENT || errno == EINVAL)) - { - /* Sorry, no race free way to hard-link a symlink. */ - errno = EOPNOTSUPP; - } - return res; - } - - char procname[64]; - sprintf(procname, "/proc/self/fd/%i", inode.fd); - return linkat(AT_FDCWD, procname, dfd, name, AT_SYMLINK_FOLLOW); -} - -static void sfs_link(fuse_req_t req, fuse_ino_t ino, fuse_ino_t parent, - const char *name) -{ - Inode &inode = get_inode(ino); - Inode &inode_p = get_inode(parent); - fuse_entry_param e{}; - - e.attr_timeout = fs.timeout; - e.entry_timeout = fs.timeout; - - auto res = linkat_empty_nofollow(inode, inode_p.fd, name); - if (res == -1) - { - fuse_reply_err(req, errno); - return; - } - - res = fstatat(inode.fd, "", &e.attr, AT_EMPTY_PATH | AT_SYMLINK_NOFOLLOW); - if (res == -1) - { - fuse_reply_err(req, errno); - return; - } - e.ino = reinterpret_cast(&inode); - { - lock_guard g{inode.m}; - inode.nlookup++; - } - - fuse_reply_entry(req, &e); - return; -} - -static void sfs_rmdir(fuse_req_t req, fuse_ino_t parent, const char *name) -{ - Inode &inode_p = get_inode(parent); - lock_guard g{inode_p.m}; - auto res = unlinkat(inode_p.fd, name, AT_REMOVEDIR); - fuse_reply_err(req, res == -1 ? errno : 0); -} - -static void sfs_rename(fuse_req_t req, fuse_ino_t parent, const char *name, - fuse_ino_t newparent, const char *newname, - unsigned int flags) -{ - Inode &inode_p = get_inode(parent); - Inode &inode_np = get_inode(newparent); - if (flags) - { - fuse_reply_err(req, EINVAL); - return; - } - - // state monitor hook. - std::string oldfilepath, newfilepath; - if (helpers::getfilepath(oldfilepath, inode_p.fd, name) == 0 && - helpers::getfilepath(newfilepath, inode_np.fd, newname) == 0) - { - statemonitor.onrename(oldfilepath, newfilepath); - } - - auto res = renameat(inode_p.fd, name, inode_np.fd, newname); - fuse_reply_err(req, res == -1 ? errno : 0); -} - -static void sfs_unlink(fuse_req_t req, fuse_ino_t parent, const char *name) -{ - Inode &inode_p = get_inode(parent); - - // state monitor hook. - std::string filepath; - if (helpers::getfilepath(filepath, inode_p.fd, name) == 0) - statemonitor.ondelete(filepath); - - auto res = unlinkat(inode_p.fd, name, 0); - fuse_reply_err(req, res == -1 ? errno : 0); -} - -static void forget_one(fuse_ino_t ino, uint64_t n) -{ - Inode &inode = get_inode(ino); - unique_lock l{inode.m}; - - if (n > inode.nlookup) - { - cerr << "INTERNAL ERROR: Negative lookup count for inode " - << inode.src_ino << endl; - abort(); - } - inode.nlookup -= n; - if (!inode.nlookup) - { - if (fs.debug) - cerr << "DEBUG: forget: cleaning up inode " << inode.src_ino << endl; - { - lock_guard g_fs{fs.mutex}; - l.unlock(); - fs.inodes.erase({inode.src_ino, inode.src_dev}); - } - } - else if (fs.debug) - cerr << "DEBUG: forget: inode " << inode.src_ino - << " lookup count now " << inode.nlookup << endl; -} - -static void sfs_forget(fuse_req_t req, fuse_ino_t ino, uint64_t nlookup) -{ - forget_one(ino, nlookup); - fuse_reply_none(req); -} - -static void sfs_forget_multi(fuse_req_t req, size_t count, - fuse_forget_data *forgets) -{ - for (int i = 0; i < count; i++) - forget_one(forgets[i].ino, forgets[i].nlookup); - fuse_reply_none(req); -} - -static void sfs_readlink(fuse_req_t req, fuse_ino_t ino) -{ - Inode &inode = get_inode(ino); - char buf[PATH_MAX + 1]; - auto res = readlinkat(inode.fd, "", buf, sizeof(buf)); - if (res == -1) - fuse_reply_err(req, errno); - else if (res == sizeof(buf)) - fuse_reply_err(req, ENAMETOOLONG); - else - { - buf[res] = '\0'; - fuse_reply_readlink(req, buf); - } -} - -struct DirHandle -{ - DIR *dp{nullptr}; - off_t offset; - - DirHandle() = default; - DirHandle(const DirHandle &) = delete; - DirHandle &operator=(const DirHandle &) = delete; - - ~DirHandle() - { - if (dp) - closedir(dp); - } -}; - -static DirHandle *get_dir_handle(fuse_file_info *fi) -{ - return reinterpret_cast(fi->fh); -} - -static void sfs_opendir(fuse_req_t req, fuse_ino_t ino, fuse_file_info *fi) -{ - Inode &inode = get_inode(ino); - auto d = new (nothrow) DirHandle; - if (d == nullptr) - { - fuse_reply_err(req, ENOMEM); - return; - } - - // Make Helgrind happy - it can't know that there's an implicit - // synchronization due to the fact that other threads cannot - // access d until we've called fuse_reply_*. - lock_guard g{inode.m}; - - auto fd = openat(inode.fd, ".", O_RDONLY); - if (fd == -1) - goto out_errno; - - // On success, dir stream takes ownership of fd, so we - // do not have to close it. - d->dp = fdopendir(fd); - if (d->dp == nullptr) - goto out_errno; - - d->offset = 0; - - fi->fh = reinterpret_cast(d); - if (fs.timeout) - { - fi->keep_cache = 1; - fi->cache_readdir = 1; - } - fuse_reply_open(req, fi); - return; - -out_errno: - auto error = errno; - delete d; - if (error == ENFILE || error == EMFILE) - cerr << "ERROR: Reached maximum number of file descriptors." << endl; - fuse_reply_err(req, error); -} - -static bool is_dot_or_dotdot(const char *name) -{ - return name[0] == '.' && - (name[1] == '\0' || (name[1] == '.' && name[2] == '\0')); -} - -static void do_readdir(fuse_req_t req, fuse_ino_t ino, size_t size, - off_t offset, fuse_file_info *fi, int plus) -{ - auto d = get_dir_handle(fi); - Inode &inode = get_inode(ino); - lock_guard g{inode.m}; - char *p; - auto rem = size; - int err = 0, count = 0; - - if (fs.debug) - cerr << "DEBUG: readdir(): started with offset " - << offset << endl; - - auto buf = new (nothrow) char[size]; - if (!buf) - { - fuse_reply_err(req, ENOMEM); - return; - } - p = buf; - - if (offset != d->offset) - { - if (fs.debug) - cerr << "DEBUG: readdir(): seeking to " << offset << endl; - seekdir(d->dp, offset); - d->offset = offset; - } - - while (1) - { - struct dirent *entry; - errno = 0; - entry = readdir(d->dp); - if (!entry) - { - if (errno) - { - err = errno; - if (fs.debug) - warn("DEBUG: readdir(): readdir failed with"); - goto error; - } - break; // End of stream - } - d->offset = entry->d_off; - if (is_dot_or_dotdot(entry->d_name)) - continue; - - fuse_entry_param e{}; - size_t entsize; - if (plus) - { - err = do_lookup(ino, entry->d_name, &e); - if (err) - goto error; - entsize = fuse_add_direntry_plus(req, p, rem, entry->d_name, &e, entry->d_off); - - if (entsize > rem) - { - if (fs.debug) - cerr << "DEBUG: readdir(): buffer full, returning data. " << endl; - forget_one(e.ino, 1); - break; - } - } - else - { - e.attr.st_ino = entry->d_ino; - e.attr.st_mode = entry->d_type << 12; - entsize = fuse_add_direntry(req, p, rem, entry->d_name, &e.attr, entry->d_off); - - if (entsize > rem) - { - if (fs.debug) - cerr << "DEBUG: readdir(): buffer full, returning data. " << endl; - break; - } - } - - p += entsize; - rem -= entsize; - count++; - if (fs.debug) - { - cerr << "DEBUG: readdir(): added to buffer: " << entry->d_name - << ", ino " << e.attr.st_ino << ", offset " << entry->d_off << endl; - } - } - err = 0; -error: - - // If there's an error, we can only signal it if we haven't stored - // any entries yet - otherwise we'd end up with wrong lookup - // counts for the entries that are already in the buffer. So we - // return what we've collected until that point. - if (err && rem == size) - { - if (err == ENFILE || err == EMFILE) - cerr << "ERROR: Reached maximum number of file descriptors." << endl; - fuse_reply_err(req, err); - } - else - { - if (fs.debug) - cerr << "DEBUG: readdir(): returning " << count - << " entries, curr offset " << d->offset << endl; - fuse_reply_buf(req, buf, size - rem); - } - delete[] buf; - return; -} - -static void sfs_readdir(fuse_req_t req, fuse_ino_t ino, size_t size, - off_t offset, fuse_file_info *fi) -{ - // operation logging is done in readdir to reduce code duplication - do_readdir(req, ino, size, offset, fi, 0); -} - -static void sfs_readdirplus(fuse_req_t req, fuse_ino_t ino, size_t size, - off_t offset, fuse_file_info *fi) -{ - // operation logging is done in readdir to reduce code duplication - do_readdir(req, ino, size, offset, fi, 1); -} - -static void sfs_releasedir(fuse_req_t req, fuse_ino_t ino, fuse_file_info *fi) -{ - (void)ino; - auto d = get_dir_handle(fi); - delete d; - fuse_reply_err(req, 0); -} - -static void sfs_create(fuse_req_t req, fuse_ino_t parent, const char *name, - mode_t mode, fuse_file_info *fi) -{ - Inode &inode_p = get_inode(parent); - - auto fd = openat(inode_p.fd, name, - (fi->flags | O_CREAT) & ~O_NOFOLLOW, mode); - if (fd == -1) - { - auto err = errno; - if (err == ENFILE || err == EMFILE) - cerr << "ERROR: Reached maximum number of file descriptors." << endl; - fuse_reply_err(req, err); - return; - } - - fi->fh = fd; - fuse_entry_param e; - auto err = do_lookup(parent, name, &e); - if (err) - { - if (err == ENFILE || err == EMFILE) - cerr << "ERROR: Reached maximum number of file descriptors." << endl; - fuse_reply_err(req, err); - } - else - { - // state monitor hook. - statemonitor.oncreate(fd); - fuse_reply_create(req, &e, fi); - } -} - -static void sfs_fsyncdir(fuse_req_t req, fuse_ino_t ino, int datasync, - fuse_file_info *fi) -{ - (void)ino; - int res; - int fd = dirfd(get_dir_handle(fi)->dp); - if (datasync) - res = fdatasync(fd); - else - res = fsync(fd); - fuse_reply_err(req, res == -1 ? errno : 0); -} - -static void sfs_open(fuse_req_t req, fuse_ino_t ino, fuse_file_info *fi) -{ - Inode &inode = get_inode(ino); - - /* With writeback cache, kernel may send read requests even - when userspace opened write-only */ - if (fs.timeout && (fi->flags & O_ACCMODE) == O_WRONLY) - { - fi->flags &= ~O_ACCMODE; - fi->flags |= O_RDWR; - } - - /* With writeback cache, O_APPEND is handled by the kernel. This - breaks atomicity (since the file may change in the underlying - filesystem, so that the kernel's idea of the end of the file - isn't accurate anymore). However, no process should modify the - file in the underlying filesystem once it has been read, so - this is not a problem. */ - if (fs.timeout && fi->flags & O_APPEND) - fi->flags &= ~O_APPEND; - - /* Unfortunately we cannot use inode.fd, because this was opened - with O_PATH (so it doesn't allow read/write access). */ - char buf[64]; - sprintf(buf, "/proc/self/fd/%i", inode.fd); - - // state monitor hook. - statemonitor.onopen(inode.fd, fi->flags); - - auto fd = open(buf, fi->flags & ~O_NOFOLLOW); - if (fd == -1) - { - auto err = errno; - if (err == ENFILE || err == EMFILE) - cerr << "ERROR: Reached maximum number of file descriptors." << endl; - fuse_reply_err(req, err); - return; - } - - fi->keep_cache = (fs.timeout != 0); - fi->fh = fd; - - fuse_reply_open(req, fi); -} - -static void sfs_release(fuse_req_t req, fuse_ino_t ino, fuse_file_info *fi) -{ - (void)ino; - close(fi->fh); - - // state monitor hook. - statemonitor.onclose(fi->fh); - - fuse_reply_err(req, 0); -} - -static void sfs_flush(fuse_req_t req, fuse_ino_t ino, fuse_file_info *fi) -{ - (void)ino; - auto res = close(dup(fi->fh)); - fuse_reply_err(req, res == -1 ? errno : 0); -} - -static void sfs_fsync(fuse_req_t req, fuse_ino_t ino, int datasync, - fuse_file_info *fi) -{ - (void)ino; - int res; - if (datasync) - res = fdatasync(fi->fh); - else - res = fsync(fi->fh); - fuse_reply_err(req, res == -1 ? errno : 0); -} - -static void do_read(fuse_req_t req, size_t size, off_t off, fuse_file_info *fi) -{ - - fuse_bufvec buf = FUSE_BUFVEC_INIT(size); - buf.buf[0].flags = static_cast( - FUSE_BUF_IS_FD | FUSE_BUF_FD_SEEK); - buf.buf[0].fd = fi->fh; - buf.buf[0].pos = off; - - fuse_reply_data(req, &buf, FUSE_BUF_COPY_FLAGS); -} - -static void sfs_read(fuse_req_t req, fuse_ino_t ino, size_t size, off_t off, - fuse_file_info *fi) -{ - (void)ino; - do_read(req, size, off, fi); -} - -static void do_write_buf(fuse_req_t req, size_t size, off_t off, - fuse_bufvec *in_buf, fuse_file_info *fi) -{ - fuse_bufvec out_buf = FUSE_BUFVEC_INIT(size); - out_buf.buf[0].flags = static_cast( - FUSE_BUF_IS_FD | FUSE_BUF_FD_SEEK); - out_buf.buf[0].fd = fi->fh; - out_buf.buf[0].pos = off; - - auto res = fuse_buf_copy(&out_buf, in_buf, FUSE_BUF_COPY_FLAGS); - if (res < 0) - fuse_reply_err(req, -res); - else - fuse_reply_write(req, (size_t)res); -} - -static void sfs_write_buf(fuse_req_t req, fuse_ino_t ino, fuse_bufvec *in_buf, - off_t off, fuse_file_info *fi) -{ - (void)ino; - auto size{fuse_buf_size(in_buf)}; - - // state monitor hook. - statemonitor.onwrite(fi->fh, off, size); - - do_write_buf(req, size, off, in_buf, fi); -} - -static void sfs_statfs(fuse_req_t req, fuse_ino_t ino) -{ - struct statvfs stbuf; - - auto res = fstatvfs(get_fs_fd(ino), &stbuf); - if (res == -1) - fuse_reply_err(req, errno); - else - fuse_reply_statfs(req, &stbuf); -} - -#ifdef HAVE_POSIX_FALLOCATE -static void sfs_fallocate(fuse_req_t req, fuse_ino_t ino, int mode, - off_t offset, off_t length, fuse_file_info *fi) -{ - (void)ino; - if (mode) - { - fuse_reply_err(req, EOPNOTSUPP); - return; - } - - auto err = posix_fallocate(fi->fh, offset, length); - fuse_reply_err(req, err); -} -#endif - -static void sfs_flock(fuse_req_t req, fuse_ino_t ino, fuse_file_info *fi, - int op) -{ - (void)ino; - auto res = flock(fi->fh, op); - fuse_reply_err(req, res == -1 ? errno : 0); -} - -#ifdef HAVE_SETXATTR -static void sfs_getxattr(fuse_req_t req, fuse_ino_t ino, const char *name, - size_t size) -{ - char *value = nullptr; - Inode &inode = get_inode(ino); - ssize_t ret; - int saverr; - - if (inode.is_symlink) - { - /* Sorry, no race free way to getxattr on symlink. */ - saverr = ENOTSUP; - goto out; - } - - char procname[64]; - sprintf(procname, "/proc/self/fd/%i", inode.fd); - - if (size) - { - value = new (nothrow) char[size]; - if (value == nullptr) - { - saverr = ENOMEM; - goto out; - } - - ret = getxattr(procname, name, value, size); - if (ret == -1) - goto out_err; - saverr = 0; - if (ret == 0) - goto out; - - fuse_reply_buf(req, value, ret); - } - else - { - ret = getxattr(procname, name, nullptr, 0); - if (ret == -1) - goto out_err; - - fuse_reply_xattr(req, ret); - } -out_free: - delete[] value; - return; - -out_err: - saverr = errno; -out: - fuse_reply_err(req, saverr); - goto out_free; -} - -static void sfs_listxattr(fuse_req_t req, fuse_ino_t ino, size_t size) -{ - char *value = nullptr; - Inode &inode = get_inode(ino); - ssize_t ret; - int saverr; - - if (inode.is_symlink) - { - /* Sorry, no race free way to listxattr on symlink. */ - saverr = ENOTSUP; - goto out; - } - - char procname[64]; - sprintf(procname, "/proc/self/fd/%i", inode.fd); - - if (size) - { - value = new (nothrow) char[size]; - if (value == nullptr) - { - saverr = ENOMEM; - goto out; - } - - ret = listxattr(procname, value, size); - if (ret == -1) - goto out_err; - saverr = 0; - if (ret == 0) - goto out; - - fuse_reply_buf(req, value, ret); - } - else - { - ret = listxattr(procname, nullptr, 0); - if (ret == -1) - goto out_err; - - fuse_reply_xattr(req, ret); - } -out_free: - delete[] value; - return; -out_err: - saverr = errno; -out: - fuse_reply_err(req, saverr); - goto out_free; -} - -static void sfs_setxattr(fuse_req_t req, fuse_ino_t ino, const char *name, - const char *value, size_t size, int flags) -{ - Inode &inode = get_inode(ino); - ssize_t ret; - int saverr; - - if (inode.is_symlink) - { - /* Sorry, no race free way to setxattr on symlink. */ - saverr = ENOTSUP; - goto out; - } - - char procname[64]; - sprintf(procname, "/proc/self/fd/%i", inode.fd); - - ret = setxattr(procname, name, value, size, flags); - saverr = ret == -1 ? errno : 0; - -out: - fuse_reply_err(req, saverr); -} - -static void sfs_removexattr(fuse_req_t req, fuse_ino_t ino, const char *name) -{ - char procname[64]; - Inode &inode = get_inode(ino); - ssize_t ret; - int saverr; - - if (inode.is_symlink) - { - /* Sorry, no race free way to setxattr on symlink. */ - saverr = ENOTSUP; - goto out; - } - - sprintf(procname, "/proc/self/fd/%i", inode.fd); - ret = removexattr(procname, name); - saverr = ret == -1 ? errno : 0; - -out: - fuse_reply_err(req, saverr); -} -#endif - -static void assign_operations(fuse_lowlevel_ops &sfs_oper) -{ - sfs_oper.init = sfs_init; - sfs_oper.lookup = sfs_lookup; - sfs_oper.mkdir = sfs_mkdir; - sfs_oper.mknod = sfs_mknod; - sfs_oper.symlink = sfs_symlink; - sfs_oper.link = sfs_link; - sfs_oper.unlink = sfs_unlink; - sfs_oper.rmdir = sfs_rmdir; - sfs_oper.rename = sfs_rename; - sfs_oper.forget = sfs_forget; - sfs_oper.forget_multi = sfs_forget_multi; - sfs_oper.getattr = sfs_getattr; - sfs_oper.setattr = sfs_setattr; - sfs_oper.readlink = sfs_readlink; - sfs_oper.opendir = sfs_opendir; - sfs_oper.readdir = sfs_readdir; - sfs_oper.readdirplus = sfs_readdirplus; - sfs_oper.releasedir = sfs_releasedir; - sfs_oper.fsyncdir = sfs_fsyncdir; - sfs_oper.create = sfs_create; - sfs_oper.open = sfs_open; - sfs_oper.release = sfs_release; - sfs_oper.flush = sfs_flush; - sfs_oper.fsync = sfs_fsync; - sfs_oper.read = sfs_read; - sfs_oper.write_buf = sfs_write_buf; - sfs_oper.statfs = sfs_statfs; -#ifdef HAVE_POSIX_FALLOCATE - sfs_oper.fallocate = sfs_fallocate; -#endif - sfs_oper.flock = sfs_flock; -#ifdef HAVE_SETXATTR - sfs_oper.setxattr = sfs_setxattr; - sfs_oper.getxattr = sfs_getxattr; - sfs_oper.listxattr = sfs_listxattr; - sfs_oper.removexattr = sfs_removexattr; -#endif -} - -void maximize_fd_limit() -{ - struct rlimit lim - { - }; - auto res = getrlimit(RLIMIT_NOFILE, &lim); - if (res != 0) - { - warn("WARNING: getrlimit() failed with"); - return; - } - lim.rlim_cur = lim.rlim_max; - res = setrlimit(RLIMIT_NOFILE, &lim); - if (res != 0) - warn("WARNING: setrlimit() failed with"); -} - -/** - * Starts hosting the fuse file system along with the state monitor. - * @param arg0 First CLI argument to be passed into fuse main. - * @param state_hist_dir Hot pocket state history directory. - * @param fuse_mnt_dir Directory to mound the fuse filesystem. - * @return 0 on success. 1 on failure. - */ -int start(const char *arg0, const char *state_hist_dir, const char *fuse_mnt_dir) -{ - // We need an fd for every entry in our the filesystem that the - // kernel knows about. This is way more than most processes need, - // so try to get rid of any resource softlimit. - maximize_fd_limit(); - - // We consider this as the first run of the history dir is empty. - const bool is_first_run = boost::filesystem::is_empty(state_hist_dir); - - statefs::init(state_hist_dir); - statemonitor.ctx = statefs::get_state_dir_context(); - fs.source = statemonitor.ctx.data_dir; - - // Create a checkpoint from the second run onwards. - if (!is_first_run) - statemonitor.create_checkpoint(); - - // Initialize filesystem root - fs.root.fd = -1; - fs.root.nlookup = 9999; - fs.root.is_symlink = false; - - // This is equivalent to specifying --nocache for fuse args. If we need to support caching, - // we need to set this to 86400.0 - fs.timeout = 0; - - struct stat stat; - auto ret = lstat(fs.source.c_str(), &stat); - if (ret == -1) - err(1, "ERROR: failed to stat source (\"%s\")", fs.source.c_str()); - if (!S_ISDIR(stat.st_mode)) - errx(1, "ERROR: source is not a directory"); - fs.src_dev = stat.st_dev; - - fs.root.fd = open(fs.source.c_str(), O_PATH); - if (fs.root.fd == -1) - err(1, "ERROR: open(\"%s\", O_PATH)", fs.source.c_str()); - - // Initialize fuse - fuse_args args = FUSE_ARGS_INIT(0, nullptr); - if (fuse_opt_add_arg(&args, arg0) || - fuse_opt_add_arg(&args, "-o") || - fuse_opt_add_arg(&args, "default_permissions,fsname=hpstatefs") - /*|| fuse_opt_add_arg(&args, "-odebug")*/) - errx(3, "ERROR: Out of memory"); - - fuse_lowlevel_ops sfs_oper{}; - assign_operations(sfs_oper); - auto se = fuse_session_new(&args, &sfs_oper, sizeof(sfs_oper), &fs); - if (se == nullptr) - goto err_out1; - - if (fuse_set_signal_handlers(se) != 0) - goto err_out2; - - // Don't apply umask, use modes exactly as specified - umask(0); - - // Mount and run main loop - struct fuse_loop_config loop_config; - loop_config.clone_fd = 0; - loop_config.max_idle_threads = 10; - if (fuse_session_mount(se, fuse_mnt_dir) != 0) - goto err_out3; - - ret = fuse_session_loop_mt(se, &loop_config); - - fuse_session_unmount(se); - -err_out3: - fuse_remove_signal_handlers(se); -err_out2: - fuse_session_destroy(se); -err_out1: - fuse_opt_free_args(&args); - - return ret ? 1 : 0; -} - -} // namespace fusefs - -int main(int argc, char *argv[]) -{ - if (argc != 3) - { - std::cerr << "Incorrect arguments.\n"; - exit(1); - } - - return fusefs::start(argv[0], argv[1], argv[2]); -} - -namespace boost -{ -/** - * Global exception handler for boost exceptions. - */ -void throw_exception(std::exception const &e) -{ - std::cerr << "Boost error: " << e.what() << "\n" - << boost::stacktrace::stacktrace(); - exit(1); -} - -inline void assertion_failed_msg(char const *expr, char const *msg, char const *function, char const * /*file*/, long /*line*/) -{ - std::cerr << "Expression '" << expr << "' is false in function '" << function << "': " << (msg ? msg : "<...>") << ".\n" - << "Backtrace:\n" - << boost::stacktrace::stacktrace() << '\n'; - std::abort(); -} - -inline void assertion_failed(char const *expr, char const *function, char const *file, long line) -{ - ::boost::assertion_failed_msg(expr, 0 /*nullptr*/, function, file, line); -} -} // namespace boost \ No newline at end of file diff --git a/src/statefs/state_monitor/fusefs.hpp b/src/statefs/state_monitor/fusefs.hpp deleted file mode 100644 index 369e24fe..00000000 --- a/src/statefs/state_monitor/fusefs.hpp +++ /dev/null @@ -1,9 +0,0 @@ -#ifndef _FUSE_FS_ -#define _FUSE_FS_ - -namespace fusefs -{ -int start(const char *arg0, const char *source, const char *mountpoint, const char *delta_dir); -} - -#endif \ No newline at end of file diff --git a/src/statefs/state_monitor/state_monitor.cpp b/src/statefs/state_monitor/state_monitor.cpp deleted file mode 100644 index 228ae9b8..00000000 --- a/src/statefs/state_monitor/state_monitor.cpp +++ /dev/null @@ -1,582 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "../hasher.hpp" -#include "../state_common.hpp" -#include "state_monitor.hpp" - -namespace statefs -{ - -/** - * Creates a new checkpoint directory. This will remove the oldest checkpoint if we have - * reached MAX_CHECKPOINTS. This is called whenever fuse filesystem is run so the contract - * always runs on a new checkpoint. - */ -void state_monitor::create_checkpoint() -{ - /** - * Checkpoints are numbered 0, -1, -2, ... - * Checkpoint 0 is the latest state containing "state", "data", "delta", "bhmap", "htree" directories. - * Checkpoints -1 and lower contains only the "delta" dirs containing older state changesets. - */ - - // Shift "-1" and older checkpoints by 1 more. And then copy checkpoint 0 delta dir to "-1". - // If MAX oldest checkpoint is there, remove it and work our way upwards. - int16_t oldest_chkpnt = MAX_CHECKPOINTS * -1; - for (int16_t chkpnt = oldest_chkpnt; chkpnt <= -1; chkpnt++) - { - std::string dir = get_state_dir_root(chkpnt); - - if (boost::filesystem::exists(dir)) - { - if (chkpnt == oldest_chkpnt) - { - boost::filesystem::remove_all(dir); - } - else - { - std::string dir_shift = get_state_dir_root(chkpnt - 1); - boost::filesystem::rename(dir, dir_shift); - } - } - - if (chkpnt == -1) - { - state_dir_context ctx = get_state_dir_context(0, true); - - // Shift 0-state delta dir to -1. - std::string delta_1 = dir + DELTA_DIR; - boost::filesystem::create_directories(delta_1); - - boost::filesystem::rename(ctx.delta_dir, delta_1); - boost::filesystem::create_directories(ctx.delta_dir); - } - } - - return; -} - -/** - * Called whenever a new file is created in the fuse fs. - * @param fd The fd of the created file. - */ -void state_monitor::oncreate(const int fd) -{ - std::lock_guard lock(monitor_mutex); - - std::string filepath; - if (extract_filepath(filepath, fd) == 0) - oncreate_filepath(filepath); -} - -/** - * Called whenever a file is going to be opened. - * @param inodefd inode fd given by fuse fs. This is used to find the physical path of the file. - * @param flags Open flags. - */ -void state_monitor::onopen(const int inodefd, const int flags) -{ - std::lock_guard lock(monitor_mutex); - - // Find the actual file path which is going to be opened and add that path to tracked file info list. - std::string filepath; - if (extract_filepath(filepath, inodefd) == 0) - { - state_file_info *fi; - if (get_tracked_fileinfo(&fi, filepath) == 0) - { - // Check whether the file is going to be opened in truncate mode. - // If so cache the entire file immediately because this is the last chance we get to backup the data. - if (flags & O_TRUNC) - cache_blocks(*fi, 0, fi->original_length); - } - } -} - -/** - * Called whenever a file is being written to. - * @param fd fd of the file being written to. - * @param offset Byte offset of the write. - * @param length Number of bytes being overwritten. - */ -void state_monitor::onwrite(const int fd, const off_t offset, const size_t length) -{ - // TODO: Known issue: onwrite can get called if the client program deletes a file before - // closing the currently open file. If there were some bytes on the write buffer, the flush happens - // when the client closes the fd. By that time the fd is invalid since the file is deleted. - // However nothing happens to us as our code simply returns on invalild fd error. - - std::lock_guard lock(monitor_mutex); - - // Find the actual filepath being written to and cache the blocks to server as backup. - std::string filepath; - if (get_fd_filepath(filepath, fd) == 0) - { - state_file_info *fi; - if (get_tracked_fileinfo(&fi, filepath) == 0) - cache_blocks(*fi, offset, length); - } -} - -/** - * Called when a file is being renamed. - * We simply treat this as delete-and-create operation. - */ -void state_monitor::onrename(const std::string &old_filepath, const std::string &new_filepath) -{ - std::lock_guard lock(monitor_mutex); - - ondelete_filepath(old_filepath); - oncreate_filepath(new_filepath); -} - -/** - * Called when a file is being deleted. - */ -void state_monitor::ondelete(const std::string &filepath) -{ - std::lock_guard lock(monitor_mutex); - ondelete_filepath(filepath); -} - -/** - * Called when a file is being truncated. - */ -void state_monitor::ontruncate(const int fd, const off_t newsize) -{ - std::lock_guard lock(monitor_mutex); - - std::string filepath; - if (get_fd_filepath(filepath, fd) == 0) - { - // If truncated size is less than the original, cache the entire file. - state_file_info *fi; - if (get_tracked_fileinfo(&fi, filepath) == 0 && newsize < fi->original_length) - cache_blocks(*fi, 0, fi->original_length); - } -} - -/** - * Called when an open file is being closed. Here, we clear any tracking information we kept for this file - * and close off any related fds associated with any backup operations for this file. - */ -void state_monitor::onclose(const int fd) -{ - std::lock_guard lock(monitor_mutex); - - // fd_path_map should contain this fd already if we were tracking it. - - auto pitr = fd_path_map.find(fd); - if (pitr != fd_path_map.end()) - { - // Close any block cache/index fds we have opened for this file. - auto fitr = file_info_map.find(pitr->second); // pitr->second is the filepath string. - if (fitr != file_info_map.end()) - close_caching_fds(fitr->second); // fitr->second is the fileinfo struct. - - fd_path_map.erase(pitr); - } -} - -/** - * Extracts the full physical file path for a given fd. - * @param filepath String to assign the extracted file path. - * @param fd The file descriptor to find the filepath. - * @return 0 on successful file path extraction. -1 on failure. - */ -int state_monitor::extract_filepath(std::string &filepath, const int fd) -{ - char proclnk[32]; - sprintf(proclnk, "/proc/self/fd/%d", fd); - - filepath.resize(PATH_MAX); - ssize_t len = readlink(proclnk, filepath.data(), PATH_MAX); - if (len > 0) - { - filepath.resize(len); - return 0; - } - return -1; -} - -/** - * Find the full physical file path for a given fd using the fd map. - * @param filepath String to assign the extracted file path. - * @param fd The file descriptor to find the filepath. - * @return 0 on successful file path extraction. -1 on failure. - */ -int state_monitor::get_fd_filepath(std::string &filepath, const int fd) -{ - // Return path from the map if found. - const auto itr = fd_path_map.find(fd); - if (itr != fd_path_map.end()) - { - filepath = itr->second; - return 0; - } - - // Extract the file path and populate the fd-->filepath map. - if (extract_filepath(filepath, fd) == 0) - { - fd_path_map[fd] = filepath; - return 0; - } - - return -1; -} - -/** - * Called when a new file is going to be created. fd is not yet open at this point. - * We need to catch this and start tracking this filepath. - */ -void state_monitor::oncreate_filepath(const std::string &filepath) -{ - // Check whether we are already tracking this file path. - // Only way we could be tracking this path already is deleting an existing file and creating - // a new file with the same name. - if (file_info_map.count(filepath) == 0) - { - // Add an entry for the new file in the file info map. This information will be used to ignore - // future operations (eg. write/delete) done to this file. - state_file_info fi; - fi.is_new = true; - fi.filepath = filepath; - file_info_map[filepath] = std::move(fi); - - // Add to the list of new files added during this session. - write_new_file_entry(filepath); - } -} - -/** - * Called when a file is going to be deleted. We use this to remove any tracking information - * regarding this file and to backup the file before deletion. - */ -void state_monitor::ondelete_filepath(const std::string &filepath) -{ - state_file_info *fi; - if (get_tracked_fileinfo(&fi, filepath) == 0) - { - if (fi->is_new) - { - // If this is a new file, just remove from existing index entries. - // No need to cache the file blocks. - remove_new_file_entry(fi->filepath); - file_info_map.erase(filepath); - } - else - { - // If not a new file, cache the entire file. - cache_blocks(*fi, 0, fi->original_length); - } - } -} - -/** - * Finds the tracked state file information for the given filepath. - * @param fi Reference pointer to assign the state file info struct. - * @param filepath Full physical path of the file. - * @return 0 on successful find. -1 on failure. - */ -int state_monitor::get_tracked_fileinfo(state_file_info **fi, const std::string &filepath) -{ - // Return from filepath-->fileinfo map if found. - const auto itr = file_info_map.find(filepath); - if (itr != file_info_map.end()) - { - *fi = &itr->second; - return 0; - } - - // Initialize a new state file info struct for the given filepath. - state_file_info &fileinfo = file_info_map[filepath]; - - // We use stat() to find out the length of the file. - struct stat stat_buf; - if (stat(filepath.c_str(), &stat_buf) != 0) - { - std::cerr << errno << ": Error occured in stat() of " << filepath << "\n"; - return -1; - } - - fileinfo.original_length = stat_buf.st_size; - fileinfo.filepath = filepath; - *fi = &fileinfo; - return 0; -} - -/** - * Backs up the specified bytes range of the given file. This is called whenever a file is being - * overwritten/deleted. - * @param fi The file info struct pointing to the file to be cached. - * @param offset The start byte position for caching. - * @param length How many bytes to cache. - * @return 0 on successful execution. -1 on failure. - */ -int state_monitor::cache_blocks(state_file_info &fi, const off_t offset, const size_t length) -{ - // No caching required if this is a new file created during this session. - if (fi.is_new) - return 0; - - uint32_t original_block_count = ceil((double)fi.original_length / (double)BLOCK_SIZE); - - // Check whether we have already cached the entire file. - if (original_block_count == fi.cached_blockids.size()) - return 0; - - // Initialize fds and indexes required for caching the file. - if (prepare_caching(fi) != 0) - return -1; - - // Return if incoming write is outside any of the original blocks. - if (offset > original_block_count * BLOCK_SIZE) - return 0; - - uint32_t startblock = offset / BLOCK_SIZE; - uint32_t endblock = (offset + length) / BLOCK_SIZE; - - // std::cout << "Cache blocks: '" << fi.filepath << "' [" << offset << "," << length << "] " << startblock << "," << endblock << "\n"; - - // If this is the first time we are caching this file, write an entry to the touched file index. - // Touched file index is used by rollback to server as a guide. - if (fi.cached_blockids.empty() && write_touched_file_entry(fi.filepath) != 0) - return -1; - - for (uint32_t i = startblock; i <= endblock; i++) - { - // Skip if we have already cached this block. - if (fi.cached_blockids.count(i) > 0) - continue; - - // Read the block being replaced and send to cache file. - // Allocating block buffer on the heap to avoid filling limited stack space. - std::unique_ptr block_buf = std::make_unique(BLOCK_SIZE); - off_t block_offset = BLOCK_SIZE * i; - size_t bytes_read = pread(fi.readfd, block_buf.get(), BLOCK_SIZE, BLOCK_SIZE * i); - if (bytes_read < 0) - { - std::cerr << errno << ": Read failed " << fi.filepath << "\n"; - return -1; - } - - // No more bytes to read in this file. - if (bytes_read == 0) - return 0; - - if (write(fi.cachefd, block_buf.get(), bytes_read) < 0) - { - std::cerr << errno << ": Write to block cache failed. " << fi.filepath << "\n"; - return -1; - } - - // Append an entry (44 bytes) into the block cache index. We maintain this index to - // help random block access for external use cases. We currently do not sort this index here. - // Whoever is using the index must sort it if required. - // Entry format: [blocknum(4 bytes) | cacheoffset(8 bytes) | blockhash(32 bytes)] - - // Calculate the block hash by combining block offset with block data. - char entrybuf[BLOCK_INDEX_ENTRY_SIZE]; - hasher::B2H hash = hasher::hash(&block_offset, 8, block_buf.get(), bytes_read); - - // Original file block id. - memcpy(entrybuf, &i, 4); - // Position of the block within the cache file. - off_t cacheoffset = fi.cached_blockids.size() * BLOCK_SIZE; - memcpy(entrybuf + 4, &cacheoffset, 8); - // The block hash. - memcpy(entrybuf + 12, hash.data, 32); - if (write(fi.indexfd, entrybuf, BLOCK_INDEX_ENTRY_SIZE) < 0) - { - std::cerr << errno << ": Write to block index failed. " << fi.filepath << "\n"; - return -1; - } - - // Mark the block as cached. - fi.cached_blockids.emplace(i); - } - - return 0; -} - -/** - * Initializes fds and indexes required for caching a particular file. - * @param fi The state file info struct pointing to the file being cached. - * @return 0 on succesful initialization. -1 on failure. - */ -int state_monitor::prepare_caching(state_file_info &fi) -{ - // If readfd is greater than 0 then we take it as caching being already initialized for this file. - if (fi.readfd > 0) - return 0; - - // Open up the file using a read-only fd. This fd will be used to fetch blocks to be cached. - fi.readfd = open(fi.filepath.c_str(), O_RDONLY); - if (fi.readfd < 0) - { - std::cerr << errno << ": Open failed " << fi.filepath << "\n"; - return -1; - } - - // Get the path of the file relative to the state dir. We maintain this same reative path for the - // corresponding cache and index files in the cache dir. - std::string relpath = get_relpath(fi.filepath, ctx.data_dir); - - std::string tmppath; - tmppath.reserve(ctx.delta_dir.length() + relpath.length() + BLOCK_CACHE_EXT_LEN); - tmppath.append(ctx.delta_dir).append(relpath).append(BLOCK_CACHE_EXT); - - // Create directory tree if not exist so we are able to create the cache and index files. - boost::filesystem::path cachesubdir = boost::filesystem::path(tmppath).parent_path(); - if (created_cache_subdirs.count(cachesubdir.string()) == 0) - { - boost::filesystem::create_directories(cachesubdir); - created_cache_subdirs.emplace(cachesubdir.string()); - } - - // Create and open the block cache file. - fi.cachefd = open(tmppath.c_str(), O_WRONLY | O_APPEND | O_CREAT, FILE_PERMS); - if (fi.cachefd <= 0) - { - std::cerr << errno << ": Open failed " << tmppath << "\n"; - return -1; - } - - // Create and open the block index file. - tmppath.replace(tmppath.length() - BLOCK_CACHE_EXT_LEN, BLOCK_INDEX_EXT_LEN, BLOCK_INDEX_EXT); - fi.indexfd = open(tmppath.c_str(), O_WRONLY | O_APPEND | O_CREAT, FILE_PERMS); - if (fi.indexfd <= 0) - { - std::cerr << errno << ": Open failed " << tmppath << "\n"; - return -1; - } - - // Write first entry (8 bytes) to the index file. First entry is the length of the original file. - // This will be helpful when restoring/rolling back a file. - if (write(fi.indexfd, &fi.original_length, 8) == -1) - { - std::cerr << errno << ": Error writing to index file " << tmppath << "\n"; - return -1; - } - - return 0; -} - -/** - * Closes any open caching fds for a given file. - */ -void state_monitor::close_caching_fds(state_file_info &fi) -{ - if (fi.readfd > 0) - close(fi.readfd); - - if (fi.cachefd > 0) - close(fi.cachefd); - - if (fi.indexfd > 0) - close(fi.indexfd); - - fi.readfd = 0; - fi.cachefd = 0; - fi.indexfd = 0; -} - -/** - * Inserts a file into the modified files list of this session. - * This index is used to restore modified files during rollback. - */ -int state_monitor::write_touched_file_entry(std::string_view filepath) -{ - if (touched_fileindex_fd <= 0) - { - std::string index_file = ctx.delta_dir + IDX_TOUCHED_FILES; - touched_fileindex_fd = open(index_file.c_str(), O_WRONLY | O_APPEND | O_CREAT, FILE_PERMS); - if (touched_fileindex_fd <= 0) - { - std::cerr << errno << ": Open failed " << index_file << "\n"; - return -1; - } - } - - // Write the relative file path line to the index. - filepath = filepath.substr(ctx.data_dir.length(), filepath.length() - ctx.data_dir.length()); - write(touched_fileindex_fd, filepath.data(), filepath.length()); - write(touched_fileindex_fd, "\n", 1); - return 0; -} - -/** - * Inserts a file into the list of new files created during this session. - * This index is used in deleting new files during restore. - */ -int state_monitor::write_new_file_entry(std::string_view filepath) -{ - std::string index_file = ctx.delta_dir + IDX_NEW_FILES; - int fd = open(index_file.c_str(), O_WRONLY | O_APPEND | O_CREAT, FILE_PERMS); - if (fd <= 0) - { - std::cerr << errno << ": Open failed " << index_file << "\n"; - return -1; - } - - // Write the relative file path line to the index. - filepath = filepath.substr(ctx.data_dir.length(), filepath.length() - ctx.data_dir.length()); - write(fd, filepath.data(), filepath.length()); - write(fd, "\n", 1); - close(fd); - return 0; -} - -/** - * Scans and removes the given filepath from the new files index. - * This is called when a file added during this session gets deleted in the same session. - */ -void state_monitor::remove_new_file_entry(std::string_view filepath) -{ - filepath = filepath.substr(ctx.data_dir.length(), filepath.length() - ctx.data_dir.length()); - - // We create a copy of the new file index and transfer lines from first file - // to the second file except the line matching the given filepath. - - std::string index_file = ctx.delta_dir + IDX_NEW_FILES; - std::string index_file_tmp = ctx.delta_dir + IDX_NEW_FILES + ".tmp"; - - std::ifstream in_file(index_file); - std::ofstream outfile(index_file_tmp); - - bool lines_transferred = false; - for (std::string line; std::getline(in_file, line);) - { - if (line != filepath) // Skip the file being removed. - { - outfile << line << "\n"; - lines_transferred = true; - } - } - - in_file.close(); - outfile.close(); - - // Remove the old index. - std::remove(index_file.c_str()); - - // If no lines transferred, delete the temp file as well. - if (lines_transferred) - std::rename(index_file_tmp.c_str(), index_file.c_str()); - else - std::remove(index_file_tmp.c_str()); -} - -} // namespace statefs \ No newline at end of file diff --git a/src/statefs/state_monitor/state_monitor.hpp b/src/statefs/state_monitor/state_monitor.hpp deleted file mode 100644 index 43ed6d8b..00000000 --- a/src/statefs/state_monitor/state_monitor.hpp +++ /dev/null @@ -1,78 +0,0 @@ -#ifndef _HP_STATEFS_STATE_MONITOR_ -#define _HP_STATEFS_STATE_MONITOR_ - -#include -#include -#include -#include -#include -#include -#include "../state_common.hpp" - -namespace statefs -{ - -/** - * Holds information about an original file in state that we are tracking. - */ -struct state_file_info -{ - bool is_new; // Whether this is a new file created during this session. - off_t original_length; // Original file length. - std::unordered_set cached_blockids; // Set of block ids cached during this session. - std::string filepath; // Actual real path of the file. (not fuse path) - int readfd; // fd used for reading the original file for caching. - int cachefd; // fd for writing into the block cache file. - int indexfd; // fd for writing into the block index file. -}; - -/** - * Invoked by fuse file system for relevent file system calls. - */ -class state_monitor -{ -private: - // Map of fd-->filepath - std::unordered_map fd_path_map; - - // Map of filepath-->fileinfo - std::unordered_map file_info_map; - - // List of new cache sub directories created during the session. - std::unordered_set created_cache_subdirs; - - // Mutex to synchronize parallel file system calls into our custom state tracking logic. - std::mutex monitor_mutex; - - // Holds the fd used to write into modified files index. This will be kept open for the entire - // life of the state monitor. - int touched_fileindex_fd = 0; - - int extract_filepath(std::string &filepath, const int fd); - int get_fd_filepath(std::string &filepath, const int fd); - void oncreate_filepath(const std::string &filepath); - void ondelete_filepath(const std::string &filepath); - int get_tracked_fileinfo(state_file_info **fileinfo, const std::string &filepath); - - int cache_blocks(state_file_info &fi, const off_t offset, const size_t length); - int prepare_caching(state_file_info &fi); - void close_caching_fds(state_file_info &fi); - int write_touched_file_entry(std::string_view filepath); - int write_new_file_entry(std::string_view filepath); - void remove_new_file_entry(std::string_view filepath); - -public: - state_dir_context ctx; - void create_checkpoint(); - void oncreate(const int fd); - void onopen(const int inodefd, const int flags); - void onwrite(const int fd, const off_t offset, const size_t length); - void onrename(const std::string &old_filepath, const std::string &new_filepath); - void ondelete(const std::string &filepath); - void ontruncate(const int fd, const off_t newsize); - void onclose(const int fd); -}; - -} // namespace statefs - -#endif \ No newline at end of file diff --git a/src/statefs/state_restore.cpp b/src/statefs/state_restore.cpp deleted file mode 100644 index 4b4ff7a5..00000000 --- a/src/statefs/state_restore.cpp +++ /dev/null @@ -1,200 +0,0 @@ -#include "../pchheader.hpp" -#include "../hplog.hpp" -#include "hasher.hpp" -#include "state_restore.hpp" -#include "hashtree_builder.hpp" -#include "state_common.hpp" - -namespace statefs -{ - -// Look at new files added and delete them if still exist. -void state_restore::delete_new_files() -{ - std::string index_file(ctx.delta_dir); - index_file.append(IDX_NEW_FILES); - - std::ifstream in_file(index_file); - for (std::string file; std::getline(in_file, file);) - { - std::string filepath(ctx.data_dir); - filepath.append(file); - - remove(filepath.c_str()); - } - - in_file.close(); -} - -// Look at touched files and restore them. -int state_restore::restore_touched_files() -{ - std::unordered_set processed; - - std::string index_file(ctx.delta_dir); - index_file.append(IDX_TOUCHED_FILES); - - std::ifstream in_file(index_file); - for (std::string file; std::getline(in_file, file);) - { - // Skip if already processed. - if (processed.count(file) > 0) - continue; - - std::vector bindex; - if (read_block_index(bindex, file) != 0) - return -1; - - if (restore_blocks(file, bindex) != 0) - return -1; - - // Add to processed file list. - processed.emplace(file); - } - - in_file.close(); - return 0; -} - -// Read the delta block index. -int state_restore::read_block_index(std::vector &buffer, std::string_view file) -{ - std::string bindex_file(ctx.delta_dir); - bindex_file.append(file).append(BLOCK_INDEX_EXT); - std::ifstream in_file(bindex_file, std::ios::binary | std::ios::ate); - std::streamsize idx_size = in_file.tellg(); - in_file.seekg(0, std::ios::beg); - - buffer.resize(idx_size); - if (!in_file.read(buffer.data(), idx_size)) - { - LOG_ERR << errno << ": Read failed " << bindex_file; - return -1; - } - - return 0; -} - -// Restore blocks mentioned in the delta block index. -int state_restore::restore_blocks(std::string_view file, const std::vector &bindex) -{ - int bcache_fd = 0, ori_file_fd = 0; - const char *idx_ptr = bindex.data(); - - // First 8 bytes of the index contains the supposed length of the original file. - off_t original_len = 0; - memcpy(&original_len, idx_ptr, 8); - - // Open block cache file. - { - std::string bcache_file(ctx.delta_dir); - bcache_file.append(file).append(BLOCK_CACHE_EXT); - bcache_fd = open(bcache_file.c_str(), O_RDONLY); - if (bcache_fd <= 0) - { - LOG_ERR << errno << ": Open failed " << bcache_file; - return -1; - } - } - - // Create or Open original file. - { - std::string original_file(ctx.data_dir); - original_file.append(file); - - // Create directory tree if not exist so we are able to create the file. - boost::filesystem::path filedir = boost::filesystem::path(original_file).parent_path(); - if (created_dirs.count(filedir.string()) == 0) - { - boost::filesystem::create_directories(filedir); - created_dirs.emplace(filedir.string()); - } - - ori_file_fd = open(original_file.c_str(), O_WRONLY | O_CREAT, FILE_PERMS); - if (ori_file_fd <= 0) - { - LOG_ERR << errno << ": Open failed " << original_file; - return -1; - } - } - - // Restore the blocks as specified in block index. - for (uint32_t idx_offset = 8; idx_offset < bindex.size();) - { - // Find the block no. of where this block is from in the original file. - uint32_t block_no = 0; - memcpy(&block_no, idx_ptr + idx_offset, 4); - idx_offset += 4; - off_t ori_file_offset = block_no * BLOCK_SIZE; - - // Find the offset where the block is located in the block cache file. - off_t bcache_offset; - memcpy(&bcache_offset, idx_ptr + idx_offset, 8); - idx_offset += 40; // Skip the hash(32) - - // Transfer the cached block to the target file. - copy_file_range(bcache_fd, &bcache_offset, ori_file_fd, &ori_file_offset, BLOCK_SIZE, 0); - } - - // If the target file is bigger than the original size, truncate it to the original size. - off_t current_len = lseek(ori_file_fd, 0, SEEK_END); - if (current_len > original_len) - ftruncate(ori_file_fd, original_len); - - close(bcache_fd); - close(ori_file_fd); - - return 0; -} - -// This is called after a rollback so the all checkpoint dirs shift by 1. -void state_restore::rewind_checkpoints() -{ - // Assuming we have restored the current state with current delta, - // we need to shift each history delta by 1 place. - - // Delete the state 0 (current) delta. - boost::filesystem::remove_all(ctx.delta_dir); - - int16_t oldest_chkpnt = (MAX_CHECKPOINTS + 1) * -1; // +1 because we maintain one extra checkpoint in case of rollbacks. - for (int16_t chkpnt = -1; chkpnt >= oldest_chkpnt; chkpnt--) - { - std::string dir = get_state_dir_root(chkpnt); - - if (boost::filesystem::exists(dir)) - { - if (chkpnt == -1) - { - // Shift -1 state delta dir to 0-state and delete -1 dir. - std::string delta_1 = dir + DELTA_DIR; - boost::filesystem::rename(delta_1, ctx.delta_dir); - boost::filesystem::remove_all(dir); - } - else - { - std::string dirshift = get_state_dir_root(chkpnt + 1); - boost::filesystem::rename(dir, dirshift); - } - } - } -} - -// Rolls back current state to previous state. -int state_restore::rollback(hasher::B2H &root_hash) -{ - ctx = get_state_dir_context(); - - delete_new_files(); - if (restore_touched_files() == -1) - return -1; - - // Update hash tree. - hashtree_builder htree_builder(ctx); - htree_builder.generate(root_hash); - - rewind_checkpoints(); - - return 0; -} - -} // namespace statefs \ No newline at end of file diff --git a/src/statefs/state_restore.hpp b/src/statefs/state_restore.hpp deleted file mode 100644 index 01ab0ea5..00000000 --- a/src/statefs/state_restore.hpp +++ /dev/null @@ -1,28 +0,0 @@ -#ifndef _HP_STATEFS_STATE_RESTORE_ -#define _HP_STATEFS_STATE_RESTORE_ - -#include "../pchheader.hpp" -#include "hasher.hpp" -#include "state_common.hpp" - -namespace statefs -{ - -class state_restore -{ -private: - state_dir_context ctx; - std::unordered_set created_dirs; - void delete_new_files(); - int restore_touched_files(); - int read_block_index(std::vector &buffer, std::string_view file); - int restore_blocks(std::string_view file, const std::vector &bindex); - void rewind_checkpoints(); - -public: - int rollback(hasher::B2H &root_hash); -}; - -} // namespace statefs - -#endif diff --git a/src/statefs/state_store.cpp b/src/statefs/state_store.cpp deleted file mode 100644 index 51fef626..00000000 --- a/src/statefs/state_store.cpp +++ /dev/null @@ -1,357 +0,0 @@ -#include "../pchheader.hpp" -#include "hasher.hpp" -#include "state_common.hpp" -#include "hashtree_builder.hpp" -#include "state_store.hpp" -#include "../hplog.hpp" -#include "state_store.hpp" - -namespace statefs -{ - -// Map of modified/deleted files with updated blockids and hashes (if modified). -std::unordered_map> touched_files; - -/** - * Checks whether the given directory exists in the state data directory. - */ -bool is_dir_exists(const std::string &dir_relpath) -{ - const std::string full_path = current_ctx.data_dir + dir_relpath; - return boost::filesystem::exists(full_path); -} - -/** - * Retrieves the hash list of the file system entries at a given directory. - * @return 0 on success. -1 on failure. - */ -int get_fs_entry_hashes(std::unordered_map &fs_entries, const std::string &dir_relpath, const hasher::B2H expected_hash) -{ - // TODO: instead of iterating the data dir, we could simply query the hash tree directory - // listing and get the hashes using the hardlink names straight away. But then we don't have - // a way to get the file names. If we could implement a mechanism for that we could make this efficient. - - if (expected_hash != hasher::B2H_empty) - { - // Check whether the existing block hash matches expected hash. - const std::string dir_hash_path = current_ctx.hashtree_dir + dir_relpath + DIR_HASH_FNAME; - - hasher::B2H existsing_hash; - if (read_file_bytes(&existsing_hash, dir_hash_path.c_str(), 0, hasher::HASH_SIZE) == -1) - return -1; - - if (existsing_hash != expected_hash) - return -1; - } - - const std::string full_path = current_ctx.data_dir + dir_relpath; - for (const boost::filesystem::directory_entry &dentry : boost::filesystem::directory_iterator(full_path)) - { - const boost::filesystem::path p = dentry.path(); - - p2p::state_fs_hash_entry fs_entry; - fs_entry.is_file = !boost::filesystem::is_directory(p); - - std::string fsentry_relpath = dir_relpath + p.filename().string(); - - // Read the first 32 bytes of the .bhmap file or dir.hash file. - - std::string hash_path; - - if (fs_entry.is_file) - { - hash_path = current_ctx.block_hashmap_dir + fsentry_relpath + BLOCK_HASHMAP_EXT; - } - else - { - fsentry_relpath += "/"; - hash_path = current_ctx.hashtree_dir + fsentry_relpath + DIR_HASH_FNAME; - // Skip the directory if it doesn't contain the dir.hash file. - // By that we assume the directory is empty so we're not interested in it. - if (!boost::filesystem::exists(hash_path)) - continue; - } - - if (read_file_bytes(&fs_entry.hash, hash_path.c_str(), 0, hasher::HASH_SIZE) == -1) - return -1; - - fs_entries.emplace(fsentry_relpath, std::move(fs_entry)); - } - return 0; -} - -/** - * Retrieves the block hash map for a file. - * @return 0 on success. -1 on failure. - */ -int get_block_hash_map(std::vector &vec, const std::string &file_relpath, const hasher::B2H expected_hash) -{ - const std::string bhmap_path = current_ctx.block_hashmap_dir + file_relpath + BLOCK_HASHMAP_EXT; - - if (expected_hash != hasher::B2H_empty) - { - // Check whether the existing block hash matches expected hash. - - if (!boost::filesystem::exists(bhmap_path) || read_file_bytes_to_end(vec, bhmap_path.c_str(), 0) == -1) - return -1; - - // Existing hash is the first 32 bytes of bhmap contents. - hasher::B2H existing_hash = *reinterpret_cast(vec.data()); - if (existing_hash != expected_hash) - return -1; - - // Return the bhmap bytes without the first 32 bytes. - vec.erase(vec.begin(), vec.begin() + hasher::HASH_SIZE); - } - else - { - // Skip the file root hash and get the rest of the bytes. - if (boost::filesystem::exists(bhmap_path) && read_file_bytes_to_end(vec, bhmap_path.c_str(), hasher::HASH_SIZE) == -1) - return -1; - } - - return 0; -} - -/** - * Retrieves the byte length of a file. - * @return 0 on success. -1 on failure. - */ -int get_file_length(const std::string &file_relpath) -{ - std::string full_path = current_ctx.data_dir + file_relpath; - int fd = open(full_path.c_str(), O_RDONLY); - if (fd == -1) - { - LOG_ERR << errno << " Open failed " << full_path; - return -1; - } - - const off_t total_len = lseek(fd, 0, SEEK_END); - close(fd); - - return total_len; -} - -/** - * Retrieves the specified data block from a state file. - * @return Number of bytes read on success. -1 on failure. - */ -int get_block(std::vector &vec, const std::string &file_relpath, const uint32_t block_id, const hasher::B2H expected_hash) -{ - // Check whether the existing block hash matches expected hash. - if (expected_hash != hasher::B2H_empty) - { - std::string bhmap_path = current_ctx.block_hashmap_dir + file_relpath + BLOCK_HASHMAP_EXT; - hasher::B2H existing_hash = hasher::B2H_empty; - - if (read_file_bytes(&existing_hash, bhmap_path.c_str(), (block_id + 1) * hasher::HASH_SIZE, hasher::HASH_SIZE) == -1) - return -1; - - if (existing_hash != expected_hash) - return -1; - } - - std::string full_path = current_ctx.data_dir + file_relpath; - vec.resize(BLOCK_SIZE); - int read_bytes = read_file_bytes(vec.data(), full_path.c_str(), block_id * BLOCK_SIZE, BLOCK_SIZE); - - if (read_bytes == -1) - return -1; - - vec.resize(read_bytes); - return read_bytes; -} - -/** - * Creates the specified directory in the state data directory. - */ -void create_dir(const std::string &dir_relpath) -{ - const std::string full_path = current_ctx.data_dir + dir_relpath; - boost::filesystem::create_directories(full_path); -} - -/** - * Deletes all files within the specified state sub directory and marks the changes. - * @return 0 on success. -1 on failure. - */ -int delete_dir(const std::string &dir_relpath) -{ - std::string full_dir_path = current_ctx.data_dir + dir_relpath; - - const boost::filesystem::directory_iterator itr_end; - for (boost::filesystem::directory_iterator itr(full_dir_path); itr != itr_end; itr++) - { - boost::filesystem::path p = itr->path(); - - if (!boost::filesystem::is_directory(p)) - { - if (!boost::filesystem::remove(p)) - return -1; - - // Add the deleted file rel path to the touched files list. - touched_files.emplace( - get_relpath(p.string(), current_ctx.data_dir), - std::map()); - } - } - - // Finally, delete the directory itself. - boost::filesystem::remove_all(full_dir_path); - - return 0; -} - -/** - * Deletes the specified state file and marks the change. - * @return 0 on success. -1 on failure. - */ -int delete_file(const std::string &file_relpath) -{ - std::string full_path = current_ctx.data_dir + file_relpath; - if (!boost::filesystem::remove(full_path)) - return -1; - - touched_files.emplace(file_relpath, std::map()); - return 0; -} - -/** - * Truncates the specified state file to the specified length and marks the change. - * @return 0 on success. -1 on failure. - */ -int truncate_file(const std::string &file_relpath, const size_t newsize) -{ - std::string full_path = current_ctx.data_dir + file_relpath; - int fd = open(full_path.c_str(), O_WRONLY | O_CREAT, FILE_PERMS); - if (fd == -1) - { - LOG_ERR << errno << " Open failed " << full_path; - return -1; - } - - int ret = ftruncate(fd, newsize); - close(fd); - if (ret == -1) - { - LOG_ERR << errno << "Truncate failed " << full_path; - return -1; - } - - return 0; -} - -/** - * Writes the specified block to a file and marks the change. - * @param file_relpath State data relative path of the file. - * @param block_id Block id to replace/write. - * @param buf The buffer containing data to be written. - * @param len Length of the buffer. - * @return 0 on success. -1 on failure. - */ -int write_block(const std::string &file_relpath, const uint32_t block_id, const void *buf, const size_t len) -{ - std::string full_path = current_ctx.data_dir + file_relpath; - int fd = open(full_path.c_str(), O_WRONLY | O_CREAT, FILE_PERMS); - if (fd == -1) - { - LOG_ERR << errno << " Open failed " << full_path; - return -1; - } - - const off_t offset = block_id * BLOCK_SIZE; - int ret = pwrite(fd, buf, len, offset); - close(fd); - if (ret == -1) - { - LOG_ERR << errno << " Write failed " << full_path; - return -1; - } - - hasher::B2H hash = hasher::hash(&offset, 8, buf, len); - touched_files[file_relpath].emplace(block_id, hash); - - return 0; -} - -/** - * Computes the latest hash tree with any changes recorded in touched files index. - * @return 0 on success. -1 on failure. - */ -int compute_hash_tree(hasher::B2H &statehash, const bool force_all) -{ - hashtree_builder htree_builder(current_ctx); - - int ret = force_all ? htree_builder.generate(statehash, true) : htree_builder.generate(statehash, touched_files); - - touched_files.clear(); - return ret; -} - -//-----Private helper functions---------// - -/** - * Reads bytes from file into a buffer. - * @param buf Buffer to fill with the read bytes. - * @param filepath Full path to the file. - * @param start Starting offset to read. - * @param len Number of bytes to read. - * @return Number of bytes read on successful read. -1 on failure. - */ -int read_file_bytes(void *buf, const char *filepath, const off_t start, const size_t len) -{ - int fd = open(filepath, O_RDONLY); - if (fd == -1) - { - LOG_ERR << errno << " Open failed " << filepath; - return -1; - } - - int read_bytes = pread(fd, buf, len, start); - close(fd); - if (read_bytes <= 0) - { - LOG_ERR << errno << " Read failed " << filepath; - return -1; - } - - return read_bytes; -} - -/** - * Reads bytes from file into a vector. The vector size will be adjusted to the actual bytes read. - * @param vec Vector to fill with the read bytes. - * @param filepath Full path to the file. - * @param start Starting offset to read. - * @return Number of bytes read on successful read. -1 on failure. - */ -int read_file_bytes_to_end(std::vector &vec, const char *filepath, const off_t start) -{ - int fd = open(filepath, O_RDONLY); - if (fd == -1) - { - LOG_ERR << errno << " Open failed " << filepath; - return -1; - } - - const off_t total_len = lseek(fd, 0, SEEK_END); - if (total_len == -1) - return -1; - - const size_t len = total_len - start; - vec.resize(len); - - int read_bytes = pread(fd, vec.data(), len, start); - close(fd); - if (read_bytes <= 0) - { - LOG_ERR << errno << " Read failed " << filepath; - return -1; - } - vec.resize(read_bytes); - - return read_bytes; -} - -} // namespace statefs \ No newline at end of file diff --git a/src/statefs/state_store.hpp b/src/statefs/state_store.hpp deleted file mode 100644 index fd1c0edc..00000000 --- a/src/statefs/state_store.hpp +++ /dev/null @@ -1,35 +0,0 @@ -#ifndef _HP_STATEFS_STATE_STORE_ -#define _HP_STATEFS_STATE_STORE_ - -#include "../pchheader.hpp" -#include "../p2p/p2p.hpp" -#include "hasher.hpp" - -namespace statefs -{ - -// Map of modified/deleted files with updated blockids and hashes (if modified). -extern std::unordered_map> touched_files; - -bool is_dir_exists(const std::string &dir_relpath); -int get_fs_entry_hashes(std::unordered_map &fs_entries, const std::string &dir_relpath, const hasher::B2H expected_hash); -int get_block_hash_map(std::vector &vec, const std::string &file_relpath, const hasher::B2H expected_hash); -int get_file_length(const std::string &file_relpath); -int get_block(std::vector &vec, const std::string &file_relpath, const uint32_t block_id, const hasher::B2H expected_hash); -void create_dir(const std::string &dir_relpath); -int delete_dir(const std::string &dir_relpath); -int delete_file(const std::string &file_relpath); -int truncate_file(const std::string &file_relpath, const size_t newsize); -int write_block(const std::string &file_relpath, const uint32_t block_id, const void *buf, const size_t len); -int compute_hash_tree(hasher::B2H &statehash, const bool force_all = false); - -/** - * Private helper functions. - */ - -int read_file_bytes(void *buf, const char *filepath, const off_t start, const size_t len); -int read_file_bytes_to_end(std::vector &vec, const char *filepath, const off_t start); - -} // namespace statefs - -#endif \ No newline at end of file diff --git a/src/usr/usr.cpp b/src/usr/usr.cpp index 7d5bdf47..5ee3dec8 100644 --- a/src/usr/usr.cpp +++ b/src/usr/usr.cpp @@ -15,167 +15,174 @@ namespace jusrmsg = jsonschema::usrmsg; namespace usr { -// Holds global connected-users and related objects. -connected_context ctx; + // Holds global connected-users and related objects. + connected_context ctx; -/** + bool init_success = false; + + /** * Initializes the usr subsystem. Must be called once during application startup. * @return 0 for successful initialization. -1 for failure. */ -int init() -{ - // Start listening for incoming user connections. - return start_listening(); -} + int init() + { + // Start listening for incoming user connections. + if (start_listening() == -1) + return -1; -/** + init_success = true; + return 0; + } + + /** * Cleanup any running processes. */ -void deinit() -{ - ctx.listener.stop(); -} + void deinit() + { + if (init_success) + ctx.listener.stop(); + } -/** + /** * Starts listening for incoming user websocket connections. */ -int start_listening() -{ - const uint64_t metric_thresholds[] = {conf::cfg.pubmaxcpm, 0, 0, conf::cfg.pubmaxbadmpm}; - if (ctx.listener.start( - conf::cfg.pubport, ".sock-user", comm::SESSION_TYPE::USER, true, true, metric_thresholds, std::set(), conf::cfg.pubmaxsize) == -1) - return -1; + int start_listening() + { + const uint64_t metric_thresholds[] = {conf::cfg.pubmaxcpm, 0, 0, conf::cfg.pubmaxbadmpm}; + if (ctx.listener.start( + conf::cfg.pubport, ".sock-user", comm::SESSION_TYPE::USER, true, true, metric_thresholds, std::set(), conf::cfg.pubmaxsize) == -1) + return -1; - LOG_INFO << "Started listening for user connections on " << std::to_string(conf::cfg.pubport); - return 0; -} + LOG_INFO << "Started listening for user connections on " << std::to_string(conf::cfg.pubport); + return 0; + } -/** + /** * Verifies the given message for a previously issued user challenge. * @param message Challenge response. * @param session The socket session that received the response. * @return 0 for successful verification. -1 for failure. */ -int verify_challenge(std::string_view message, comm::comm_session &session) -{ - // The received message must be the challenge response. We need to verify it. - if (session.issued_challenge.empty()) + int verify_challenge(std::string_view message, comm::comm_session &session) { - LOG_DBG << "No challenge found for the session " << session.uniqueid; - return -1; - } - - std::string userpubkeyhex; - std::string_view original_challenge = session.issued_challenge; - if (jusrmsg::verify_user_challenge_response(userpubkeyhex, message, original_challenge) == 0) - { - // Challenge signature verification successful. - - // Decode hex pubkey and get binary pubkey. We are only going to keep - // the binary pubkey due to reduced memory footprint. - std::string userpubkey; - userpubkey.resize(userpubkeyhex.length() / 2); - util::hex2bin( - reinterpret_cast(userpubkey.data()), - userpubkey.length(), - userpubkeyhex); - - // Now check whether this user public key is a duplicate. - if (ctx.sessionids.count(userpubkey) == 0) + // The received message must be the challenge response. We need to verify it. + if (session.issued_challenge.empty()) { - // All good. Unique public key. - // Promote the connection from pending-challenges to authenticated users. + LOG_DBG << "No challenge found for the session " << session.uniqueid; + return -1; + } - session.challenge_status = comm::CHALLENGE_VERIFIED; // Set as challenge verified - add_user(session, userpubkey); // Add the user to the global authed user list - session.issued_challenge.clear(); // Remove the stored challenge + std::string userpubkeyhex; + std::string_view original_challenge = session.issued_challenge; + if (jusrmsg::verify_user_challenge_response(userpubkeyhex, message, original_challenge) == 0) + { + // Challenge signature verification successful. - LOG_DBG << "User connection " << session.uniqueid << " authenticated. Public key " - << userpubkeyhex; - return 0; + // Decode hex pubkey and get binary pubkey. We are only going to keep + // the binary pubkey due to reduced memory footprint. + std::string userpubkey; + userpubkey.resize(userpubkeyhex.length() / 2); + util::hex2bin( + reinterpret_cast(userpubkey.data()), + userpubkey.length(), + userpubkeyhex); + + // Now check whether this user public key is a duplicate. + if (ctx.sessionids.count(userpubkey) == 0) + { + // All good. Unique public key. + // Promote the connection from pending-challenges to authenticated users. + + session.challenge_status = comm::CHALLENGE_VERIFIED; // Set as challenge verified + add_user(session, userpubkey); // Add the user to the global authed user list + session.issued_challenge.clear(); // Remove the stored challenge + + LOG_DBG << "User connection " << session.uniqueid << " authenticated. Public key " + << userpubkeyhex; + return 0; + } + else + { + LOG_DBG << "Duplicate user public key " << session.uniqueid; + } } else { - LOG_DBG << "Duplicate user public key " << session.uniqueid; + LOG_DBG << "Challenge verification failed " << session.uniqueid; } - } - else - { - LOG_DBG << "Challenge verification failed " << session.uniqueid; + + return -1; } - return -1; -} - -/** + /** * Processes a message sent by a connected user. This will be invoked by web socket on_message handler. * @param user The authenticated user who sent the message. * @param message The message sent by user. * @return 0 on successful processing. -1 for failure. */ -int handle_user_message(connected_user &user, std::string_view message) -{ - rapidjson::Document d; - const char *msg_type = jusrmsg::MSGTYPE_UNKNOWN; - - if (jusrmsg::parse_user_message(d, message) == 0) + int handle_user_message(connected_user &user, std::string_view message) { - const char *msg_type = d[jusrmsg::FLD_TYPE].GetString(); + rapidjson::Document d; + const char *msg_type = jusrmsg::MSGTYPE_UNKNOWN; - // Message is a contract input message. - if (d[jusrmsg::FLD_TYPE] == jusrmsg::MSGTYPE_CONTRACT_INPUT) + if (jusrmsg::parse_user_message(d, message) == 0) { - std::string contentjson; - std::string sig; - if (jusrmsg::extract_signed_input_container(contentjson, sig, d) == 0) - { - std::lock_guard lock(ctx.users_mutex); + const char *msg_type = d[jusrmsg::FLD_TYPE].GetString(); - //Add to the submitted input list. - user.submitted_inputs.push_back(user_submitted_message( - std::move(contentjson), - std::move(sig))); + // Message is a contract input message. + if (d[jusrmsg::FLD_TYPE] == jusrmsg::MSGTYPE_CONTRACT_INPUT) + { + std::string contentjson; + std::string sig; + if (jusrmsg::extract_signed_input_container(contentjson, sig, d) == 0) + { + std::lock_guard lock(ctx.users_mutex); + + //Add to the submitted input list. + user.submitted_inputs.push_back(user_submitted_message( + std::move(contentjson), + std::move(sig))); + return 0; + } + else + { + send_request_status_result(user.session, jusrmsg::STATUS_REJECTED, jusrmsg::REASON_BAD_SIG, msg_type, jusrmsg::origin_data_for_contract_input(sig)); + return -1; + } + } + else if (d[jusrmsg::FLD_TYPE] == jusrmsg::MSGTYPE_STAT) + { + std::string msg; + jusrmsg::create_status_response(msg); + user.session.send(msg); return 0; } else { - send_request_status_result(user.session, jusrmsg::STATUS_REJECTED, jusrmsg::REASON_BAD_SIG, msg_type, jusrmsg::origin_data_for_contract_input(sig)); + LOG_DBG << "Invalid user message type: " << msg_type; + send_request_status_result(user.session, jusrmsg::STATUS_REJECTED, jusrmsg::REASON_INVALID_MSG_TYPE, msg_type, ""); return -1; } } - else if (d[jusrmsg::FLD_TYPE] == jusrmsg::MSGTYPE_STAT) - { - std::string msg; - jusrmsg::create_status_response(msg); - user.session.send(msg); - return 0; - } else { - LOG_DBG << "Invalid user message type: " << msg_type; - send_request_status_result(user.session, jusrmsg::STATUS_REJECTED, jusrmsg::REASON_INVALID_MSG_TYPE, msg_type, ""); + // Bad message. + send_request_status_result(user.session, jusrmsg::STATUS_REJECTED, jusrmsg::REASON_BAD_MSG_FORMAT, msg_type, ""); return -1; } } - else - { - // Bad message. - send_request_status_result(user.session, jusrmsg::STATUS_REJECTED, jusrmsg::REASON_BAD_MSG_FORMAT, msg_type, ""); - return -1; - } -} -/** + /** * Send the specified status result via the provided session. */ -void send_request_status_result(const comm::comm_session &session, std::string_view status, std::string_view reason, std::string_view origin_type, std::string_view origin_extra_data) -{ - std::string msg; - jusrmsg::create_request_status_result(msg, status, reason, origin_type, origin_extra_data); - session.send(msg); -} + void send_request_status_result(const comm::comm_session &session, std::string_view status, std::string_view reason, std::string_view origin_type, std::string_view origin_extra_data) + { + std::string msg; + jusrmsg::create_request_status_result(msg, status, reason, origin_type, origin_extra_data); + session.send(msg); + } -/** + /** * Adds the user denoted by specified session id and public key to the global authed user list. * This should get called after the challenge handshake is verified. * @@ -183,70 +190,70 @@ void send_request_status_result(const comm::comm_session &session, std::string_v * @param pubkey User's binary public key. * @return 0 on successful additions. -1 on failure. */ -int add_user(const comm::comm_session &session, const std::string &pubkey) -{ - const std::string &sessionid = session.uniqueid; - if (ctx.users.count(sessionid) == 1) + int add_user(const comm::comm_session &session, const std::string &pubkey) { - LOG_INFO << sessionid << " already exist. Cannot add user."; - return -1; + const std::string &sessionid = session.uniqueid; + if (ctx.users.count(sessionid) == 1) + { + LOG_INFO << sessionid << " already exist. Cannot add user."; + return -1; + } + + { + std::lock_guard lock(ctx.users_mutex); + ctx.users.emplace(sessionid, usr::connected_user(session, pubkey)); + } + + // Populate sessionid map so we can lookup by user pubkey. + ctx.sessionids.try_emplace(pubkey, sessionid); + + return 0; } - { - std::lock_guard lock(ctx.users_mutex); - ctx.users.emplace(sessionid, usr::connected_user(session, pubkey)); - } - - // Populate sessionid map so we can lookup by user pubkey. - ctx.sessionids.try_emplace(pubkey, sessionid); - - return 0; -} - -/** + /** * Removes the specified public key from the global user list. * This must get called when a user disconnects from HP. * * @param sessionid User socket session id. * @return 0 on successful removals. -1 on failure. */ -int remove_user(const std::string &sessionid) -{ - const auto itr = ctx.users.find(sessionid); - - if (itr == ctx.users.end()) + int remove_user(const std::string &sessionid) { - LOG_INFO << sessionid << " does not exist. Cannot remove user."; - return -1; + const auto itr = ctx.users.find(sessionid); + + if (itr == ctx.users.end()) + { + LOG_INFO << sessionid << " does not exist. Cannot remove user."; + return -1; + } + + usr::connected_user &user = itr->second; + + { + std::lock_guard lock(ctx.users_mutex); + ctx.sessionids.erase(user.pubkey); + } + + ctx.users.erase(itr); + return 0; } - usr::connected_user &user = itr->second; - - { - std::lock_guard lock(ctx.users_mutex); - ctx.sessionids.erase(user.pubkey); - } - - ctx.users.erase(itr); - return 0; -} - -/** + /** * Finds and returns the socket session for the proided user pubkey. * @param pubkey User binary pubkey. * @return Pointer to the socket session. NULL of not found. */ -const comm::comm_session *get_session_by_pubkey(const std::string &pubkey) -{ - const auto sessionid_itr = ctx.sessionids.find(pubkey); - if (sessionid_itr != ctx.sessionids.end()) + const comm::comm_session *get_session_by_pubkey(const std::string &pubkey) { - const auto user_itr = ctx.users.find(sessionid_itr->second); - if (user_itr != ctx.users.end()) - return &user_itr->second.session; + const auto sessionid_itr = ctx.sessionids.find(pubkey); + if (sessionid_itr != ctx.sessionids.end()) + { + const auto user_itr = ctx.users.find(sessionid_itr->second); + if (user_itr != ctx.users.end()) + return &user_itr->second.session; + } + + return NULL; } - return NULL; -} - } // namespace usr \ No newline at end of file diff --git a/src/util.cpp b/src/util.cpp index ed442d9f..2b5e020c 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -1,84 +1,85 @@ #include "pchheader.hpp" +#include "hplog.hpp" #include "util.hpp" namespace util { -// rollover_hashset class methods + // rollover_hashset class methods -rollover_hashset::rollover_hashset(const uint32_t maxsize) -{ - this->maxsize = maxsize == 0 ? 1 : maxsize; -} + rollover_hashset::rollover_hashset(const uint32_t maxsize) + { + this->maxsize = maxsize == 0 ? 1 : maxsize; + } -/** + /** * Inserts the given hash to the list. * @return True on succesful insertion. False if hash already exists. */ -bool rollover_hashset::try_emplace(const std::string hash) -{ - const auto itr = recent_hashes.find(hash); - if (itr == recent_hashes.end()) // Not found + bool rollover_hashset::try_emplace(const std::string hash) { - // Add the new message hash to the set. - const auto [newitr, success] = recent_hashes.emplace(std::move(hash)); - - // Insert a pointer to the stored hash value to the back of the ordered list of hashes. - recent_hashes_list.push_back(&(*newitr)); - - // Remove oldest hash if exceeding max size. - if (recent_hashes_list.size() > maxsize) + const auto itr = recent_hashes.find(hash); + if (itr == recent_hashes.end()) // Not found { - const std::string &oldest_hash = *recent_hashes_list.front(); - recent_hashes.erase(oldest_hash); - recent_hashes_list.pop_front(); + // Add the new message hash to the set. + const auto [newitr, success] = recent_hashes.emplace(std::move(hash)); + + // Insert a pointer to the stored hash value to the back of the ordered list of hashes. + recent_hashes_list.push_back(&(*newitr)); + + // Remove oldest hash if exceeding max size. + if (recent_hashes_list.size() > maxsize) + { + const std::string &oldest_hash = *recent_hashes_list.front(); + recent_hashes.erase(oldest_hash); + recent_hashes_list.pop_front(); + } + + return true; // Hash was inserted successfuly. } - return true; // Hash was inserted successfuly. + return false; // Hash already exists. } - return false; // Hash already exists. -} + // ttl_set class methods. -// ttl_set class methods. - -/** + /** * If key does not exist, inserts it with the specified ttl. If key exists, * renews the expiration time to match the time-to-live from now onwards. * @param key Object to insert. * @param ttl Time to live in milliseonds. */ -void ttl_set::emplace(const std::string key, uint64_t ttl_milli) -{ - ttlmap[key] = util::get_epoch_milliseconds() + ttl_milli; -} + void ttl_set::emplace(const std::string key, uint64_t ttl_milli) + { + ttlmap[key] = util::get_epoch_milliseconds() + ttl_milli; + } -void ttl_set::erase(const std::string &key) -{ - const auto itr = ttlmap.find(key); - if (itr != ttlmap.end()) - ttlmap.erase(itr); -} + void ttl_set::erase(const std::string &key) + { + const auto itr = ttlmap.find(key); + if (itr != ttlmap.end()) + ttlmap.erase(itr); + } -/** + /** * Returns true of the key exists and not expired. Returns false if key does not exist * or has expired. */ -bool ttl_set::exists(const std::string &key) -{ - const auto itr = ttlmap.find(key); - if (itr == ttlmap.end()) // Not found - return false; + bool ttl_set::exists(const std::string &key) + { + const auto itr = ttlmap.find(key); + if (itr == ttlmap.end()) // Not found + return false; - // Check whether we are passed the expiration time (itr->second is the expiration time) - const bool expired = util::get_epoch_milliseconds() > itr->second; - if (expired) - ttlmap.erase(itr); + // Check whether we are passed the expiration time (itr->second is the expiration time) + const bool expired = util::get_epoch_milliseconds() > itr->second; + if (expired) + ttlmap.erase(itr); - return !expired; -} + return !expired; + } -/** + /** * Encodes provided bytes to hex string. * * @param encoded_string String reference to assign the hex encoded output. @@ -86,63 +87,63 @@ bool ttl_set::exists(const std::string &key) * @param bin_len Bytes length. * @return Always returns 0. */ -int bin2hex(std::string &encoded_string, const unsigned char *bin, const size_t bin_len) -{ - // Allocate the target string. - encoded_string.resize(bin_len * 2); + int bin2hex(std::string &encoded_string, const unsigned char *bin, const size_t bin_len) + { + // Allocate the target string. + encoded_string.resize(bin_len * 2); - // Get encoded string. - sodium_bin2hex( - encoded_string.data(), - encoded_string.length() + 1, // + 1 because sodium writes ending '\0' character as well. - bin, - bin_len); + // Get encoded string. + sodium_bin2hex( + encoded_string.data(), + encoded_string.length() + 1, // + 1 because sodium writes ending '\0' character as well. + bin, + bin_len); - return 0; -} + return 0; + } -/** + /** * Decodes provided hex string into bytes. * * @param decodedbuf Buffer to assign decoded bytes. * @param decodedbuf_len Decoded buffer size. * @param hex_str hex string to decode. */ -int hex2bin(unsigned char *decodedbuf, const size_t decodedbuf_len, std::string_view hex_str) -{ - const char *hex_end; - size_t bin_len; - if (sodium_hex2bin( - decodedbuf, decodedbuf_len, - hex_str.data(), - hex_str.length(), - "", &bin_len, &hex_end)) + int hex2bin(unsigned char *decodedbuf, const size_t decodedbuf_len, std::string_view hex_str) { - return -1; + const char *hex_end; + size_t bin_len; + if (sodium_hex2bin( + decodedbuf, decodedbuf_len, + hex_str.data(), + hex_str.length(), + "", &bin_len, &hex_end)) + { + return -1; + } + + return 0; } - return 0; -} - -/** + /** * Returns current time in UNIX epoch milliseconds. */ -int64_t get_epoch_milliseconds() -{ - return std::chrono::duration_cast( - std::chrono::system_clock::now().time_since_epoch()) - .count(); -} + int64_t get_epoch_milliseconds() + { + return std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count(); + } -/** + /** * Sleeps the current thread for specified no. of milliseconds. */ -void sleep(const uint64_t milliseconds) -{ - std::this_thread::sleep_for(std::chrono::milliseconds(milliseconds)); -} + void sleep(const uint64_t milliseconds) + { + std::this_thread::sleep_for(std::chrono::milliseconds(milliseconds)); + } -/** + /** * Compare two version strings in the format of "1.12.3". * v1 < v2 -> returns -1 * v1 == v2 -> returns 0 @@ -154,56 +155,75 @@ void sleep(const uint64_t milliseconds) * syntax because istringstream doesn't support string_view. It's not worth optmising * this code as it's not being used in high-scale processing. */ -int version_compare(const std::string &x, const std::string &y) -{ - std::istringstream ix(x), iy(y); - while (ix.good() || iy.good()) + int version_compare(const std::string &x, const std::string &y) { - int cx = 0, cy = 0; - ix >> cx; - iy >> cy; + std::istringstream ix(x), iy(y); + while (ix.good() || iy.good()) + { + int cx = 0, cy = 0; + ix >> cx; + iy >> cy; - if ((!ix.eof() && !ix.good()) || (!iy.eof() && !iy.good())) - return -2; + if ((!ix.eof() && !ix.good()) || (!iy.eof() && !iy.good())) + return -2; - if (cx > cy) - return 1; - if (cx < cy) - return -1; + if (cx > cy) + return 1; + if (cx < cy) + return -1; - ix.ignore(); - iy.ignore(); + ix.ignore(); + iy.ignore(); + } + + return 0; } - return 0; -} - -/** + /** * Returns a std::string_view pointing to the rapidjson Value which is assumed * to be a string. We use this function because rapidjson does not have built-in string_view * support. Passing a non-string 'v' is not supported. */ -std::string_view getsv(const rapidjson::Value &v) -{ - return std::string_view(v.GetString(), v.GetStringLength()); -} + std::string_view getsv(const rapidjson::Value &v) + { + return std::string_view(v.GetString(), v.GetStringLength()); + } -// Provide a safe std::string overload for realpath -std::string realpath(std::string path) -{ - std::array buffer; - ::realpath(path.c_str(), buffer.data()); - buffer[PATH_MAX] = '\0'; - return buffer.data(); -} + // Provide a safe std::string overload for realpath + std::string realpath(std::string path) + { + std::array buffer; + ::realpath(path.c_str(), buffer.data()); + buffer[PATH_MAX] = '\0'; + return buffer.data(); + } -// Applies signal mask to the calling thread. -void mask_signal() -{ - sigset_t mask; - sigemptyset(&mask); - sigaddset(&mask, SIGINT); - pthread_sigmask(SIG_BLOCK, &mask, NULL); -} + // Applies signal mask to the calling thread. + void mask_signal() + { + sigset_t mask; + sigemptyset(&mask); + sigaddset(&mask, SIGINT); + pthread_sigmask(SIG_BLOCK, &mask, NULL); + } + + // Kill a process with SIGINT and wait until it stops running. + int kill_process(const pid_t pid, const bool wait) + { + if (kill(pid, SIGINT) == -1) + { + LOG_ERR << errno << ": Error issuing SIGINT to pid " << pid; + return -1; + } + + int pid_status; + if (wait && waitpid(pid, &pid_status, 0) == -1) + { + LOG_ERR << errno << ": waitpid failed."; + return -1; + } + + return 0; + } } // namespace util diff --git a/src/util.hpp b/src/util.hpp index 5728c7e9..a0162f11 100644 --- a/src/util.hpp +++ b/src/util.hpp @@ -80,6 +80,8 @@ std::string realpath(std::string path); void mask_signal(); +int kill_process(const pid_t pid, const bool wait); + } // namespace util #endif diff --git a/test/bin/hpfs b/test/bin/hpfs new file mode 100755 index 00000000..0c51e627 Binary files /dev/null and b/test/bin/hpfs differ diff --git a/test/bin/libb2.so.1 b/test/bin/libb2.so.1 new file mode 100755 index 00000000..29a4d1f6 Binary files /dev/null and b/test/bin/libb2.so.1 differ diff --git a/test/local-cluster/Dockerfile b/test/local-cluster/Dockerfile index 87b56dc0..8954f865 100644 --- a/test/local-cluster/Dockerfile +++ b/test/local-cluster/Dockerfile @@ -4,18 +4,19 @@ FROM node:10.17.0-buster-slim RUN apt-get update # Install netcat for websocketd <--> domain socket redirection. -RUN apt-get install -y netcat-openbsd +RUN apt-get install -y netcat-openbsd libgomp1 -# Install fuse. -# Copy fuse shared library and register it. +# Install shared libraries. +# Copy shared libraries and register it. COPY ./bin/libfuse3.so.3 /usr/local/lib/ +COPY ./bin/libb2.so.1 /usr/local/lib/ RUN ldconfig COPY ./bin/fusermount3 /usr/local/bin/ # hpcore binary is copied to /hp directory withtin the docker image. WORKDIR /hp COPY ./bin/hpcore . -COPY ./bin/hpstatemon . +COPY ./bin/hpfs . COPY ./bin/websocketd . COPY ./bin/websocat . diff --git a/test/local-cluster/cluster-create.sh b/test/local-cluster/cluster-create.sh index 4550a841..706ab5b3 100755 --- a/test/local-cluster/cluster-create.sh +++ b/test/local-cluster/cluster-create.sh @@ -47,7 +47,7 @@ do # Update contract config. node -p "JSON.stringify({...require('./tmp.json'), \ binary: '/usr/local/bin/node', \ - binargs: './bin/contract.js', \ + binargs: '/contract/bin/contract.js', \ appbill: '', \ appbillargs: '', \ peerport: ${peerport}, \ diff --git a/test/vm-cluster/cluster.sh b/test/vm-cluster/cluster.sh index b4d9131b..eddb129d 100755 --- a/test/vm-cluster/cluster.sh +++ b/test/vm-cluster/cluster.sh @@ -42,7 +42,7 @@ fi if [ $mode = "check" ]; then let nodeid=$2-1 vmip=${vmips[$nodeid]} - sshpass -f vmpass.txt ssh geveo@$vmip 'echo hpcore pid:$(pidof hpcore) hpstatemon pid:$(pidof hpstatemon) websocketd pid:$(pidof websocketd)' + sshpass -f vmpass.txt ssh geveo@$vmip 'echo hpcore pid:$(pidof hpcore) hpfs pid:$(pidof hpfs) websocketd pid:$(pidof websocketd)' exit 0 fi @@ -57,7 +57,7 @@ if [ $mode = "kill" ]; then let nodeid=$2-1 vmip=${vmips[$nodeid]} sshpass -f vmpass.txt ssh geveo@$vmip 'sudo kill $(pidof hpcore) > /dev/null 2>&1' - sshpass -f vmpass.txt ssh geveo@$vmip 'sudo kill $(pidof hpstatemon) > /dev/null 2>&1' + sshpass -f vmpass.txt ssh geveo@$vmip 'sudo kill $(pidof hpfs) > /dev/null 2>&1' sshpass -f vmpass.txt ssh geveo@$vmip 'sudo kill $(pidof websocketd) > /dev/null 2>&1' exit 0 fi diff --git a/test/vm-cluster/setup-hp.sh b/test/vm-cluster/setup-hp.sh index 3bdd7180..c0f75e66 100755 --- a/test/vm-cluster/setup-hp.sh +++ b/test/vm-cluster/setup-hp.sh @@ -13,8 +13,9 @@ fi if [ -x "$(command -v fusermount3)" ]; then echo "FUSE already installed." else - echo "Installing FUSE..." - sudo cp ./libfuse3.so.3 /usr/local/lib/ + echo "Installing FUSE and other shared libraries..." + sudo apt-get -y install libgomp1 + sudo cp ./libfuse3.so.3 ./libb2.so.1 /usr/local/lib/ sudo ldconfig sudo cp ./fusermount3 /usr/local/bin/ fi diff --git a/test/vm-cluster/setup-vm.sh b/test/vm-cluster/setup-vm.sh index d2e91d3e..bb3bb1d2 100755 --- a/test/vm-cluster/setup-vm.sh +++ b/test/vm-cluster/setup-vm.sh @@ -11,12 +11,13 @@ echo $nodeid. $vmip if [ $mode = "new" ]; then sshpass -f vmpass.txt scp $hpcore/build/hpcore \ - $hpcore/build/hpstatemon \ $hpcore/examples/echo_contract/contract.js \ ../bin/libfuse3.so.3 \ + ../bin/libb2.so.1 \ ../bin/fusermount3 \ ../bin/websocketd \ ../bin/websocat \ + ../bin/hpfs \ ./consensus-test-continuous.sh \ ./setup-hp.sh \ geveo@$vmip:~/ @@ -25,7 +26,6 @@ if [ $mode = "new" ]; then sshpass -f vmpass.txt scp geveo@$vmip:~/contract/cfg/hp.cfg ./cfg/node$nodeid.json else sshpass -f vmpass.txt scp $hpcore/build/hpcore \ - $hpcore/build/hpstatemon \ $hpcore/examples/echo_contract/contract.js \ ./consensus-test-continuous.sh \ geveo@$vmip:~/