diff --git a/CMakeLists.txt b/CMakeLists.txt index 35062a0..599e1da 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -21,6 +21,7 @@ add_executable(sagent src/crypto.cpp src/sqlite.cpp src/hp_manager.cpp + src/hpfs_manager.cpp src/msg/msg_parser.cpp src/msg/json/msg_json.cpp src/main.cpp @@ -36,6 +37,7 @@ target_link_libraries(sagent add_custom_command(TARGET sagent POST_BUILD COMMAND cp ./dependencies/bin/hpws ./build/ + COMMAND cp ./dependencies/bin/hpfs ./build/ COMMAND cp -r ./dependencies/default_contract ./build/ COMMAND cp ./bootstrap-contract/script.sh ./build/default_contract/contract_fs/seed/state/script.sh ) diff --git a/README.md b/README.md index 761824b..dd3a3d6 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,8 @@ Code is divided into subsystems via namespaces. **hp::** Contains hotpocket instance management related helper functions. +**hpfs::** Contains hpfs instance management related helper functions. + **msg::** Extract message data from received raw messages. **salog::** Handles logging. Creates and prints the logs according to the configured log section in the json config. diff --git a/dependencies/bin/hpfs b/dependencies/bin/hpfs new file mode 100755 index 0000000..512fb88 Binary files /dev/null and b/dependencies/bin/hpfs differ diff --git a/src/conf.cpp b/src/conf.cpp index 8221978..7013ec4 100644 --- a/src/conf.cpp +++ b/src/conf.cpp @@ -105,6 +105,7 @@ namespace conf // Take the parent directory path. ctx.exe_dir = dirname(exepath.data()); ctx.hpws_exe_path = ctx.exe_dir + "/hpws"; + ctx.hpfs_exe_path = ctx.exe_dir + "/hpfs"; ctx.default_contract_path = ctx.exe_dir + "/default_contract"; ctx.config_dir = ctx.exe_dir + "/cfg"; ctx.config_file = ctx.config_dir + "/sa.cfg"; @@ -118,11 +119,12 @@ namespace conf */ int validate_dir_paths() { - const std::string paths[4] = { + const std::string paths[5] = { ctx.config_file, ctx.log_dir, ctx.data_dir, - ctx.hpws_exe_path}; + ctx.hpws_exe_path, + ctx.default_contract_path}; for (const std::string &path : paths) { diff --git a/src/conf.hpp b/src/conf.hpp index 189b00b..0e6f4e0 100644 --- a/src/conf.hpp +++ b/src/conf.hpp @@ -83,6 +83,7 @@ namespace conf std::string command; // The CLI command issued to launch Sashimono agent std::string exe_dir; // Hot Pocket executable dir. std::string hpws_exe_path; // hpws executable file path. + std::string hpfs_exe_path; // hpfs executable file path. std::string default_contract_path; // Path to default contract. std::string config_dir; // Config dir full path. diff --git a/src/hp_manager.cpp b/src/hp_manager.cpp index b981b3b..4d5ddad 100644 --- a/src/hp_manager.cpp +++ b/src/hp_manager.cpp @@ -143,8 +143,11 @@ namespace hp } const std::string name = crypto::generate_uuid(); // This will be the docker container name as well as the contract folder name. - + std::string hpfs_log_level; + bool is_full_history; if (create_contract(info, name, owner_pubkey, instance_ports) != 0 || + read_contract_cfg_values(name, hpfs_log_level, is_full_history) == -1 || + hpfs::start_fs_processes(name, hpfs_log_level, is_full_history) == -1 || run_container(name, instance_ports) != 0 || // Gives 3200 if docker failed. sqlite::insert_hp_instance_row(db, info) == -1) { @@ -169,12 +172,11 @@ namespace hp int run_container(const std::string &folder_name, const ports &assigned_ports) { // We instruct the demon to restart the container automatically once the container exits except manually stopping. - const std::string command = "docker run -t -i -d --network=hpnet --stop-signal=SIGINT --name=" + folder_name + " \ + const std::string command = "docker run -t -i -d --stop-signal=SIGINT --name=" + folder_name + " \ -p " + std::to_string(assigned_ports.user_port) + ":" + std::to_string(assigned_ports.user_port) + " \ -p " + std::to_string(assigned_ports.peer_port) + ":" + std::to_string(assigned_ports.peer_port) + " \ - --device /dev/fuse --cap-add SYS_ADMIN --security-opt apparmor:unconfined \ --restart unless-stopped --mount type=bind,source=" + conf::cfg.hp.instance_folder + "/" + folder_name + ",target=/contract \ @@ -232,7 +234,12 @@ namespace hp return -1; } - if (docker_start(container_name) != 0 || sqlite::update_status_in_container(db, container_name, CONTAINER_STATES[STATES::RUNNING]) == -1) + std::string hpfs_log_level; + bool is_full_history; + if (read_contract_cfg_values(container_name, hpfs_log_level, is_full_history) == -1 || + hpfs::start_fs_processes(container_name, hpfs_log_level, is_full_history) == -1 || + docker_start(container_name) != 0 || + sqlite::update_status_in_container(db, container_name, CONTAINER_STATES[STATES::RUNNING]) == -1) { LOG_ERROR << "Error when starting container. name: " << container_name; return -1; @@ -346,7 +353,6 @@ namespace hp d["log"]["max_mbytes_per_file"] = 5; d["log"]["max_file_count"] = 10; - d["node"]["public_key"] = pubkey_hex; d["node"]["private_key"] = util::to_hex(seckey); d["contract"]["id"] = contract_id; @@ -356,6 +362,7 @@ namespace hp d["contract"]["bin_args"] = owner_pubkey; d["mesh"]["port"] = assigned_ports.peer_port; d["user"]["port"] = assigned_ports.user_port; + d["hpfs"]["external"] = true; if (write_json_file(config_fd, d) == -1) { @@ -439,4 +446,82 @@ namespace hp return -1; } + /** + * Read only required contract config values + * @param contract_name Name of the contract. + * @param log_level Log level to be read. + * @param is_full_history Contract history mode. + * @return 0 on success. -1 on failure. + */ + int read_contract_cfg_values(std::string_view contract_name, std::string &log_level, bool &is_full_history) + { + const std::string folder_path = conf::cfg.hp.instance_folder + "/" + contract_name.data(); + + // Read the config file into json document object. + const std::string config_file_path = folder_path + "/cfg/hp.cfg"; + const int config_fd = open(config_file_path.data(), O_RDONLY); + if (config_fd == -1) + { + LOG_ERROR << errno << ": Error opening hp config file " << config_file_path; + return -1; + } + + std::string buf; + if (util::read_from_fd(config_fd, buf) == -1) + { + LOG_ERROR << "Error reading from the config file. " << errno; + close(config_fd); + return -1; + } + + jsoncons::ojson d; + try + { + d = jsoncons::ojson::parse(buf, jsoncons::strict_json_parsing()); + } + catch (const std::exception &e) + { + LOG_ERROR << "Invalid contract config file format. " << e.what(); + return -1; + } + buf.clear(); + + try + { + log_level = d["hpfs"]["log"]["log_level"].as(); + } + catch (const std::exception &e) + { + LOG_ERROR << "Invalid contract config hpfs log. " << e.what(); + return -1; + } + + const std::unordered_set valid_loglevels({"dbg", "inf", "wrn", "err"}); + if (valid_loglevels.count(log_level) != 1) + { + LOG_ERROR << "Invalid hpfs loglevel configured. Valid values: dbg|inf|wrn|err"; + return -1; + } + + try + { + if (d["node"]["history"] == "full") + is_full_history = true; + else if (d["node"]["history"] == "custom") + is_full_history = false; + else + { + LOG_ERROR << "Invalid history mode. 'full' or 'custom' expected."; + return -1; + } + } + catch (const std::exception &e) + { + LOG_ERROR << "Invalid contract config history mode. " << e.what(); + return -1; + } + + return 0; + } + } // namespace hp diff --git a/src/hp_manager.hpp b/src/hp_manager.hpp index f67d632..b5ed195 100644 --- a/src/hp_manager.hpp +++ b/src/hp_manager.hpp @@ -2,6 +2,7 @@ #define _SA_HP_MANAGER_ #include "pchheader.hpp" +#include "hpfs_manager.hpp" namespace hp { @@ -58,5 +59,6 @@ namespace hp int create_contract(instance_info &info, const std::string &folder_name, std::string_view owner_pubkey, const ports &assigned_ports); int write_json_file(const int fd, const jsoncons::ojson &d); int check_instance_status(std::string_view name, std::string &status); + int read_contract_cfg_values(std::string_view contract_name, std::string &log_level, bool &is_full_history); } // namespace hp #endif \ No newline at end of file diff --git a/src/hpfs_manager.cpp b/src/hpfs_manager.cpp new file mode 100644 index 0000000..f5d5f97 --- /dev/null +++ b/src/hpfs_manager.cpp @@ -0,0 +1,130 @@ +#include "hpfs_manager.hpp" +#include "util/util.hpp" +#include "conf.hpp" + +namespace hpfs +{ + constexpr ino_t ROOT_INO = 1; + constexpr uint16_t HPFS_PROCESS_INIT_TIMEOUT = 2000; + constexpr uint16_t HPFS_INIT_CHECK_INTERVAL = 20; + + /** + * Starts the hpfs process for the instance. + * @param fs_dir File system directory + * @param mount_dir Mount directory. + * @param log_level Log level for the hpfs. + * @param merge Whether changes are needed to be merged. + * @return -1 on error, pid of the spawned hpfs process if success. + */ + int start_hpfs_process(std::string_view fs_dir, std::string_view mount_dir, std::string_view log_level, const bool merge) + { + const pid_t pid = fork(); + if (pid > 0) + { + // Sashimono process. + + LOG_DEBUG << "Starting hpfs process at " << mount_dir << "."; + + // Wait until hpfs is initialized properly. + const uint16_t max_retries = HPFS_PROCESS_INIT_TIMEOUT / HPFS_INIT_CHECK_INTERVAL; + bool hpfs_initialized = false; + uint16_t retry_count = 0; + do + { + util::sleep(HPFS_INIT_CHECK_INTERVAL); + + // Check if hpfs process is still running. + // Sending signal 0 to test whether process exist. + if (util::kill_process(pid, false, 0) == -1) + { + LOG_ERROR << "hpfs process " << pid << " has stopped."; + break; + } + + // We check for the specific inode no. of the mounted root dir. That means hpfs FUSE interface is up. + struct stat st; + if (stat(mount_dir.data(), &st) == -1) + { + LOG_ERROR << errno << ": Error in checking hpfs status at mount " << mount_dir << "."; + break; + } + + hpfs_initialized = (st.st_ino == ROOT_INO); + // Keep retrying until root inode no. matches or timeout occurs. + + } while (!hpfs_initialized && ++retry_count <= max_retries); + + // Kill the process if hpfs couldn't be initialized properly. + if (!hpfs_initialized) + { + LOG_ERROR << "Couldn't initialize hpfs process at mount " << mount_dir << "."; + util::kill_process(pid, true); + return -1; + } + + LOG_DEBUG << "hpfs process started. pid:" << pid; + } + else if (pid == 0) + { + // hpfs process. + util::fork_detach(); + + // Detach hpfs terminal outputs from the sagent terminal, These will be printed in the trace log of particular hpfs mount. + int fd = open("/dev/null", O_WRONLY); + dup2(fd, STDOUT_FILENO); + dup2(fd, STDERR_FILENO); + close(fd); + + std::string trace_arg = "trace="; + trace_arg.append(log_level); + char *execv_args[] = { + conf::ctx.hpfs_exe_path.data(), + (char *)"fs", + (char *)fs_dir.data(), + (char *)mount_dir.data(), + (char *)(merge ? "merge=true" : "merge=false"), + (char *)trace_arg.data(), + NULL}; + + const int ret = execv(execv_args[0], execv_args); + std::cerr << errno << ": hpfs process start failed at mount " << mount_dir << ".\n"; + exit(1); + } + else + { + LOG_ERROR << errno << ": fork() failed when starting hpfs process at mount " << mount_dir << "."; + return -1; + } + + return pid; + } + + /** + * Creates hpfs processes for the instance. + * @param contract_dir Contract directory. + * @param log_level Log level for hpfs. + * @param is_full_history Whether hpfs instances are for full history node. + * @param pids pids of the hpfs instances. + * @return -1 on error and 0 on success and pids will be populated. + * + */ + int start_fs_processes(std::string_view contract_dir, std::string_view log_level, const bool is_full_history) + { + const std::string contract_fs_path = conf::cfg.hp.instance_folder + "/" + (const char *)contract_dir.data() + "/contract_fs"; + if (start_hpfs_process(contract_fs_path, contract_fs_path + "/mnt", log_level, !is_full_history) <= 0) + { + LOG_ERROR << errno << " : Error occured while starting contract_fs processes - " << contract_dir; + return -1; + } + + const std::string ledger_fs_path = conf::cfg.hp.instance_folder + "/" + (const char *)contract_dir.data() + "/ledger_fs"; + if (start_hpfs_process(ledger_fs_path, ledger_fs_path + "/mnt", log_level, true) <= 0) + { + LOG_ERROR << errno << " : Error occured while starting ledger_fs processes - " << contract_dir; + return -1; + } + + return 0; + } + +} // namespace hp diff --git a/src/hpfs_manager.hpp b/src/hpfs_manager.hpp new file mode 100644 index 0000000..d549c92 --- /dev/null +++ b/src/hpfs_manager.hpp @@ -0,0 +1,11 @@ +#ifndef _SA_HPFS_MANAGER_ +#define _SA_HPFS_MANAGER_ + +#include "pchheader.hpp" + +namespace hpfs +{ + int start_hpfs_process(std::string_view fs_dir, std::string_view mount_dir, std::string_view log_level, const bool merge = true); + int start_fs_processes(std::string_view contract_dir, std::string_view log_level, const bool is_full_history); +} // namespace hpfs +#endif \ No newline at end of file diff --git a/src/pchheader.hpp b/src/pchheader.hpp index 99e3173..e8a2259 100644 --- a/src/pchheader.hpp +++ b/src/pchheader.hpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include diff --git a/src/sqlite.cpp b/src/sqlite.cpp index d71feb2..ac0892a 100644 --- a/src/sqlite.cpp +++ b/src/sqlite.cpp @@ -347,8 +347,8 @@ namespace sqlite { // Populate only the necessary fields. info.status = std::string(reinterpret_cast(sqlite3_column_text(stmt, 2))); - info.assigned_ports.peer_port = sqlite3_column_int64(stmt, 5); - info.assigned_ports.user_port = sqlite3_column_int64(stmt, 6); + info.assigned_ports.peer_port = sqlite3_column_int64(stmt, 6); + info.assigned_ports.user_port = sqlite3_column_int64(stmt, 7); // Finalize and distroys the statement. sqlite3_finalize(stmt); diff --git a/src/util/util.cpp b/src/util/util.cpp index b97c8f2..54bece4 100644 --- a/src/util/util.cpp +++ b/src/util/util.cpp @@ -193,4 +193,23 @@ namespace util 1, FTW_DEPTH | FTW_PHYS); } + // Kill a process with a signal and if specified, wait until it stops running. + int kill_process(const pid_t pid, const bool wait, const int signal) + { + if (kill(pid, signal) == -1) + { + LOG_ERROR << errno << ": Error issuing signal to pid " << pid; + return -1; + } + + const int wait_options = wait ? 0 : WNOHANG; + if (waitpid(pid, NULL, wait_options) == -1) + { + LOG_ERROR << errno << ": waitpid after kill (pid:" << pid << ") failed."; + return -1; + } + + return 0; + } + } // namespace util diff --git a/src/util/util.hpp b/src/util/util.hpp index 4855cef..340ed06 100644 --- a/src/util/util.hpp +++ b/src/util/util.hpp @@ -30,7 +30,9 @@ namespace util uint64_t get_epoch_milliseconds(); - int remove_directory_recursively(std::string_view dir_path); + int remove_directory_recursively(std::string_view dir_path); + + int kill_process(const pid_t pid, const bool wait, const int signal = SIGINT); } // namespace util