From 3e2b7dbcfa5556f9c6c4d1fbd0cf1f254489584a Mon Sep 17 00:00:00 2001 From: Chalith Desaman Date: Wed, 23 Sep 2020 12:49:06 +0530 Subject: [PATCH] Replaced boost filesystem operations with linux system calls (#125) * Replaced boost filesystem operations with linux syscalls * Removed boost filesystem dependency from CMAKE --- CMakeLists.txt | 1 - src/conf.cpp | 17 +- src/cons/ledger_handler.cpp | 732 ++++++++++++++------------- src/pchheader.hpp | 2 +- src/util.cpp | 100 ++++ src/util.hpp | 10 + test/local-cluster/cluster-create.sh | 2 +- test/vm-cluster/cluster.sh | 2 +- 8 files changed, 491 insertions(+), 375 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d528c887..e8edcfa5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -58,7 +58,6 @@ target_link_libraries(hpcore libblake3.so libboost_system.a libboost_thread.a - libboost_filesystem.a libboost_stacktrace_backtrace.a backtrace ${CMAKE_DL_LIBS} # Needed for stacktrace support diff --git a/src/conf.cpp b/src/conf.cpp index beee2386..b1dbc8b1 100644 --- a/src/conf.cpp +++ b/src/conf.cpp @@ -68,16 +68,16 @@ namespace conf */ int create_contract() { - if (boost::filesystem::exists(ctx.contract_dir)) + if (util::is_dir_exists(ctx.contract_dir)) { std::cout << "Contract dir already exists. Cannot create contract at the same location.\n"; return -1; } - boost::filesystem::create_directories(ctx.config_dir); - boost::filesystem::create_directories(ctx.hist_dir); - boost::filesystem::create_directories(ctx.state_rw_dir); - boost::filesystem::create_directories(ctx.log_dir); + // Recursivly create contract directories. + util::create_dir_tree_recursive(ctx.config_dir); + util::create_dir_tree_recursive(ctx.hist_dir); + util::create_dir_tree_recursive(ctx.state_rw_dir); //Create config file with default settings. @@ -134,8 +134,11 @@ namespace conf // resolving the path through realpath will remove any trailing slash if present basedir = util::realpath(basedir); + exepath = util::realpath(exepath); + + // Take the parent directory path. + ctx.exe_dir = dirname(exepath.data()); - ctx.exe_dir = boost::filesystem::path(util::realpath(exepath)).parent_path().string(); ctx.websocketd_exe_path = ctx.exe_dir + "/" + "websocketd"; ctx.websocat_exe_path = ctx.exe_dir + "/" + "websocat"; ctx.hpfs_exe_path = ctx.exe_dir + "/" + "hpfs"; @@ -507,7 +510,7 @@ namespace conf for (const std::string &path : paths) { - if (!boost::filesystem::exists(path)) + if (!util::is_file_exists(path) && !util::is_dir_exists(path)) { if (path == ctx.tls_key_file || path == ctx.tls_cert_file) { diff --git a/src/cons/ledger_handler.cpp b/src/cons/ledger_handler.cpp index a7a826a0..92b0472e 100644 --- a/src/cons/ledger_handler.cpp +++ b/src/cons/ledger_handler.cpp @@ -12,461 +12,465 @@ namespace cons { -namespace p2pmsg = msg::fbuf::p2pmsg; + namespace p2pmsg = msg::fbuf::p2pmsg; -/** + /** * Create and save ledger from the given proposal message. * @param proposal consensus reached Satge 3 proposal. * @return tuple of current lcl sequence number and file name of the saved lcl. */ -const std::tuple save_ledger(const p2p::proposal &proposal) -{ - const size_t pos = proposal.lcl.find("-"); - uint64_t led_seq_no = 0; + const std::tuple save_ledger(const p2p::proposal &proposal) + { + const size_t pos = proposal.lcl.find("-"); + uint64_t led_seq_no = 0; - if (pos != std::string::npos) - { - led_seq_no = std::stoull(proposal.lcl.substr(0, pos)); //get lcl sequence number. - led_seq_no++; //current lcl sequence number. - } - else - { - //lcl records should follow [ledger sequnce numer]-lcl[lcl hex] format. - LOG_ERROR << "Invalid lcl name: " << proposal.lcl << " when saving ledger."; + if (pos != std::string::npos) + { + led_seq_no = std::stoull(proposal.lcl.substr(0, pos)); //get lcl sequence number. + led_seq_no++; //current lcl sequence number. + } + else + { + //lcl records should follow [ledger sequnce numer]-lcl[lcl hex] format. + LOG_ERROR << "Invalid lcl name: " << proposal.lcl << " when saving ledger."; + } + + //Serialize lcl using flatbuffer ledger schema. + flatbuffers::FlatBufferBuilder builder(1024); + const std::string_view ledger_str = msg::fbuf::ledger::create_ledger_from_proposal(builder, proposal, led_seq_no); + + //Get binary hash of the the serialized lcl. + const std::string lcl = crypto::get_hash(ledger_str); + + //Get hex from binary hash. + std::string lcl_hash; + util::bin2hex(lcl_hash, + reinterpret_cast(lcl.data()), + lcl.size()); + + //construct lcl file name. + //lcl file name should follow [ledger sequnce numer]-lcl[lcl hex] format. + const std::string seq_no_str = std::to_string(led_seq_no); + std::string file_name; + file_name.reserve(lcl_hash.size() + seq_no_str.size() + 1); + file_name.append(seq_no_str) + .append("-") + .append(lcl_hash); + + write_ledger(file_name, ledger_str.data(), ledger_str.size()); + + ledger_cache_entry c; + c.lcl = file_name; + c.state = proposal.state.to_string_view(); + cons::ctx.ledger_cache.emplace(led_seq_no, std::move(c)); + + //Remove old ledgers that exceeds max sequence range. + if (led_seq_no > MAX_LEDGER_SEQUENCE) + { + remove_old_ledgers(led_seq_no - MAX_LEDGER_SEQUENCE); + } + + return std::make_tuple(led_seq_no, std::move(file_name)); } - //Serialize lcl using flatbuffer ledger schema. - flatbuffers::FlatBufferBuilder builder(1024); - const std::string_view ledger_str = msg::fbuf::ledger::create_ledger_from_proposal(builder, proposal, led_seq_no); - - //Get binary hash of the the serialized lcl. - const std::string lcl = crypto::get_hash(ledger_str); - - //Get hex from binary hash. - std::string lcl_hash; - util::bin2hex(lcl_hash, - reinterpret_cast(lcl.data()), - lcl.size()); - - //construct lcl file name. - //lcl file name should follow [ledger sequnce numer]-lcl[lcl hex] format. - const std::string seq_no_str = std::to_string(led_seq_no); - std::string file_name; - file_name.reserve(lcl_hash.size() + seq_no_str.size() + 1); - file_name.append(seq_no_str) - .append("-") - .append(lcl_hash); - - write_ledger(file_name, ledger_str.data(), ledger_str.size()); - - ledger_cache_entry c; - c.lcl = file_name; - c.state = proposal.state.to_string_view(); - cons::ctx.ledger_cache.emplace(led_seq_no, std::move(c)); - - //Remove old ledgers that exceeds max sequence range. - if (led_seq_no > MAX_LEDGER_SEQUENCE) - { - remove_old_ledgers(led_seq_no - MAX_LEDGER_SEQUENCE); - } - - return std::make_tuple(led_seq_no, std::move(file_name)); -} - -/** + /** * Remove old ledgers that exceeds max sequence range from file system and ledger history cache. * @param led_seq_no minimum sequence number to be in history. */ -void remove_old_ledgers(const uint64_t led_seq_no) -{ - std::map::iterator itr; - - std::string dir_path; - - dir_path.reserve(conf::ctx.hist_dir.size() + 1); - dir_path.append(conf::ctx.hist_dir) - .append("/"); - - for (itr = cons::ctx.ledger_cache.begin(); - itr != cons::ctx.ledger_cache.lower_bound(led_seq_no + 1); - itr++) + void remove_old_ledgers(const uint64_t led_seq_no) { - const std::string file_name = itr->second.lcl; - std::string file_path; - file_path.reserve(dir_path.size() + itr->second.lcl.size() + 4); - file_path.append(dir_path) - .append(file_name) - .append(".lcl"); + std::map::iterator itr; - if (boost::filesystem::exists(file_path)) - boost::filesystem::remove(file_path); + std::string dir_path; + + dir_path.reserve(conf::ctx.hist_dir.size() + 1); + dir_path.append(conf::ctx.hist_dir) + .append("/"); + + for (itr = cons::ctx.ledger_cache.begin(); + itr != cons::ctx.ledger_cache.lower_bound(led_seq_no + 1); + itr++) + { + const std::string file_name = itr->second.lcl; + std::string file_path; + file_path.reserve(dir_path.size() + itr->second.lcl.size() + 4); + file_path.append(dir_path) + .append(file_name) + .append(".lcl"); + + if (util::is_file_exists(file_path)) + util::remove_file(file_path); + } + + if (!cons::ctx.ledger_cache.empty()) + cons::ctx.ledger_cache.erase(cons::ctx.ledger_cache.begin(), cons::ctx.ledger_cache.lower_bound(led_seq_no + 1)); } - if (!cons::ctx.ledger_cache.empty()) - cons::ctx.ledger_cache.erase(cons::ctx.ledger_cache.begin(), cons::ctx.ledger_cache.lower_bound(led_seq_no + 1)); -} - -/** + /** * Write ledger to file system. * @param file_name current ledger sequence number. * @param ledger_raw raw lcl data. * @param ledger_size size of the raw lcl data. */ -void write_ledger(const std::string &file_name, const char *ledger_raw, size_t ledger_size) -{ - //create file path to save ledger. - //file name -> [ledger sequnce numer]-[lcl hex] + void write_ledger(const std::string &file_name, const char *ledger_raw, size_t ledger_size) + { + //create file path to save ledger. + //file name -> [ledger sequnce numer]-[lcl hex] - std::string path; + std::string path; - path.reserve(file_name.size() + conf::ctx.hist_dir.size() + 5); - path.append(conf::ctx.hist_dir) - .append("/") - .append(file_name) - .append(".lcl"); + path.reserve(file_name.size() + conf::ctx.hist_dir.size() + 5); + path.append(conf::ctx.hist_dir) + .append("/") + .append(file_name) + .append(".lcl"); - //write ledger to file system - std::ofstream ofs(std::move(path)); - ofs.write(ledger_raw, ledger_size); - ofs.close(); -} + //write ledger to file system + std::ofstream ofs(std::move(path)); + ofs.write(ledger_raw, ledger_size); + ofs.close(); + } -/** + /** * Delete ledger from file system. * @param file_name name of ledger to be deleted. */ -void remove_ledger(const std::string &file_name) -{ - std::string file_path; - file_path.reserve(conf::ctx.hist_dir.size() + file_name.size() + 5); - file_path.append(conf::ctx.hist_dir) - .append("/") - .append(file_name) - .append(".lcl"); - boost::filesystem::remove(file_path); -} + void remove_ledger(const std::string &file_name) + { + std::string file_path; + file_path.reserve(conf::ctx.hist_dir.size() + file_name.size() + 5); + file_path.append(conf::ctx.hist_dir) + .append("/") + .append(file_name) + .append(".lcl"); + util::remove_file(file_path); + } -/** + /** * Retrieve lcl(last closed ledger) information from ledger history. * @return A ledger_history struct representing the lcl. */ -const ledger_history load_ledger() -{ - ledger_history ldg_hist; - //Get all records at lcl history direcory and find the last closed ledger. - size_t latest_pos = 0; - for (const auto &entry : boost::filesystem::directory_iterator(conf::ctx.hist_dir)) + const ledger_history load_ledger() { - const boost::filesystem::path file_path = entry.path(); - const std::string file_name = file_path.stem().string(); + ledger_history ldg_hist; + //Get all records at lcl history direcory and find the last closed ledger. + size_t latest_pos = 0; + for (const auto &entry : util::fetch_dir_entries(conf::ctx.hist_dir)) + { + std::string file_path(conf::ctx.hist_dir); + file_path.append("/").append(entry.d_name); - if (boost::filesystem::is_directory(file_path)) - { - LOG_ERROR << "Found directory " << file_name << " in " << conf::ctx.hist_dir << ". There should be no folders in this directory"; - } - else if (file_path.extension() != ".lcl") - { - LOG_ERROR << "Found invalid file extension: " << file_path.extension() << " for lcl file " << file_name << " in " << conf::ctx.hist_dir; - } - else - { - const size_t pos = file_name.find("-"); - uint64_t seq_no = 0; - - if (pos != std::string::npos) + if (util::is_dir_exists(file_path)) { - seq_no = std::stoull(file_name.substr(0, pos)); - - std::ifstream file(file_path.string(), std::ios::binary | std::ios::ate); - std::streamsize size = file.tellg(); - file.seekg(0, std::ios::beg); - - std::vector buffer(size); - if (file.read(buffer.data(), size)) - { - const uint8_t *ledger_buf_ptr = reinterpret_cast(buffer.data()); - const msg::fbuf::ledger::Ledger *ledger = msg::fbuf::ledger::GetLedger(ledger_buf_ptr); - ledger_cache_entry c; - c.lcl = file_name; - c.state = msg::fbuf::flatbuff_bytes_to_sv(ledger->state()); - - ldg_hist.cache.emplace(seq_no, std::move(c)); //lcl_cache -> [seq_no-hash] - } + LOG_ERROR << "Found directory " << entry.d_name << " in " << conf::ctx.hist_dir << ". There should be no folders in this directory"; } else { - //lcl records should follow [ledger sequnce numer]-lcl[lcl hex] format. - LOG_ERROR << "Invalid lcl file name: " << file_name << " in " << conf::ctx.hist_dir; + const std::string_view extension = util::fetch_file_extension(file_path); + const std::string file_name(util::remove_file_extension(entry.d_name)); + + if (extension != ".lcl") + { + LOG_ERROR << "Found invalid file extension: " << extension << " for lcl file " << entry.d_name << " in " << conf::ctx.hist_dir; + } + + const size_t pos = file_name.find("-"); + uint64_t seq_no = 0; + + if (pos != std::string::npos) + { + seq_no = std::stoull(file_name.substr(0, pos)); + + std::ifstream file(file_path, std::ios::binary | std::ios::ate); + std::streamsize size = file.tellg(); + file.seekg(0, std::ios::beg); + + std::vector buffer(size); + if (file.read(buffer.data(), size)) + { + const uint8_t *ledger_buf_ptr = reinterpret_cast(buffer.data()); + const msg::fbuf::ledger::Ledger *ledger = msg::fbuf::ledger::GetLedger(ledger_buf_ptr); + ledger_cache_entry c; + c.lcl = file_name; + c.state = msg::fbuf::flatbuff_bytes_to_sv(ledger->state()); + + ldg_hist.cache.emplace(seq_no, std::move(c)); //lcl_cache -> [seq_no-hash] + } + } + else + { + //lcl records should follow [ledger sequnce numer]-lcl[lcl hex] format. + LOG_ERROR << "Invalid lcl file name: " << file_name << " in " << conf::ctx.hist_dir; + } } } - } - //check if there is a saved lcl file -> if no send genesis lcl. - if (ldg_hist.cache.empty()) - { - ldg_hist.led_seq_no = 0; - ldg_hist.lcl = GENESIS_LEDGER; - } - else - { - ldg_hist.led_seq_no = ldg_hist.cache.rbegin()->first; - ldg_hist.lcl = ldg_hist.cache.rbegin()->second.lcl; - - //Remove old ledgers that exceeds max sequence range. - if (ldg_hist.led_seq_no > MAX_LEDGER_SEQUENCE) + //check if there is a saved lcl file -> if no send genesis lcl. + if (ldg_hist.cache.empty()) { - remove_old_ledgers(ldg_hist.led_seq_no - MAX_LEDGER_SEQUENCE); + ldg_hist.led_seq_no = 0; + ldg_hist.lcl = GENESIS_LEDGER; } + else + { + ldg_hist.led_seq_no = ldg_hist.cache.rbegin()->first; + ldg_hist.lcl = ldg_hist.cache.rbegin()->second.lcl; + + //Remove old ledgers that exceeds max sequence range. + if (ldg_hist.led_seq_no > MAX_LEDGER_SEQUENCE) + { + remove_old_ledgers(ldg_hist.led_seq_no - MAX_LEDGER_SEQUENCE); + } + } + + return ldg_hist; } - return ldg_hist; -} - -/** + /** * Create and send ledger history request to random node from unl list. * @param minimum_lcl hash of the minimum lcl from which node need lcl history. * @param required_lcl hash of the required lcl. */ -void send_ledger_history_request(const std::string &minimum_lcl, const std::string &required_lcl) -{ - p2p::history_request hr; - hr.required_lcl = required_lcl; - hr.minimum_lcl = minimum_lcl; - - flatbuffers::FlatBufferBuilder fbuf(1024); - p2pmsg::create_msg_from_history_request(fbuf, hr); - p2p::send_message_to_random_peer(fbuf); + void send_ledger_history_request(const std::string &minimum_lcl, const std::string &required_lcl) + { + p2p::history_request hr; + hr.required_lcl = required_lcl; + hr.minimum_lcl = minimum_lcl; - ctx.last_requested_lcl = required_lcl; + flatbuffers::FlatBufferBuilder fbuf(1024); + p2pmsg::create_msg_from_history_request(fbuf, hr); + p2p::send_message_to_random_peer(fbuf); - LOG_DEBUG << "Ledger history request sent. Required lcl:" << required_lcl.substr(0, 15); -} + ctx.last_requested_lcl = required_lcl; -/** + LOG_DEBUG << "Ledger history request sent. Required lcl:" << required_lcl.substr(0, 15); + } + + /** * Check requested lcl is in node's lcl history cache. * @param hr lcl history request information. * @return true if requested lcl is in lcl history cache. */ -bool check_required_lcl_availability(const p2p::history_request &hr) -{ - size_t pos = hr.required_lcl.find("-"); - uint64_t req_seq_no = 0; - - //get sequence number of required lcl - if (pos != std::string::npos) + bool check_required_lcl_availability(const p2p::history_request &hr) { - req_seq_no = std::stoull(hr.required_lcl.substr(0, pos)); //get required lcl sequence number - } + size_t pos = hr.required_lcl.find("-"); + uint64_t req_seq_no = 0; - if (req_seq_no > 0) - { - const auto itr = cons::ctx.ledger_cache.find(req_seq_no); - if (itr == cons::ctx.ledger_cache.end()) + //get sequence number of required lcl + if (pos != std::string::npos) { - LOG_DEBUG << "Required lcl peer asked for is not in our lcl cache."; - //either this node is also not in consesnsus ledger or other node requesting a lcl that is older than node's current - // minimum lcl sequence becuase of maximum ledger history range. - return false; + req_seq_no = std::stoull(hr.required_lcl.substr(0, pos)); //get required lcl sequence number } - else if (itr->second.lcl != hr.required_lcl) - { - LOG_DEBUG << "Required lcl peer asked for is not in our lcl cache."; - //either this node or requesting node is in a fork condition. - return false; - } - } - else - { - return false; //Very rare case: node asking for the genisis lcl. - } - return true; -} -/** + if (req_seq_no > 0) + { + const auto itr = cons::ctx.ledger_cache.find(req_seq_no); + if (itr == cons::ctx.ledger_cache.end()) + { + LOG_DEBUG << "Required lcl peer asked for is not in our lcl cache."; + //either this node is also not in consesnsus ledger or other node requesting a lcl that is older than node's current + // minimum lcl sequence becuase of maximum ledger history range. + return false; + } + else if (itr->second.lcl != hr.required_lcl) + { + LOG_DEBUG << "Required lcl peer asked for is not in our lcl cache."; + //either this node or requesting node is in a fork condition. + return false; + } + } + else + { + return false; //Very rare case: node asking for the genisis lcl. + } + return true; + } + + /** * Retrieve lcl(last closed ledger) information from ledger history. * @param hr lcl history request information. * @return A ledger history response containing requested ledger details. */ -const p2p::history_response retrieve_ledger_history(const p2p::history_request &hr) -{ - p2p::history_response history_response; - size_t pos = hr.minimum_lcl.find("-"); - uint64_t min_seq_no = 0; - - //get sequence number of minimum lcl required - if (pos != std::string::npos) + const p2p::history_response retrieve_ledger_history(const p2p::history_request &hr) { - min_seq_no = std::stoull(hr.minimum_lcl.substr(0, pos)); //get required lcl sequence number - } + p2p::history_response history_response; + size_t pos = hr.minimum_lcl.find("-"); + uint64_t min_seq_no = 0; - const auto itr = cons::ctx.ledger_cache.find(min_seq_no); - if (itr != cons::ctx.ledger_cache.end()) //requested minimum lcl is not in our lcl history cache - { - min_seq_no = itr->first; - //check whether minimum lcl node ask for is same as this node's. - //eventhough sequence number are same, lcl hash can be changed if one of node is in a fork condition. - if (hr.minimum_lcl != itr->second.lcl) + //get sequence number of minimum lcl required + if (pos != std::string::npos) { - LOG_DEBUG << "Invalid minimum ledger. Recieved min hash: " << hr.minimum_lcl << " Node hash: " << itr->second.lcl; + min_seq_no = std::stoull(hr.minimum_lcl.substr(0, pos)); //get required lcl sequence number + } + + const auto itr = cons::ctx.ledger_cache.find(min_seq_no); + if (itr != cons::ctx.ledger_cache.end()) //requested minimum lcl is not in our lcl history cache + { + min_seq_no = itr->first; + //check whether minimum lcl node ask for is same as this node's. + //eventhough sequence number are same, lcl hash can be changed if one of node is in a fork condition. + if (hr.minimum_lcl != itr->second.lcl) + { + LOG_DEBUG << "Invalid minimum ledger. Recieved min hash: " << hr.minimum_lcl << " Node hash: " << itr->second.lcl; + history_response.error = p2p::LEDGER_RESPONSE_ERROR::INVALID_MIN_LEDGER; + return history_response; + } + } + else if (min_seq_no > cons::ctx.ledger_cache.rbegin()->first) //Recieved minimum lcl sequence is ahead of node's lcl sequence. + { + LOG_DEBUG << "Invalid minimum ledger. Recieved minimum sequence number is ahead of node current lcl sequence. Recvd hash: " << hr.minimum_lcl; history_response.error = p2p::LEDGER_RESPONSE_ERROR::INVALID_MIN_LEDGER; return history_response; } - } - else if (min_seq_no > cons::ctx.ledger_cache.rbegin()->first) //Recieved minimum lcl sequence is ahead of node's lcl sequence. - { - LOG_DEBUG << "Invalid minimum ledger. Recieved minimum sequence number is ahead of node current lcl sequence. Recvd hash: " << hr.minimum_lcl; - history_response.error = p2p::LEDGER_RESPONSE_ERROR::INVALID_MIN_LEDGER; + else + { + LOG_DEBUG << "Minimum lcl peer asked for is not in our lcl cache. Therefore sending from node minimum lcl"; + min_seq_no = cons::ctx.ledger_cache.begin()->first; + } + + //LOG_DBG << "history request min seq: " << std::to_string(min_seq_no); + + //copy current history cache. + std::map led_cache = cons::ctx.ledger_cache; + + //filter out cache and get raw files here. + led_cache.erase( + led_cache.begin(), + led_cache.lower_bound(min_seq_no)); + + //Get raw content of lcls that going to be send. + for (auto &[seq_no, cache] : led_cache) + { + p2p::history_ledger ledger; + ledger.lcl = cache.lcl; + ledger.state = cache.state; + + std::string path; + + path.reserve(conf::ctx.hist_dir.size() + cache.lcl.size() + 5); + path.append(conf::ctx.hist_dir) + .append("/") + .append(cache.lcl) + .append(".lcl"); + + //read lcl file + std::ifstream file(path, std::ios::binary | std::ios::ate); + std::streamsize size = file.tellg(); + file.seekg(0, std::ios::beg); + + std::vector buffer(size); + if (file.read(buffer.data(), size)) + { + ledger.raw_ledger = reinterpret_cast &>(buffer); + history_response.hist_ledgers.emplace(seq_no, ledger); + } + } + return history_response; } - else - { - LOG_DEBUG << "Minimum lcl peer asked for is not in our lcl cache. Therefore sending from node minimum lcl"; - min_seq_no = cons::ctx.ledger_cache.begin()->first; - } - //LOG_DEBUG << "history request min seq: " << std::to_string(min_seq_no); - - //copy current history cache. - std::map led_cache = cons::ctx.ledger_cache; - - //filter out cache and get raw files here. - led_cache.erase( - led_cache.begin(), - led_cache.lower_bound(min_seq_no)); - - //Get raw content of lcls that going to be send. - for (auto &[seq_no, cache] : led_cache) - { - p2p::history_ledger ledger; - ledger.lcl = cache.lcl; - ledger.state = cache.state; - - std::string path; - - path.reserve(conf::ctx.hist_dir.size() + cache.lcl.size() + 5); - path.append(conf::ctx.hist_dir) - .append("/") - .append(cache.lcl) - .append(".lcl"); - - //read lcl file - std::ifstream file(path, std::ios::binary | std::ios::ate); - std::streamsize size = file.tellg(); - file.seekg(0, std::ios::beg); - - std::vector buffer(size); - if (file.read(buffer.data(), size)) - { - ledger.raw_ledger = reinterpret_cast &>(buffer); - history_response.hist_ledgers.emplace(seq_no, ledger); - } - } - - return history_response; -} - -/** + /** * Handle recieved ledger history response. * @param hr lcl history request information. * @return peer outbound message object with ledger history response. */ -void handle_ledger_history_response(const p2p::history_response &hr) -{ - //check response object contains - if (ctx.last_requested_lcl.empty()) + void handle_ledger_history_response(const p2p::history_response &hr) { - LOG_DEBUG << "Peer sent us a history response but we never asked for one!"; - return; - } - - if (hr.error == p2p::LEDGER_RESPONSE_ERROR::INVALID_MIN_LEDGER) - { - // This means we are in a fork ledger.Remove/rollback current ledger. - // Basically in the long run we'll rolback one by one untill we catch up to valid minimum ledger . - remove_ledger(ctx.lcl); - cons::ctx.ledger_cache.erase(ctx.ledger_cache.rbegin()->first); - LOG_DEBUG << "Invalid min ledger. Removed last ledger."; - } - else - { - //check whether recieved lcl history contains the current lcl node required. - bool have_requested_lcl = false; - for (auto &[seq_no, ledger] : hr.hist_ledgers) + //check response object contains + if (ctx.last_requested_lcl.empty()) { - if (ctx.last_requested_lcl == ledger.lcl) - { - have_requested_lcl = true; - break; - } - } - - if (!have_requested_lcl) - { - LOG_DEBUG << "Peer sent us a history response but not containing the lcl we asked for! " << hr.hist_ledgers.size(); + LOG_DEBUG << "Peer sent us a history response but we never asked for one!"; return; } - //Check integrity of recieved lcl list. - //By checking recieved lcl hashes matches lcl content by applying hashing for each raw content. - for (auto &[seq_no, ledger] : hr.hist_ledgers) + if (hr.error == p2p::LEDGER_RESPONSE_ERROR::INVALID_MIN_LEDGER) { - const size_t pos = ledger.lcl.find("-"); - std::string rec_lcl_hash = ledger.lcl.substr((pos + 1), (ledger.lcl.size() - 1)); - - //Get binary hash of the the serialized lcl. - const std::string lcl = crypto::get_hash(&ledger.raw_ledger[0], ledger.raw_ledger.size()); - - //Get hex from binary hash - std::string lcl_hash; - - util::bin2hex(lcl_hash, - reinterpret_cast(lcl.data()), - lcl.size()); - - //LOG_DEBUG << "passed lcl: " << ledger.lcl << " gen lcl: " << lcl_hash; - - //recieved lcl hash and hash generated from recieved lcl content doesn't match -> abandon applying it - if (lcl_hash != rec_lcl_hash) + // This means we are in a fork ledger.Remove/rollback current ledger. + // Basically in the long run we'll rolback one by one untill we catch up to valid minimum ledger . + remove_ledger(ctx.lcl); + cons::ctx.ledger_cache.erase(ctx.ledger_cache.rbegin()->first); + LOG_DEBUG << "Invalid min ledger. Removed last ledger."; + } + else + { + //check whether recieved lcl history contains the current lcl node required. + bool have_requested_lcl = false; + for (auto &[seq_no, ledger] : hr.hist_ledgers) { - LOG_WARNING << "peer sent us a history response we asked for but the ledger data does not match the ledger hashes"; - //todo: we should penalize peer who send this? + if (ctx.last_requested_lcl == ledger.lcl) + { + have_requested_lcl = true; + break; + } + } + + if (!have_requested_lcl) + { + LOG_DEBUG << "Peer sent us a history response but not containing the lcl we asked for! " << hr.hist_ledgers.size(); return; } - } - } - //Execution to here means the history data sent checks out - //Save recieved lcl in file system and update lcl history cache - for (auto &[seq_no, ledger] : hr.hist_ledgers) - { - auto prev_dup_itr = cons::ctx.ledger_cache.find(seq_no); - if (prev_dup_itr != cons::ctx.ledger_cache.end()) + //Check integrity of recieved lcl list. + //By checking recieved lcl hashes matches lcl content by applying hashing for each raw content. + for (auto &[seq_no, ledger] : hr.hist_ledgers) + { + const size_t pos = ledger.lcl.find("-"); + std::string rec_lcl_hash = ledger.lcl.substr((pos + 1), (ledger.lcl.size() - 1)); + + //Get binary hash of the the serialized lcl. + const std::string lcl = crypto::get_hash(&ledger.raw_ledger[0], ledger.raw_ledger.size()); + + //Get hex from binary hash + std::string lcl_hash; + + util::bin2hex(lcl_hash, + reinterpret_cast(lcl.data()), + lcl.size()); + + //LOG_DBG << "passed lcl: " << ledger.lcl << " gen lcl: " << lcl_hash; + + //recieved lcl hash and hash generated from recieved lcl content doesn't match -> abandon applying it + if (lcl_hash != rec_lcl_hash) + { + LOG_WARNING << "peer sent us a history response we asked for but the ledger data does not match the ledger hashes"; + //todo: we should penalize peer who send this? + return; + } + } + } + + //Execution to here means the history data sent checks out + //Save recieved lcl in file system and update lcl history cache + for (auto &[seq_no, ledger] : hr.hist_ledgers) { - remove_ledger(prev_dup_itr->second.lcl); - cons::ctx.ledger_cache.erase(prev_dup_itr); + auto prev_dup_itr = cons::ctx.ledger_cache.find(seq_no); + if (prev_dup_itr != cons::ctx.ledger_cache.end()) + { + remove_ledger(prev_dup_itr->second.lcl); + cons::ctx.ledger_cache.erase(prev_dup_itr); + } + write_ledger(ledger.lcl, reinterpret_cast(&ledger.raw_ledger[0]), ledger.raw_ledger.size()); + ledger_cache_entry l; + l.lcl = ledger.lcl; + l.state = ledger.state; + cons::ctx.ledger_cache.emplace(seq_no, std::move(l)); } - write_ledger(ledger.lcl, reinterpret_cast(&ledger.raw_ledger[0]), ledger.raw_ledger.size()); - ledger_cache_entry l; - l.lcl = ledger.lcl; - l.state = ledger.state; - cons::ctx.ledger_cache.emplace(seq_no, std::move(l)); - } - ctx.last_requested_lcl = ""; + ctx.last_requested_lcl = ""; - if (cons::ctx.ledger_cache.empty()) - { - cons::ctx.led_seq_no = 0; - cons::ctx.lcl = GENESIS_LEDGER; - } - else - { - const auto latest_lcl_itr = cons::ctx.ledger_cache.rbegin(); - cons::ctx.lcl = latest_lcl_itr->second.lcl; - cons::ctx.led_seq_no = latest_lcl_itr->first; - } + if (cons::ctx.ledger_cache.empty()) + { + cons::ctx.led_seq_no = 0; + cons::ctx.lcl = GENESIS_LEDGER; + } + else + { + const auto latest_lcl_itr = cons::ctx.ledger_cache.rbegin(); + cons::ctx.lcl = latest_lcl_itr->second.lcl; + cons::ctx.led_seq_no = latest_lcl_itr->first; + } - LOG_INFO << "lcl sync complete. New lcl:" << cons::ctx.lcl.substr(0, 15); -} + LOG_INFO << "lcl sync complete. New lcl:" << cons::ctx.lcl.substr(0, 15); + } } // namespace cons \ No newline at end of file diff --git a/src/pchheader.hpp b/src/pchheader.hpp index dd519bfa..1cc89610 100644 --- a/src/pchheader.hpp +++ b/src/pchheader.hpp @@ -5,10 +5,10 @@ #define BOOST_STACKTRACE_USE_BACKTRACE #include -#include #include #include #include +#include #include #include #include diff --git a/src/util.cpp b/src/util.cpp index 6e656b27..f28f63c3 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -234,12 +234,33 @@ namespace util return 0; } + /** + * Check whether given directory exists. + * @param path Directory path. + * @return Returns true if given directory exists otherwise false. + */ bool is_dir_exists(std::string_view path) { struct stat st; return (stat(path.data(), &st) == 0 && S_ISDIR(st.st_mode)); } + /** + * Check whether given file exists. + * @param path File path. + * @return Returns true if give file exists otherwise false. + */ + bool is_file_exists(std::string_view path) + { + struct stat st; + return (stat(path.data(), &st) == 0 && S_ISREG(st.st_mode)); + } + + /** + * Recursively creates directories and sub-directories if not exist. + * @param path Directory path. + * @return Returns 0 operations succeeded otherwise -1. + */ int create_dir_tree_recursive(std::string_view path) { if (strcmp(path.data(), "/") == 0) // No need of checking if we are at root. @@ -263,4 +284,83 @@ namespace util return 0; } + /** + * Fetch all the files and directiries inside the given directory. + * @param path Directory path. + * @return Returns the list of entries inside the directory. + */ + std::list fetch_dir_entries(std::string_view path) + { + std::list entries; + DIR *dr; + + // Open the directory stream. + if (dr = opendir(path.data())) + { + // Take next directory entry from the directory stream. + struct dirent *en; + while (en = readdir(dr)) + { + // Push into the entries list if reading directory entry is not current directory entry + // or previous directory entry. + if (std::strcmp(en->d_name, ".") != 0 && std::strcmp(en->d_name, "..") != 0) + { + entries.push_back(*en); + } + } + // Close directory stream. + closedir(dr); + } + + return entries; + } + + /** + * Fetch file extension from the file path. + * @param path File path. + * @return Returns the file extension as a string_view. + */ + std::string_view fetch_file_extension(std::string_view path) + { + // Get the position of right most "." in the file path. + const std::size_t pos = path.rfind('.'); + + if (pos != std::string::npos) + { + // Take the sub string after the ".". + return path.substr(pos); + } + + return ""; + } + + /** + * Remove file extension from file name. + * @param file_name File name. + * @return Returns the file name without extension. + */ + std::string_view remove_file_extension(std::string_view file_name) + { + // Get the position of right most "." in the file name. + const std::size_t pos = file_name.rfind('.'); + + if (pos != std::string::npos) + { + // Take the sub string till the "." from the beginning. + return file_name.substr(0, pos); + } + + return file_name; + } + + /** + * Deletes a file. + * @param path File path. + * @return Returs 0 if succeed else -1. + */ + int remove_file(std::string_view path) + { + return remove(path.data()); + } + } // namespace util diff --git a/src/util.hpp b/src/util.hpp index e326c9b1..e9236ac7 100644 --- a/src/util.hpp +++ b/src/util.hpp @@ -96,8 +96,18 @@ namespace util bool is_dir_exists(std::string_view path); + bool is_file_exists(std::string_view path); + int create_dir_tree_recursive(std::string_view path); + std::list fetch_dir_entries(std::string_view path); + + std::string_view fetch_file_extension(std::string_view path); + + std::string_view remove_file_extension(std::string_view file_name); + + int remove_file(std::string_view path); + } // namespace util #endif diff --git a/test/local-cluster/cluster-create.sh b/test/local-cluster/cluster-create.sh index 3c063e9b..5479bac2 100755 --- a/test/local-cluster/cluster-create.sh +++ b/test/local-cluster/cluster-create.sh @@ -53,7 +53,7 @@ do peerport: ${peerport}, \ pubport: ${pubport}, \ roundtime: 2000, \ - loglevel: 'debug', \ + loglevel: 'dbg', \ loggers:['console', 'file'] \ }, null, 2)" > hp.cfg rm tmp.json diff --git a/test/vm-cluster/cluster.sh b/test/vm-cluster/cluster.sh index 9e734764..bd5649a0 100755 --- a/test/vm-cluster/cluster.sh +++ b/test/vm-cluster/cluster.sh @@ -249,7 +249,7 @@ do peers:${mypeers}, \ unl:${myunl}, \ roundtime: 1000, \ - loglevel: 'debug', \ + loglevel: 'dbg', \ loggers:['console', 'file'] \ }, null, 2)" > ./cfg/node$n.cfg