From b89dbe0a2c2a6a1b68d2db1896d4ace246e9748b Mon Sep 17 00:00:00 2001 From: Ravin Perera <33562092+ravinsp@users.noreply.github.com> Date: Wed, 10 Jun 2020 20:51:45 +0530 Subject: [PATCH] Rearchitected state sync with hpfs. (#96) --- CMakeLists.txt | 7 +- README.md | 2 +- examples/echo_contract/contract.js | 14 +- src/comm/comm_client.cpp | 2 + src/comm/comm_server.cpp | 29 +- src/cons/cons.cpp | 127 ++++--- src/cons/cons.hpp | 13 +- src/cons/ledger_handler.cpp | 2 +- src/cons/state_handler.cpp | 361 -------------------- src/cons/state_handler.hpp | 52 --- src/fbschema/ledger_helpers.cpp | 2 +- src/fbschema/p2pmsg_content.fbs | 8 +- src/fbschema/p2pmsg_content_generated.h | 56 ++-- src/fbschema/p2pmsg_helpers.cpp | 76 ++--- src/fbschema/p2pmsg_helpers.hpp | 96 +++--- src/hpfs/hpfs.cpp | 92 ++++- src/hpfs/hpfs.hpp | 10 +- src/main.cpp | 9 +- src/p2p/p2p.cpp | 5 +- src/p2p/p2p.hpp | 5 +- src/p2p/peer_session_handler.cpp | 321 +++++++++--------- src/pchheader.hpp | 1 + src/sc.cpp | 8 +- src/sc.hpp | 4 +- src/state/state_serve.cpp | 252 ++++++++++++++ src/state/state_serve.hpp | 23 ++ src/state/state_sync.cpp | 429 ++++++++++++++++++++++++ src/state/state_sync.hpp | 83 +++++ src/util.cpp | 48 ++- src/util.hpp | 105 +++--- test/bin/hpfs | Bin 174232 -> 174232 bytes test/local-cluster/cluster-create.sh | 8 +- 32 files changed, 1383 insertions(+), 867 deletions(-) delete mode 100644 src/cons/state_handler.cpp delete mode 100644 src/cons/state_handler.hpp create mode 100644 src/state/state_serve.cpp create mode 100644 src/state/state_serve.hpp create mode 100644 src/state/state_sync.cpp create mode 100644 src/state/state_sync.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 0fbccbe0..ef0b1ab5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -54,7 +54,8 @@ add_executable(hpcore src/usr/usr.cpp src/cons/cons.cpp src/cons/ledger_handler.cpp - src/cons/state_handler.cpp + src/state/state_sync.cpp + src/state/state_serve.cpp src/main.cpp ) target_link_libraries(hpcore @@ -75,8 +76,8 @@ add_dependencies(hpcore ) add_custom_command(TARGET hpcore POST_BUILD - COMMAND strip ./build/hpcore - COMMAND strip ./build/appbill + # COMMAND strip ./build/hpcore + # COMMAND strip ./build/appbill COMMAND cp ./test/bin/websocketd ./test/bin/websocat ./test/bin/hpfs ./build/ ) diff --git a/README.md b/README.md index 0bf740cd..fe80a571 100644 --- a/README.md +++ b/README.md @@ -96,7 +96,7 @@ Code is divided into subsystems via namespaces. **p2p::** Handles peer-to-peer connections and message exchange between nodes. Makes use of **crypto** and **comm**. -**cons::** Handles consensus and proposal rounds. Makes use of **usr**, **p2p** and **proc** +**cons::** Handles consensus and proposal rounds. Makes use of **usr**, **p2p** and **sc** **comm::** Handles generic web sockets communication functionality. Mainly acts as a wrapper for websocketd/websocat. diff --git a/examples/echo_contract/contract.js b/examples/echo_contract/contract.js index c15cb925..3a8086c4 100644 --- a/examples/echo_contract/contract.js +++ b/examples/echo_contract/contract.js @@ -22,17 +22,17 @@ Object.keys(hpargs.usrfd).forEach(function (key, index) { } }); -let nplinput = fs.readFileSync(hpargs.nplfd[0], 'utf8'); -if (nplinput.length > 0) { - console.log("Input received from hp:"); +if (hpargs.nplfd[0] != -1) { + let nplinput = fs.readFileSync(hpargs.nplfd[0], 'utf8'); + console.log("Input received from peers:"); console.log(nplinput); fs.writeSync(hpargs.nplfd[1], "Echoing: " + nplinput); } -let hpinput = fs.readFileSync(hpargs.hpfd[0], 'utf8'); -if (hpinput.length > 0) { - //console.log("Input received from hp:"); - //console.log(hpinput); +if (hpargs.hpfd[0] != -1) { + let hpinput = fs.readFileSync(hpargs.hpfd[0], 'utf8'); + console.log("Input received from hp:"); + console.log(hpinput); fs.writeSync(hpargs.hpfd[1], "Echoing: " + hpinput); } diff --git a/src/comm/comm_client.cpp b/src/comm/comm_client.cpp index 658495d8..0638c669 100644 --- a/src/comm/comm_client.cpp +++ b/src/comm/comm_client.cpp @@ -59,6 +59,8 @@ namespace comm else if (pid == 0) { // Websocat process. + util::unmask_signal(); + close(write_pipe[1]); //parent write close(read_pipe[0]); //parent read diff --git a/src/comm/comm_server.cpp b/src/comm/comm_server.cpp index b01e99ab..ef7fe6b1 100644 --- a/src/comm/comm_server.cpp +++ b/src/comm/comm_server.cpp @@ -196,17 +196,24 @@ namespace comm { // New client connected. const std::string ip = get_cgi_ip(client_fd); - - if (corebill::is_banned(ip)) + if (!ip.empty()) { - LOG_DBG << "Dropping connection for banned host " << ip; - close(client_fd); + if (corebill::is_banned(ip)) + { + LOG_DBG << "Dropping connection for banned host " << ip; + close(client_fd); + } + else + { + comm_session session(ip, client_fd, client_fd, session_type, is_binary, true, metric_thresholds); + if (session.on_connect() == 0) + sessions.try_emplace(client_fd, std::move(session)); + } } else { - comm_session session(ip, client_fd, client_fd, session_type, is_binary, true, metric_thresholds); - if (session.on_connect() == 0) - sessions.try_emplace(client_fd, std::move(session)); + close(client_fd); + LOG_ERR << "Closed bad client socket: " << client_fd; } } } @@ -292,6 +299,8 @@ namespace comm else if (pid == 0) { // Websocketd process. + util::unmask_signal(); + // We are using websocketd forked repo: https://github.com/codetsunami/websocketd if (firewall_out > 0) @@ -303,9 +312,9 @@ namespace comm } std::string max_frame = std::string("--maxframe=") - .append(use_size_header - ? "4294967296" // 4GB - : std::to_string(max_msg_size)); + .append(use_size_header + ? "4294967296" // 4GB + : std::to_string(max_msg_size)); // Fill process args. char *execv_args[] = { diff --git a/src/cons/cons.cpp b/src/cons/cons.cpp index b4d87c2d..5c3add80 100644 --- a/src/cons/cons.cpp +++ b/src/cons/cons.cpp @@ -12,8 +12,8 @@ #include "../sc.hpp" #include "../hpfs/h32.hpp" #include "../hpfs/hpfs.hpp" +#include "../state/state_sync.hpp" #include "ledger_handler.hpp" -#include "state_handler.hpp" #include "cons.hpp" namespace p2pmsg = fbschema::p2pmsg; @@ -36,19 +36,19 @@ namespace cons int init() { - //set start stage - ctx.stage = 0; - //load lcl details from lcl history. ledger_history ldr_hist = load_ledger(); ctx.led_seq_no = ldr_hist.led_seq_no; ctx.lcl = ldr_hist.lcl; ctx.ledger_cache.swap(ldr_hist.cache); - if (hpfs::get_root_hash(ctx.curr_state_hash) == -1) + if (get_initial_state_hash(ctx.state) == -1) + { + LOG_ERR << "Failed to get initial state hash."; return -1; + } - LOG_INFO << "Initial state: " << ctx.curr_state_hash; + LOG_INFO << "Initial state: " << ctx.state; // We allocate 1/5 of the round time to each stage expect stage 3. For stage 3 we allocate 2/5. // Stage 3 is allocated an extra stage_time unit becayse a node needs enough time to @@ -65,10 +65,6 @@ namespace cons */ void deinit() { - if (init_success) - { - - } } int run_consensus() @@ -158,14 +154,12 @@ namespace cons vote_counter votes; // check if we're ahead/behind of consensus lcl - bool is_lcl_desync, should_request_history; + bool is_lcl_desync = false, should_request_history = false; std::string majority_lcl; check_lcl_votes(is_lcl_desync, should_request_history, majority_lcl, votes); if (is_lcl_desync) { - ctx.is_lcl_syncing = true; - if (should_request_history) { LOG_INFO << "Syncing lcl. Curr lcl:" << cons::ctx.lcl.substr(0, 15) << " majority:" << majority_lcl.substr(0, 15); @@ -191,14 +185,16 @@ namespace cons } else { - const bool lcl_syncing_just_finished = ctx.is_lcl_syncing; - ctx.is_lcl_syncing = false; + bool is_state_desync = false; + hpfs::h32 majority_state = hpfs::h32_empty; + check_state_votes(is_state_desync, majority_state, votes); - if (lcl_syncing_just_finished) - ; //TODO: Check and compare majotiry state and start state sync. - bool is_state_syncing = false; - - if (!is_state_syncing) + if (is_state_desync) + { + conf::change_operating_mode(conf::OPERATING_MODE::OBSERVER); + state_sync::set_target(majority_state, on_state_sync_completion); + } + else { conf::change_operating_mode(conf::OPERATING_MODE::PROPOSER); @@ -214,7 +210,7 @@ namespace cons // node has finished a consensus round (all 4 stages). LOG_INFO << "****Stage 3 consensus reached**** (lcl:" << ctx.lcl.substr(0, 15) - << " state:" << ctx.curr_state_hash << ")"; + << " state:" << ctx.state << ")"; } } } @@ -247,8 +243,8 @@ namespace cons << " hout:" << cp.hash_outputs.size() << " ts:" << std::to_string(cp.time) << " lcl:" << cp.lcl.substr(0, 15) - << " state:" << cp.curr_state_hash - << " self:" << self; + << " state:" << cp.state + << (self ? " [self]" : ""); } else { @@ -479,6 +475,9 @@ namespace cons int pid = fork(); if (pid == 0) { + // appbill process. + util::unmask_signal(); + // before execution chdir into a valid the latest state data directory that contains an appbill.table chdir(conf::ctx.state_rw_dir.c_str()); int ret = execv(execv_args[0], execv_args); @@ -513,7 +512,7 @@ namespace cons stg_prop.time = ctx.time_now; stg_prop.stage = 0; stg_prop.lcl = ctx.lcl; - stg_prop.curr_state_hash = ctx.curr_state_hash; + stg_prop.state = ctx.state; // Populate the proposal with set of candidate user pubkeys. for (const std::string &pubkey : ctx.candidate_users) @@ -545,7 +544,7 @@ namespace cons // if there's a fork condition we will either request history and state from // our peers or we will halt depending on level of consensus on the sides of the fork stg_prop.lcl = ctx.lcl; - stg_prop.curr_state_hash = ctx.curr_state_hash; + stg_prop.state = ctx.state; // Vote for rest of the proposal fields by looking at candidate proposals. for (const auto &[pubkey, cp] : ctx.candidate_proposals) @@ -651,10 +650,6 @@ namespace cons { LOG_DBG << "Not enough peers proposing to perform consensus. votes:" << std::to_string(total_lcl_votes) << " needed:" << std::to_string(MAJORITY_THRESHOLD * conf::cfg.unl.size()); is_desync = true; - - //Not enough nodes are propsing. So Node is switching to Proposer if it's in observer mode. - conf::change_operating_mode(conf::OPERATING_MODE::PROPOSER); - return; } @@ -692,6 +687,33 @@ namespace cons } } + /** + * Check state against the winning and canonical state + * @param votes The voting table. + */ + void check_state_votes(bool &is_desync, hpfs::h32 &majority_state, vote_counter &votes) + { + for (const auto &[pubkey, cp] : ctx.candidate_proposals) + { + increment(votes.state, cp.state); + } + + int32_t winning_votes = 0; + for (const auto [state, votes] : votes.state) + { + if (votes > winning_votes) + { + winning_votes = votes; + majority_state = state; + } + } + + { + std::lock_guard(ctx.state_sync_lock); + is_desync = (ctx.state != majority_state); + } + } + /** * Returns the consensus percentage threshold for the specified stage. * @param stage The consensus stage [1, 2, 3] @@ -797,30 +819,6 @@ namespace cons } } - /** - * Check state against the winning and canonical state - * @param votes The voting table. - */ - void check_state(vote_counter &votes) - { - hpfs::h32 majority_state = hpfs::h32_empty; - - for (const auto &[pubkey, cp] : ctx.candidate_proposals) - { - increment(votes.state, cp.curr_state_hash); - } - - int32_t winning_votes = 0; - for (const auto [state, votes] : votes.state) - { - if (votes > winning_votes) - { - winning_votes = votes; - majority_state = state; - } - } - } - /** * Transfers consensus-reached inputs into the provided contract buf map so it can be fed into the contract process. * @param bufmap The contract bufmap which needs to be populated with inputs. @@ -908,7 +906,7 @@ namespace cons sc::contract_iobuf_pair hpscbufpair; return sc::exec_contract( sc::contract_exec_args(time_now, useriobufmap, nplbufpair, hpscbufpair), - ctx.curr_state_hash); + ctx.state); } /** @@ -925,4 +923,25 @@ namespace cons counter.try_emplace(candidate, 1); } + /** + * Get the contract state hash. + */ + int get_initial_state_hash(hpfs::h32 &hash) + { + pid_t pid; + std::string mount_dir; + if (hpfs::start_fs_session(pid, mount_dir, "ro", true) == -1) + return -1; + + int res = get_hash(hash, mount_dir, "/"); + util::kill_process(pid, true); + return res; + } + + void on_state_sync_completion(const hpfs::h32 new_state) + { + std::lock_guard(ctx.state_sync_lock); + ctx.state = new_state; + } + } // namespace cons diff --git a/src/cons/cons.hpp b/src/cons/cons.hpp index 4eccda76..05cafb48 100644 --- a/src/cons/cons.hpp +++ b/src/cons/cons.hpp @@ -8,7 +8,6 @@ #include "../usr/user_input.hpp" #include "../hpfs/h32.hpp" #include "ledger_handler.hpp" -#include "state_handler.hpp" namespace cons { @@ -74,7 +73,7 @@ struct consensus_context uint64_t time_now = 0; std::string lcl; uint64_t led_seq_no = 0; - hpfs::h32 curr_state_hash; + hpfs::h32 state = hpfs::h32_empty; //Map of closed ledgers(only lrdgername[sequnece_number-hash], state hash) with sequence number as map key. //contains closed ledgers from latest to latest - MAX_LEDGER_SEQUENCE. @@ -82,12 +81,12 @@ struct consensus_context //We will use this to track lcls related logic.- track state, lcl request, response. std::map ledger_cache; std::string last_requested_lcl; - bool is_lcl_syncing = false; //ledger close time of previous hash uint16_t stage_time = 0; // Time allocated to a consensus stage. uint16_t stage_reset_wait_threshold = 0; // Minimum stage wait time to reset the stage. + std::mutex state_sync_lock; bool is_shutting_down = false; consensus_context() @@ -134,6 +133,8 @@ void broadcast_proposal(const p2p::proposal &p); void check_lcl_votes(bool &is_desync, bool &should_request_history, std::string &majority_lcl, vote_counter &votes); +void check_state_votes(bool &is_desync, hpfs::h32 &majority_state, vote_counter &votes); + float_t get_stage_threshold(const uint8_t stage); void timewait_stage(const bool reset, const uint64_t time); @@ -146,8 +147,6 @@ int apply_ledger(const p2p::proposal &proposal); void dispatch_user_outputs(const p2p::proposal &cons_prop); -void check_state(vote_counter &votes); - void feed_user_inputs_to_contract_bufmap(sc::contract_bufmap_t &bufmap, const p2p::proposal &cons_prop); void extract_user_outputs_from_contract_bufmap(sc::contract_bufmap_t &bufmap); @@ -159,6 +158,10 @@ int run_contract_binary(const int64_t time_now, sc::contract_bufmap_t &useriobuf template void increment(std::map &counter, const T &candidate); +int get_initial_state_hash(hpfs::h32 &hash); + +void on_state_sync_completion(const hpfs::h32 new_state); + } // namespace cons #endif diff --git a/src/cons/ledger_handler.cpp b/src/cons/ledger_handler.cpp index 12c14367..0f1edab0 100644 --- a/src/cons/ledger_handler.cpp +++ b/src/cons/ledger_handler.cpp @@ -61,7 +61,7 @@ const std::tuple save_ledger(const p2p::proposal &p ledger_cache_entry c; c.lcl = file_name; - c.state = proposal.curr_state_hash.to_string_view(); + c.state = proposal.state.to_string_view(); cons::ctx.ledger_cache.emplace(led_seq_no, std::move(c)); //Remove old ledgers that exceeds max sequence range. diff --git a/src/cons/state_handler.cpp b/src/cons/state_handler.cpp deleted file mode 100644 index 40cc7b98..00000000 --- a/src/cons/state_handler.cpp +++ /dev/null @@ -1,361 +0,0 @@ -#include "state_handler.hpp" -#include "../fbschema/p2pmsg_helpers.hpp" -#include "../fbschema/p2pmsg_content_generated.h" -#include "../fbschema/common_helpers.hpp" -#include "../p2p/p2p.hpp" -#include "../pchheader.hpp" -#include "../cons/cons.hpp" -#include "../hplog.hpp" -#include "../util.hpp" - -namespace cons -{ - -// Max number of requests that can be awaiting response at any given time. -constexpr uint16_t MAX_AWAITING_REQUESTS = 1; -// Syncing loop sleep delay. -constexpr uint16_t SYNC_LOOP_WAIT = 100; - -// List of state responses flatbuffer messages to be processed. -std::list candidate_state_responses; - -// List of pending sync requests to be sent out. -std::list pending_requests; - -// List of submitted requests we are awaiting responses for, keyed by expected response hash. -std::unordered_map submitted_requests; - -/** - * Sends a state request to a random peer. - * @param path Requested file or dir path. - * @param is_file Whether the requested path if a file or dir. - * @param block_id The requested block id. Only relevant if requesting a file block. Otherwise -1. - * @param expected_hash The expected hash of the requested data. The peer will ignore the request if their hash is different. - */ -void request_state_from_peer(const std::string &path, const bool is_file, const int32_t block_id, const hpfs::h32 expected_hash) -{ - p2p::state_request sr; - sr.parent_path = path; - sr.is_file = is_file; - sr.block_id = block_id; - sr.expected_hash = expected_hash; - - flatbuffers::FlatBufferBuilder fbuf(1024); - fbschema::p2pmsg::create_msg_from_state_request(fbuf, sr, ctx.lcl); - p2p::send_message_to_random_peer(fbuf); //todo: send to a node that hold the majority state to improve reliability of retrieving state. -} - -/** - * Creats the reply message for a given state request. - * @param msg The peer outbound message reference to build up the reply message. - * @param sr The state request which should be replied to. - */ -int create_state_response(flatbuffers::FlatBufferBuilder &fbuf, const p2p::state_request &sr) -{ - // If block_id > -1 this means this is a file block data request. - if (sr.block_id > -1) - { - // Vector to hold the block bytes. Normally block size is constant BLOCK_SIZE (4MB), but the - // last block of a file may have a smaller size. - std::vector block; - - // TODO: get block - - p2p::block_response resp; - resp.path = sr.parent_path; - resp.block_id = sr.block_id; - resp.hash = sr.expected_hash; - resp.data = std::string_view(reinterpret_cast(block.data()), block.size()); - - fbschema::p2pmsg::create_msg_from_block_response(fbuf, resp, ctx.lcl); - } - else - { - // File state request means we have to reply with the file block hash map. - if (sr.is_file) - { - std::vector existing_block_hashmap; - - // TODO: get block hash list - // TODO: get file length - std::size_t file_length = 0; - - fbschema::p2pmsg::create_msg_from_filehashmap_response(fbuf, sr.parent_path, existing_block_hashmap, file_length, sr.expected_hash, ctx.lcl); - } - else - { - // If the state request is for a directory we need to reply with the file system entries and their hashes inside that dir. - std::unordered_map existing_fs_entries; - - // TODO: get fs entry hashes - - fbschema::p2pmsg::create_msg_from_fsentry_response(fbuf, sr.parent_path, existing_fs_entries, sr.expected_hash, ctx.lcl); - } - } - - return 0; -} - -/** - * Initiates state sync process by setting up context variables and sending the initial state request. - * @param state_hash_to_request Peer's expected state hash. If peer doesn't have this as its state hash the - * request will be ignord. - */ -void start_state_sync(const hpfs::h32 state_hash_to_request) -{ - { - std::lock_guard lock(p2p::ctx.collected_msgs.state_response_mutex); - p2p::ctx.collected_msgs.state_response.clear(); - } - - { - candidate_state_responses.clear(); - pending_requests.clear(); - submitted_requests.clear(); - } - - // Send the root state request. - submit_request(backlog_item{BACKLOG_ITEM_TYPE::DIR, "/", -1, state_hash_to_request}); -} - -/** - * Runs the state sync loop. - */ -int run_state_sync_iterator() -{ - util::mask_signal(); - - while (true) - { - if (ctx.is_shutting_down) - break; - - util::sleep(SYNC_LOOP_WAIT); - - // TODO: Also bypass peer session handler state responses if we're not syncing. - - { - std::lock_guard lock(p2p::ctx.collected_msgs.state_response_mutex); - - // Move collected state responses over to local candidate responses list. - if (!p2p::ctx.collected_msgs.state_response.empty()) - candidate_state_responses.splice(candidate_state_responses.end(), p2p::ctx.collected_msgs.state_response); - } - - for (auto &response : candidate_state_responses) - { - if (ctx.is_shutting_down) - break; - - const fbschema::p2pmsg::Content *content = fbschema::p2pmsg::GetContent(response.data()); - const fbschema::p2pmsg::State_Response_Message *resp_msg = content->message_as_State_Response_Message(); - - // Check whether we are actually waiting for this response's hash. If not, ignore it. - hpfs::h32 response_hash = fbschema::flatbuff_bytes_to_hash(resp_msg->hash()); - const auto pending_resp_itr = submitted_requests.find(response_hash); - if (pending_resp_itr == submitted_requests.end()) - continue; - - // Now that we have received matching hash, remove it from the waiting list. - submitted_requests.erase(pending_resp_itr); - - // Process the message based on response type. - const fbschema::p2pmsg::State_Response msg_type = resp_msg->state_response_type(); - - if (msg_type == fbschema::p2pmsg::State_Response_Fs_Entry_Response) - { - if (handle_fs_entry_response(resp_msg->state_response_as_Fs_Entry_Response()) == -1) - return -1; - } - else if (msg_type == fbschema::p2pmsg::State_Response_File_HashMap_Response) - { - if (handle_file_hashmap_response(resp_msg->state_response_as_File_HashMap_Response()) == -1) - return -1; - } - else if (msg_type == fbschema::p2pmsg::State_Response_Block_Response) - { - if (handle_file_block_response(resp_msg->state_response_as_Block_Response()) == -1) - return -1; - } - } - - candidate_state_responses.clear(); - - // Check for long-awaited responses and re-request them. - for (auto &[hash, request] : submitted_requests) - { - if (ctx.is_shutting_down) - break; - - // We wait for half of round time before each request is resubmitted. - if (request.waiting_cycles < (conf::cfg.roundtime / (SYNC_LOOP_WAIT * 2))) - { - // Increment counter. - request.waiting_cycles++; - } - else - { - // Reset the counter and re-submit request. - request.waiting_cycles = 0; - LOG_DBG << "Resubmitting state request..."; - submit_request(request); - } - } - - // Check whether we can submit any more requests. - if (!pending_requests.empty() && submitted_requests.size() < MAX_AWAITING_REQUESTS) - { - const uint16_t available_slots = MAX_AWAITING_REQUESTS - submitted_requests.size(); - for (int i = 0; i < available_slots && !pending_requests.empty(); i++) - { - if (ctx.is_shutting_down) - break; - - const backlog_item &request = pending_requests.front(); - submit_request(request); - pending_requests.pop_front(); - } - } - } - - return 0; -} - -/** - * Submits a pending state request to the peer. - */ -void submit_request(const backlog_item &request) -{ - LOG_DBG << "Submitting state request. type:" << request.type << " path:" << request.path << " block_id:" << request.block_id; - - submitted_requests.try_emplace(request.expected_hash, request); - - const bool is_file = request.type != BACKLOG_ITEM_TYPE::DIR; - request_state_from_peer(request.path, is_file, request.block_id, request.expected_hash); -} - -/** - * Process state file system entry response for a directory. - */ -int handle_fs_entry_response(const fbschema::p2pmsg::Fs_Entry_Response *fs_entry_resp) -{ - std::unordered_map state_fs_entry_list; - fbschema::p2pmsg::flatbuf_statefshashentry_to_statefshashentry(state_fs_entry_list, fs_entry_resp->entries()); - - std::unordered_map existing_fs_entries; - std::string_view root_path_sv = fbschema::flatbuff_str_to_sv(fs_entry_resp->path()); - std::string root_path_str(root_path_sv.data(), root_path_sv.size()); - - // TODO: Create state path dir if not exist. - // TODO: Get existing fs entries hash map. - // if (!statefs::is_dir_exists(root_path_str)) - // { - // statefs::create_dir(root_path_str); - // } - // else - // { - // if (statefs::get_fs_entry_hashes(existing_fs_entries, std::move(root_path_str), hpfs::h32_empty) == -1) - // return -1; - // } - - // Request more info on fs entries that exist on both sides but are different. - for (const auto &[path, fs_entry] : existing_fs_entries) - { - const auto fs_itr = state_fs_entry_list.find(path); - if (fs_itr != state_fs_entry_list.end()) - { - if (fs_itr->second.hash != fs_entry.hash) - { - if (fs_entry.is_file) - pending_requests.push_front(backlog_item{BACKLOG_ITEM_TYPE::FILE, path, -1, fs_itr->second.hash}); - else - pending_requests.push_back(backlog_item{BACKLOG_ITEM_TYPE::DIR, path, -1, fs_itr->second.hash}); - } - - state_fs_entry_list.erase(fs_itr); - } - else - { - // If there was an entry that does not exist on other side, delete it from this node. - if (fs_entry.is_file) - { - //if (statefs::delete_file(path) == -1) - // return -1; - } - else - { - //if (statefs::delete_dir(path) == -1) - // return -1; - } - } - } - - // Queue the remaining fs entries (that this node does not have at all) to request. - for (const auto &[path, fs_entry] : state_fs_entry_list) - { - if (fs_entry.is_file) - pending_requests.push_front(backlog_item{BACKLOG_ITEM_TYPE::FILE, path, -1, fs_entry.hash}); - else - pending_requests.push_back(backlog_item{BACKLOG_ITEM_TYPE::DIR, path, -1, fs_entry.hash}); - } - - return 0; -} - -int handle_file_hashmap_response(const fbschema::p2pmsg::File_HashMap_Response *file_resp) -{ - std::string_view path_sv = fbschema::flatbuff_str_to_sv(file_resp->path()); - const std::string path_str(path_sv.data(), path_sv.size()); - - std::vector existing_block_hashmap; - //if (statefs::get_block_hash_map(existing_block_hashmap, path_str, hpfs::h32_empty) == -1) - // return -1; - - const hpfs::h32 *existing_hashes = reinterpret_cast(existing_block_hashmap.data()); - auto existing_hash_count = existing_block_hashmap.size() / sizeof(hpfs::h32); - - const hpfs::h32 *resp_hashes = reinterpret_cast(file_resp->hash_map()->data()); - auto resp_hash_count = file_resp->hash_map()->size() / sizeof(hpfs::h32); - - auto insert_itr = pending_requests.begin(); - - for (int block_id = 0; block_id < existing_hash_count; ++block_id) - { - if (block_id >= resp_hash_count) - break; - - if (existing_hashes[block_id] != resp_hashes[block_id]) - { - // Insert at front to give priority to block requests while preserving block order. - pending_requests.insert(insert_itr, backlog_item{BACKLOG_ITEM_TYPE::BLOCK, path_str, block_id, resp_hashes[block_id]}); - } - } - - if (existing_hash_count > resp_hash_count) - { - //if (statefs::truncate_file(path_str, file_resp->file_length()) == -1) - // return -1; - } - else if (existing_hash_count < resp_hash_count) - { - for (int block_id = existing_hash_count; block_id < resp_hash_count; ++block_id) - { - // Insert at front to give priority to block requests while preserving block order. - pending_requests.insert(insert_itr, backlog_item{BACKLOG_ITEM_TYPE::BLOCK, path_str, block_id, resp_hashes[block_id]}); - } - } - - return 0; -} - -int handle_file_block_response(const fbschema::p2pmsg::Block_Response *block_msg) -{ - p2p::block_response block_resp = fbschema::p2pmsg::create_block_response_from_msg(*block_msg); - - //if (statefs::write_block(block_resp.path, block_resp.block_id, block_resp.data.data(), block_resp.data.size()) == -1) - // return -1; - - return 0; -} - -} // namespace cons \ No newline at end of file diff --git a/src/cons/state_handler.hpp b/src/cons/state_handler.hpp deleted file mode 100644 index 47963fd3..00000000 --- a/src/cons/state_handler.hpp +++ /dev/null @@ -1,52 +0,0 @@ -#ifndef _HP_CONS_STATE_HANDLER_ -#define _HP_CONS_STATE_HANDLER_ - -#include "../pchheader.hpp" -#include "../p2p/p2p.hpp" -#include "../fbschema/p2pmsg_content_generated.h" -#include "../hpfs/h32.hpp" - -namespace cons -{ - -enum BACKLOG_ITEM_TYPE -{ - DIR = 0, - FILE = 1, - BLOCK = 2 -}; - -// Represents a queued up state sync operation which needs to be performed. -struct backlog_item -{ - BACKLOG_ITEM_TYPE type; - std::string path; - int32_t block_id = -1; // Only relevant if type=BLOCK - hpfs::h32 expected_hash; - - // No. of cycles that this item has been waiting in pending state. - // Used by pending_responses list to increase wait count. - int16_t waiting_cycles = 0; -}; - -extern std::list candidate_state_responses; - -int create_state_response(flatbuffers::FlatBufferBuilder &fbuf, const p2p::state_request &sr); - -void request_state_from_peer(const std::string &path, const bool is_file, const int32_t block_id, const hpfs::h32 expected_hash); - -void start_state_sync(const hpfs::h32 state_hash_to_request); - -int run_state_sync_iterator(); - -void submit_request(const backlog_item &request); - -int handle_fs_entry_response(const fbschema::p2pmsg::Fs_Entry_Response *fs_entry_resp); - -int handle_file_hashmap_response(const fbschema::p2pmsg::File_HashMap_Response *file_resp); - -int handle_file_block_response(const fbschema::p2pmsg::Block_Response *block_msg); - -} // namespace cons - -#endif \ No newline at end of file diff --git a/src/fbschema/ledger_helpers.cpp b/src/fbschema/ledger_helpers.cpp index 90f814c3..08206df1 100644 --- a/src/fbschema/ledger_helpers.cpp +++ b/src/fbschema/ledger_helpers.cpp @@ -19,7 +19,7 @@ const std::string_view create_ledger_from_proposal(flatbuffers::FlatBufferBuilde seq_no, p.time, sv_to_flatbuff_bytes(builder, p.lcl), - sv_to_flatbuff_bytes(builder, p.curr_state_hash.to_string_view()), + sv_to_flatbuff_bytes(builder, p.state.to_string_view()), stringlist_to_flatbuf_bytearrayvector(builder, p.users), stringlist_to_flatbuf_bytearrayvector(builder, p.hash_inputs), stringlist_to_flatbuf_bytearrayvector(builder, p.hash_outputs)); diff --git a/src/fbschema/p2pmsg_content.fbs b/src/fbschema/p2pmsg_content.fbs index ba169f34..bebe1142 100644 --- a/src/fbschema/p2pmsg_content.fbs +++ b/src/fbschema/p2pmsg_content.fbs @@ -1,4 +1,6 @@ -//IDL file for p2p message content schema. +// IDL file for p2p message content schema. +// flatc -o src/fbschema/ --gen-mutable --cpp src/fbschema/p2pmsg_content.fbs + include "common_schema.fbs"; namespace fbschema.p2pmsg; @@ -37,7 +39,7 @@ table Proposal_Message { //Proposal type message schema users:[ByteArray]; hash_inputs:[ByteArray]; //stage > 0 inputs (hash of stage 0 inputs) hash_outputs:[ByteArray]; //stage > 0 outputs (hash of stage 0 outputs) - curr_state_hash: [ubyte]; + state: [ubyte]; } table Npl_Message { //NPL type message schema @@ -104,7 +106,7 @@ table Block_Response{ } table State_FS_Hash_Entry{ - path: string; + name: string; is_file: bool; hash: [ubyte]; } diff --git a/src/fbschema/p2pmsg_content_generated.h b/src/fbschema/p2pmsg_content_generated.h index 0df19a81..ba4e6f3d 100644 --- a/src/fbschema/p2pmsg_content_generated.h +++ b/src/fbschema/p2pmsg_content_generated.h @@ -700,7 +700,7 @@ struct Proposal_Message FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { VT_USERS = 8, VT_HASH_INPUTS = 10, VT_HASH_OUTPUTS = 12, - VT_CURR_STATE_HASH = 14 + VT_STATE = 14 }; uint8_t stage() const { return GetField(VT_STAGE, 0); @@ -732,11 +732,11 @@ struct Proposal_Message FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { flatbuffers::Vector> *mutable_hash_outputs() { return GetPointer> *>(VT_HASH_OUTPUTS); } - const flatbuffers::Vector *curr_state_hash() const { - return GetPointer *>(VT_CURR_STATE_HASH); + const flatbuffers::Vector *state() const { + return GetPointer *>(VT_STATE); } - flatbuffers::Vector *mutable_curr_state_hash() { - return GetPointer *>(VT_CURR_STATE_HASH); + flatbuffers::Vector *mutable_state() { + return GetPointer *>(VT_STATE); } bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && @@ -751,8 +751,8 @@ struct Proposal_Message FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { VerifyOffset(verifier, VT_HASH_OUTPUTS) && verifier.VerifyVector(hash_outputs()) && verifier.VerifyVectorOfTables(hash_outputs()) && - VerifyOffset(verifier, VT_CURR_STATE_HASH) && - verifier.VerifyVector(curr_state_hash()) && + VerifyOffset(verifier, VT_STATE) && + verifier.VerifyVector(state()) && verifier.EndTable(); } }; @@ -776,8 +776,8 @@ struct Proposal_MessageBuilder { void add_hash_outputs(flatbuffers::Offset>> hash_outputs) { fbb_.AddOffset(Proposal_Message::VT_HASH_OUTPUTS, hash_outputs); } - void add_curr_state_hash(flatbuffers::Offset> curr_state_hash) { - fbb_.AddOffset(Proposal_Message::VT_CURR_STATE_HASH, curr_state_hash); + void add_state(flatbuffers::Offset> state) { + fbb_.AddOffset(Proposal_Message::VT_STATE, state); } explicit Proposal_MessageBuilder(flatbuffers::FlatBufferBuilder &_fbb) : fbb_(_fbb) { @@ -797,10 +797,10 @@ inline flatbuffers::Offset CreateProposal_Message( flatbuffers::Offset>> users = 0, flatbuffers::Offset>> hash_inputs = 0, flatbuffers::Offset>> hash_outputs = 0, - flatbuffers::Offset> curr_state_hash = 0) { + flatbuffers::Offset> state = 0) { Proposal_MessageBuilder builder_(_fbb); builder_.add_time(time); - builder_.add_curr_state_hash(curr_state_hash); + builder_.add_state(state); builder_.add_hash_outputs(hash_outputs); builder_.add_hash_inputs(hash_inputs); builder_.add_users(users); @@ -815,11 +815,11 @@ inline flatbuffers::Offset CreateProposal_MessageDirect( const std::vector> *users = nullptr, const std::vector> *hash_inputs = nullptr, const std::vector> *hash_outputs = nullptr, - const std::vector *curr_state_hash = nullptr) { + const std::vector *state = nullptr) { auto users__ = users ? _fbb.CreateVector>(*users) : 0; auto hash_inputs__ = hash_inputs ? _fbb.CreateVector>(*hash_inputs) : 0; auto hash_outputs__ = hash_outputs ? _fbb.CreateVector>(*hash_outputs) : 0; - auto curr_state_hash__ = curr_state_hash ? _fbb.CreateVector(*curr_state_hash) : 0; + auto state__ = state ? _fbb.CreateVector(*state) : 0; return fbschema::p2pmsg::CreateProposal_Message( _fbb, stage, @@ -827,7 +827,7 @@ inline flatbuffers::Offset CreateProposal_MessageDirect( users__, hash_inputs__, hash_outputs__, - curr_state_hash__); + state__); } struct Npl_Message FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { @@ -1623,15 +1623,15 @@ inline flatbuffers::Offset CreateBlock_ResponseDirect( struct State_FS_Hash_Entry FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef State_FS_Hash_EntryBuilder Builder; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { - VT_PATH = 4, + VT_NAME = 4, VT_IS_FILE = 6, VT_HASH = 8 }; - const flatbuffers::String *path() const { - return GetPointer(VT_PATH); + const flatbuffers::String *name() const { + return GetPointer(VT_NAME); } - flatbuffers::String *mutable_path() { - return GetPointer(VT_PATH); + flatbuffers::String *mutable_name() { + return GetPointer(VT_NAME); } bool is_file() const { return GetField(VT_IS_FILE, 0) != 0; @@ -1647,8 +1647,8 @@ struct State_FS_Hash_Entry FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table } bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && - VerifyOffset(verifier, VT_PATH) && - verifier.VerifyString(path()) && + VerifyOffset(verifier, VT_NAME) && + verifier.VerifyString(name()) && VerifyField(verifier, VT_IS_FILE) && VerifyOffset(verifier, VT_HASH) && verifier.VerifyVector(hash()) && @@ -1660,8 +1660,8 @@ struct State_FS_Hash_EntryBuilder { typedef State_FS_Hash_Entry Table; flatbuffers::FlatBufferBuilder &fbb_; flatbuffers::uoffset_t start_; - void add_path(flatbuffers::Offset path) { - fbb_.AddOffset(State_FS_Hash_Entry::VT_PATH, path); + void add_name(flatbuffers::Offset name) { + fbb_.AddOffset(State_FS_Hash_Entry::VT_NAME, name); } void add_is_file(bool is_file) { fbb_.AddElement(State_FS_Hash_Entry::VT_IS_FILE, static_cast(is_file), 0); @@ -1682,26 +1682,26 @@ struct State_FS_Hash_EntryBuilder { inline flatbuffers::Offset CreateState_FS_Hash_Entry( flatbuffers::FlatBufferBuilder &_fbb, - flatbuffers::Offset path = 0, + flatbuffers::Offset name = 0, bool is_file = false, flatbuffers::Offset> hash = 0) { State_FS_Hash_EntryBuilder builder_(_fbb); builder_.add_hash(hash); - builder_.add_path(path); + builder_.add_name(name); builder_.add_is_file(is_file); return builder_.Finish(); } inline flatbuffers::Offset CreateState_FS_Hash_EntryDirect( flatbuffers::FlatBufferBuilder &_fbb, - const char *path = nullptr, + const char *name = nullptr, bool is_file = false, const std::vector *hash = nullptr) { - auto path__ = path ? _fbb.CreateString(path) : 0; + auto name__ = name ? _fbb.CreateString(name) : 0; auto hash__ = hash ? _fbb.CreateVector(*hash) : 0; return fbschema::p2pmsg::CreateState_FS_Hash_Entry( _fbb, - path__, + name__, is_file, hash__); } diff --git a/src/fbschema/p2pmsg_helpers.cpp b/src/fbschema/p2pmsg_helpers.cpp index e08f8cbc..a5fc3004 100644 --- a/src/fbschema/p2pmsg_helpers.cpp +++ b/src/fbschema/p2pmsg_helpers.cpp @@ -4,6 +4,8 @@ #include "../util.hpp" #include "../hplog.hpp" #include "../p2p/p2p.hpp" +#include "../hpfs/h32.hpp" +#include "../hpfs/hpfs.hpp" #include "p2pmsg_container_generated.h" #include "p2pmsg_content_generated.h" #include "common_helpers.hpp" @@ -222,7 +224,7 @@ namespace fbschema::p2pmsg p.time = msg.time(); p.stage = msg.stage(); p.lcl = flatbuff_bytes_to_sv(lcl); - p.curr_state_hash = flatbuff_bytes_to_sv(msg.curr_state_hash()); + p.state = flatbuff_bytes_to_sv(msg.state()); if (msg.users()) p.users = flatbuf_bytearrayvector_to_stringlist(msg.users()); @@ -271,21 +273,6 @@ namespace fbschema::p2pmsg return sr; } - /** - * Creates a block response struct from the given block response message. - * @param msg Flatbuffer block response message received from the peer. - * @return A Block response struct representing the message. - */ - const p2p::block_response create_block_response_from_msg(const Block_Response &msg) - { - p2p::block_response br; - - br.path = flatbuff_str_to_sv(msg.path()); - br.block_id = msg.block_id(); - br.data = flatbuff_bytes_to_sv(msg.data()); - return br; - } - //---Message creation helpers---// /** @@ -372,7 +359,7 @@ namespace fbschema::p2pmsg stringlist_to_flatbuf_bytearrayvector(builder, p.users), stringlist_to_flatbuf_bytearrayvector(builder, p.hash_inputs), stringlist_to_flatbuf_bytearrayvector(builder, p.hash_outputs), - sv_to_flatbuff_bytes(builder, p.curr_state_hash.to_string_view())); + sv_to_flatbuff_bytes(builder, p.state.to_string_view())); const flatbuffers::Offset message = CreateContent(builder, Message_Proposal_Message, proposal.Union()); builder.Finish(message); // Finished building message content to get serialised content. @@ -480,11 +467,13 @@ namespace fbschema::p2pmsg * Create content response message from the given content response. * @param container_builder Flatbuffer builder for the container message. * @param path The path of the directory. - * @param fs_entries File or directory entries in the given parent path. + * @param hash_nodes File or directory entries with hashes in the given parent path. * @param expected_hash The exptected hash of the requested path. * @param lcl Lcl to be include in the container msg. */ - void create_msg_from_fsentry_response(flatbuffers::FlatBufferBuilder &container_builder, const std::string_view path, std::unordered_map &fs_entries, hpfs::h32 expected_hash, std::string_view lcl) + void create_msg_from_fsentry_response( + flatbuffers::FlatBufferBuilder &container_builder, const std::string_view path, + std::vector &hash_nodes, hpfs::h32 expected_hash, std::string_view lcl) { flatbuffers::FlatBufferBuilder builder(1024); @@ -492,7 +481,7 @@ namespace fbschema::p2pmsg CreateFs_Entry_Response( builder, sv_to_flatbuff_str(builder, path), - statefshashentry_to_flatbuff_statefshashentry(builder, fs_entries)); + statefshashentry_to_flatbuff_statefshashentry(builder, hash_nodes)); const flatbuffers::Offset st_resp = CreateState_Response_Message( builder, State_Response_Fs_Entry_Response, @@ -514,12 +503,14 @@ namespace fbschema::p2pmsg * @param hashmap Hashmap of the file * @param lcl Lcl to be include in the container msg. */ - void create_msg_from_filehashmap_response(flatbuffers::FlatBufferBuilder &container_builder, std::string_view path, std::vector &hashmap, std::size_t file_length, hpfs::h32 expected_hash, std::string_view lcl) + void create_msg_from_filehashmap_response( + flatbuffers::FlatBufferBuilder &container_builder, std::string_view path, + std::vector &hashmap, std::size_t file_length, hpfs::h32 expected_hash, std::string_view lcl) { // todo:get a average propsal message size and allocate content builder based on that. flatbuffers::FlatBufferBuilder builder(1024); - std::string_view hashmap_sv(reinterpret_cast(hashmap.data()), hashmap.size()); + std::string_view hashmap_sv(reinterpret_cast(hashmap.data()), hashmap.size() * sizeof(hpfs::h32)); const flatbuffers::Offset resp = CreateFile_HashMap_Response( @@ -574,21 +565,6 @@ namespace fbschema::p2pmsg create_containermsg_from_content(container_builder, builder, lcl, true); } - void create_msg_from_state_error_response(flatbuffers::FlatBufferBuilder &container_builder, std::string_view lcl) - { - // todo:get a average propsal message size and allocate content builder based on that. - flatbuffers::FlatBufferBuilder builder(1024); - - const flatbuffers::Offset st_resp = CreateState_Response_Message(builder, State_Response_NONE, 0, true); - - flatbuffers::Offset message = CreateContent(builder, Message_State_Response_Message, st_resp.Union()); - builder.Finish(message); // Finished building message content to get serialised content. - - // Now that we have built the content message, - // we need to sign it and place it inside a container message. - create_containermsg_from_content(container_builder, builder, lcl, true); - } - /** * Creates a Flatbuffer container message from the given Content message. * @param container_builder The Flatbuffer builder to which the final container message should be written to. @@ -728,29 +704,31 @@ namespace fbschema::p2pmsg void flatbuf_statefshashentry_to_statefshashentry(std::unordered_map &fs_entries, const flatbuffers::Vector> *fhashes) { - for (const State_FS_Hash_Entry *f_hash : *fhashes) { - p2p::state_fs_hash_entry h; - - h.is_file = f_hash->is_file(); - h.hash = flatbuff_bytes_to_hash(f_hash->hash()); - fs_entries.emplace(flatbuff_str_to_sv(f_hash->path()), std::move(h)); + p2p::state_fs_hash_entry entry; + entry.name = flatbuff_str_to_sv(f_hash->name()); + entry.is_file = f_hash->is_file(); + entry.hash = flatbuff_bytes_to_hash(f_hash->hash()); + + fs_entries.emplace(entry.name, std::move(entry)); } } flatbuffers::Offset>> - statefshashentry_to_flatbuff_statefshashentry(flatbuffers::FlatBufferBuilder &builder, std::unordered_map &fs_entries) + statefshashentry_to_flatbuff_statefshashentry( + flatbuffers::FlatBufferBuilder &builder, + std::vector &hash_nodes) { std::vector> fbvec; - fbvec.reserve(fs_entries.size()); - for (auto const &[path, fs_entry] : fs_entries) + fbvec.reserve(hash_nodes.size()); + for (auto const &hash_node : hash_nodes) { flatbuffers::Offset state_fs_entry = CreateState_FS_Hash_Entry( builder, - sv_to_flatbuff_str(builder, path), - fs_entry.is_file, - hash_to_flatbuff_bytes(builder, fs_entry.hash)); + sv_to_flatbuff_str(builder, hash_node.name), + hash_node.is_file, + hash_to_flatbuff_bytes(builder, hash_node.hash)); fbvec.push_back(state_fs_entry); } diff --git a/src/fbschema/p2pmsg_helpers.hpp b/src/fbschema/p2pmsg_helpers.hpp index 9e789418..d2b9ffd4 100644 --- a/src/fbschema/p2pmsg_helpers.hpp +++ b/src/fbschema/p2pmsg_helpers.hpp @@ -6,90 +6,94 @@ #include "p2pmsg_content_generated.h" #include "../p2p/p2p.hpp" #include "../hpfs/h32.hpp" +#include "../hpfs/hpfs.hpp" namespace fbschema::p2pmsg { -/** + /** * This section contains Flatbuffer p2p message reading/writing helpers. */ -//---Message validation helpers---/ + //---Message validation helpers---/ -int validate_and_extract_container(const Container **container_ref, std::string_view container_buf); + int validate_and_extract_container(const Container **container_ref, std::string_view container_buf); -int validate_container_trust(const Container *container); + int validate_container_trust(const Container *container); -int validate_and_extract_content(const Content **content_ref, const uint8_t *content_ptr, const flatbuffers::uoffset_t content_size); + int validate_and_extract_content(const Content **content_ref, const uint8_t *content_ptr, const flatbuffers::uoffset_t content_size); -//---Message reading helpers---/ + //---Message reading helpers---/ -const std::string_view get_peer_challenge_from_msg(const Peer_Challenge_Message &msg); + const std::string_view get_peer_challenge_from_msg(const Peer_Challenge_Message &msg); -const p2p::peer_challenge_response create_peer_challenge_response_from_msg(const Peer_Challenge_Response_Message &msg, const flatbuffers::Vector *pubkey); + const p2p::peer_challenge_response create_peer_challenge_response_from_msg(const Peer_Challenge_Response_Message &msg, const flatbuffers::Vector *pubkey); -const p2p::nonunl_proposal create_nonunl_proposal_from_msg(const NonUnl_Proposal_Message &msg, const uint64_t timestamp); + const p2p::nonunl_proposal create_nonunl_proposal_from_msg(const NonUnl_Proposal_Message &msg, const uint64_t timestamp); -const p2p::proposal create_proposal_from_msg(const Proposal_Message &msg, const flatbuffers::Vector *pubkey, const uint64_t timestamp, const flatbuffers::Vector *lcl); + const p2p::proposal create_proposal_from_msg(const Proposal_Message &msg, const flatbuffers::Vector *pubkey, const uint64_t timestamp, const flatbuffers::Vector *lcl); -const p2p::history_request create_history_request_from_msg(const History_Request_Message &msg); + const p2p::history_request create_history_request_from_msg(const History_Request_Message &msg); -const p2p::history_response create_history_response_from_msg(const History_Response_Message &msg); + const p2p::history_response create_history_response_from_msg(const History_Response_Message &msg); -const p2p::state_request create_state_request_from_msg(const State_Request_Message &msg); + const p2p::state_request create_state_request_from_msg(const State_Request_Message &msg); -const p2p::block_response create_block_response_from_msg(const Block_Response &msg); + //---Message creation helpers---// + void create_peer_challenge_response_from_challenge(flatbuffers::FlatBufferBuilder &container_builder, const std::string &challenge); -//---Message creation helpers---// -void create_peer_challenge_response_from_challenge(flatbuffers::FlatBufferBuilder &container_builder, const std::string &challenge); + void create_msg_from_peer_challenge(flatbuffers::FlatBufferBuilder &container_builder, std::string &challenge); -void create_msg_from_peer_challenge(flatbuffers::FlatBufferBuilder &container_builder, std::string &challenge); + void create_msg_from_nonunl_proposal(flatbuffers::FlatBufferBuilder &container_builder, const p2p::nonunl_proposal &nup); -void create_msg_from_nonunl_proposal(flatbuffers::FlatBufferBuilder &container_builder, const p2p::nonunl_proposal &nup); + void create_msg_from_proposal(flatbuffers::FlatBufferBuilder &container_builder, const p2p::proposal &p); -void create_msg_from_proposal(flatbuffers::FlatBufferBuilder &container_builder, const p2p::proposal &p); + void create_msg_from_history_request(flatbuffers::FlatBufferBuilder &container_builder, const p2p::history_request &hr); -void create_msg_from_history_request(flatbuffers::FlatBufferBuilder &container_builder, const p2p::history_request &hr); + void create_msg_from_history_response(flatbuffers::FlatBufferBuilder &container_builder, const p2p::history_response &hr); -void create_msg_from_history_response(flatbuffers::FlatBufferBuilder &container_builder, const p2p::history_response &hr); + void create_msg_from_npl_output(flatbuffers::FlatBufferBuilder &container_builder, const p2p::npl_message &npl, std::string_view lcl); -void create_msg_from_npl_output(flatbuffers::FlatBufferBuilder &container_builder, const p2p::npl_message &npl, std::string_view lcl); + void create_msg_from_state_request(flatbuffers::FlatBufferBuilder &container_builder, const p2p::state_request &hr, std::string_view lcl); -void create_msg_from_state_request(flatbuffers::FlatBufferBuilder &container_builder, const p2p::state_request &hr, std::string_view lcl); + void create_msg_from_fsentry_response( + flatbuffers::FlatBufferBuilder &container_builder, const std::string_view path, + std::vector &hash_nodes, hpfs::h32 expected_hash, std::string_view lcl); -void create_msg_from_fsentry_response(flatbuffers::FlatBufferBuilder &container_builder, const std::string_view path, - std::unordered_map &fs_entries, hpfs::h32 expected_hash, std::string_view lcl); + void create_msg_from_filehashmap_response( + flatbuffers::FlatBufferBuilder &container_builder, std::string_view path, + std::vector &hashmap, std::size_t file_length, hpfs::h32 expected_hash, std::string_view lcl); -void create_msg_from_filehashmap_response(flatbuffers::FlatBufferBuilder &container_builder, std::string_view path, std::vector &hashmap, std::size_t file_length, hpfs::h32 expected_hash, std::string_view lcl); + void create_msg_from_block_response(flatbuffers::FlatBufferBuilder &container_builder, p2p::block_response &block_resp, std::string_view lcl); -void create_msg_from_block_response(flatbuffers::FlatBufferBuilder &container_builder, p2p::block_response &block_resp, std::string_view lcl); + void create_containermsg_from_content( + flatbuffers::FlatBufferBuilder &container_builder, const flatbuffers::FlatBufferBuilder &content_builder, std::string_view lcl, const bool sign); -void create_containermsg_from_content( - flatbuffers::FlatBufferBuilder &container_builder, const flatbuffers::FlatBufferBuilder &content_builder, std::string_view lcl, const bool sign); + //---Conversion helpers from flatbuffers data types to std data types---// -//---Conversion helpers from flatbuffers data types to std data types---// + const std::unordered_map> + flatbuf_usermsgsmap_to_usermsgsmap(const flatbuffers::Vector> *fbvec); -const std::unordered_map> -flatbuf_usermsgsmap_to_usermsgsmap(const flatbuffers::Vector> *fbvec); + //---Conversion helpers from std data types to flatbuffers data types---// -//---Conversion helpers from std data types to flatbuffers data types---// + const flatbuffers::Offset>> + usermsgsmap_to_flatbuf_usermsgsmap(flatbuffers::FlatBufferBuilder &builder, const std::unordered_map> &map); -const flatbuffers::Offset>> -usermsgsmap_to_flatbuf_usermsgsmap(flatbuffers::FlatBufferBuilder &builder, const std::unordered_map> &map); + const std::map + flatbuf_historyledgermap_to_historyledgermap(const flatbuffers::Vector> *fbvec); -const std::map -flatbuf_historyledgermap_to_historyledgermap(const flatbuffers::Vector> *fbvec); + const flatbuffers::Offset>> + historyledgermap_to_flatbuf_historyledgermap(flatbuffers::FlatBufferBuilder &builder, const std::map &map); -const flatbuffers::Offset>> -historyledgermap_to_flatbuf_historyledgermap(flatbuffers::FlatBufferBuilder &builder, const std::map &map); + void flatbuf_statefshashentry_to_statefshashentry(std::unordered_map &fs_entries, + const flatbuffers::Vector> *fhashes); -void flatbuf_statefshashentry_to_statefshashentry(std::unordered_map &fs_entries, - const flatbuffers::Vector> *fhashes); + void statefilehash_to_flatbuf_statefilehash(flatbuffers::FlatBufferBuilder &builder, std::vector> &list, + std::string_view full_path, bool is_file, std::string_view hash); -void statefilehash_to_flatbuf_statefilehash(flatbuffers::FlatBufferBuilder &builder, std::vector> &list, - std::string_view full_path, bool is_file, std::string_view hash); - -flatbuffers::Offset>> -statefshashentry_to_flatbuff_statefshashentry(flatbuffers::FlatBufferBuilder &builder, std::unordered_map &fs_entries); + flatbuffers::Offset>> + statefshashentry_to_flatbuff_statefshashentry( + flatbuffers::FlatBufferBuilder &builder, + std::vector &hash_nodes); } // namespace fbschema::p2pmsg diff --git a/src/hpfs/hpfs.cpp b/src/hpfs/hpfs.cpp index 20752804..63ba560e 100644 --- a/src/hpfs/hpfs.cpp +++ b/src/hpfs/hpfs.cpp @@ -48,6 +48,8 @@ namespace hpfs else if (pid == 0) { // hpfs process. + util::unmask_signal(); + // Fill process args. char *execv_args[] = { conf::ctx.hpfs_exe_path.data(), @@ -77,12 +79,17 @@ namespace hpfs { // HotPocket process. - // If the mound dir is not specified, assign a mount dir based on hpfs process id. + // If the mount dir is not specified, assign a mount dir based on hpfs process id. if (mount_dir.empty()) mount_dir = std::string(conf::ctx.state_dir) .append("/") .append(std::to_string(pid)); + // The path used for checking whether hpfs has finished initializing. + const std::string check_path = hash_map_enabled + ? std::string(mount_dir).append("/::hpfs.hmap.hash") + : mount_dir; + // Wait until hpfs is initialized properly. bool hpfs_initialized = false; uint8_t retry_count = 0; @@ -94,9 +101,11 @@ namespace hpfs if (kill(pid, 0) == -1) break; - // If hpfs is initialized, the inode no. of the mounted root dir is always 1. + // If hash map is enabled we check whether stat succeeds on the root hash. + // If not, we check whether the inode no. of the mounted root dir is 1. struct stat st; - hpfs_initialized = (stat(mount_dir.c_str(), &st) == 0 && st.st_ino == 1); + hpfs_initialized = (stat(check_path.c_str(), &st) == 0 && + (hash_map_enabled || st.st_ino == 1)); } while (!hpfs_initialized && ++retry_count < 100); @@ -113,8 +122,9 @@ namespace hpfs else if (pid == 0) { // hpfs process. + util::unmask_signal(); - // If the mound dir is not specified, assign a mount dir based on hpfs process id. + // If the mount dir is not specified, assign a mount dir based on hpfs process id. const pid_t self_pid = getpid(); if (mount_dir.empty()) mount_dir = std::string(conf::ctx.state_dir) @@ -127,7 +137,7 @@ namespace hpfs (char *)mode, // hpfs mode: rw | ro conf::ctx.state_dir.data(), mount_dir.data(), - (char *)(hash_map_enabled ? "hmap=true" : "hmap-false"), + (char *)(hash_map_enabled ? "hmap=true" : "hmap=false"), NULL}; const int ret = execv(execv_args[0], execv_args); @@ -143,19 +153,6 @@ namespace hpfs return 0; } - int get_root_hash(h32 &hash) - { - pid_t pid; - std::string mount_dir; - if (start_fs_session(pid, mount_dir, "ro", true) == -1) - return -1; - - int res = get_hash(hash, mount_dir, "/"); - util::kill_process(pid, true); - - return res; - } - int get_hash(h32 &hash, const std::string_view mount_dir, const std::string_view vpath) { std::string path = std::string(mount_dir).append(vpath).append("::hpfs.hmap.hash"); @@ -175,4 +172,63 @@ namespace hpfs return 0; } + int get_file_block_hashes(std::vector &hashes, const std::string_view mount_dir, const std::string_view vpath) + { + std::string path = std::string(mount_dir).append(vpath).append("::hpfs.hmap.children"); + int fd = open(path.c_str(), O_RDONLY); + if (fd == -1) + return -1; + + struct stat st; + if (fstat(fd, &st) == -1) + { + close(fd); + LOG_ERR << errno << ": Error reading block hashes length."; + return -1; + } + + const int children_count = st.st_size / sizeof(h32); + hashes.resize(children_count); + + int res = read(fd, hashes.data(), st.st_size); + close(fd); + if (res == -1) + { + LOG_ERR << errno << ": Error reading hash block hashes."; + return -1; + } + return 0; + } + + int get_dir_children_hashes(std::vector &hash_nodes, const std::string_view mount_dir, const std::string_view dir_vpath) + { + std::string path = std::string(mount_dir).append(dir_vpath).append("::hpfs.hmap.children"); + int fd = open(path.c_str(), O_RDONLY); + if (fd == -1) + { + LOG_ERR << errno << ": Error opening hash children nodes."; + return -1; + } + + struct stat st; + if (fstat(fd, &st) == -1) + { + close(fd); + LOG_ERR << errno << ": Error reading hash children nodes length."; + return -1; + } + + const int children_count = st.st_size / sizeof(child_hash_node); + hash_nodes.resize(children_count); + + int res = read(fd, hash_nodes.data(), st.st_size); + close(fd); + if (res == -1) + { + LOG_ERR << errno << ": Error reading hash children nodes."; + return -1; + } + return 0; + } + } // namespace hpfs \ No newline at end of file diff --git a/src/hpfs/hpfs.hpp b/src/hpfs/hpfs.hpp index 41361b84..780624b1 100644 --- a/src/hpfs/hpfs.hpp +++ b/src/hpfs/hpfs.hpp @@ -6,13 +6,21 @@ namespace hpfs { + struct child_hash_node + { + bool is_file; + char name[256]; + h32 hash; + }; + int init(); void deinit(); int start_merge_process(); int start_fs_session(pid_t &session_pid, std::string &mount_dir, const char *mode, const bool hash_map_enabled); - int get_root_hash(h32 &hash); int get_hash(h32 &hash, const std::string_view mount_dir, const std::string_view vpath); + int get_file_block_hashes(std::vector &hashes, const std::string_view mount_dir, const std::string_view vpath); + int get_dir_children_hashes(std::vector &hash_nodes, const std::string_view mount_dir, const std::string_view dir_vpath); } // namespace hpfs #endif \ No newline at end of file diff --git a/src/main.cpp b/src/main.cpp index 83140166..396c82c1 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -12,6 +12,7 @@ #include "p2p/p2p.hpp" #include "cons/cons.hpp" #include "hpfs/hpfs.hpp" +#include "state/state_sync.hpp" /** * Parses CLI args and extracts hot pocket command and parameters given. @@ -64,10 +65,11 @@ int parse_cmd(int argc, char **argv) */ void deinit() { - usr::deinit(); - p2p::deinit(); cons::deinit(); sc::deinit(); + state_sync::deinit(); + usr::deinit(); + p2p::deinit(); hpfs::deinit(); hplog::deinit(); } @@ -188,7 +190,8 @@ int main(int argc, char **argv) LOG_INFO << "Operating mode: " << (conf::cfg.startup_mode == conf::OPERATING_MODE::OBSERVER ? "Observer" : "Proposer"); - if (hpfs::init() != 0 || p2p::init() != 0 || usr::init() != 0 || cons::init() != 0) + if (hpfs::init() != 0 || p2p::init() != 0 || usr::init() != 0 || + state_sync::init() != 0 || cons::init() != 0) { deinit(); return -1; diff --git a/src/p2p/p2p.cpp b/src/p2p/p2p.cpp index e11af096..3787ff31 100644 --- a/src/p2p/p2p.cpp +++ b/src/p2p/p2p.cpp @@ -89,6 +89,7 @@ namespace p2p if (iter == p2p::ctx.peer_connections.end()) { // Add the new connection straight away, if we haven't seen it before. + session.is_self = (res == 0); session.uniqueid.swap(pubkeyhex); session.challenge_status = comm::CHALLENGE_VERIFIED; p2p::ctx.peer_connections.try_emplace(session.uniqueid, &session); @@ -198,12 +199,12 @@ namespace p2p const size_t connected_peers = ctx.peer_connections.size(); if (connected_peers == 0) { - LOG_DBG << "No peers to send (not even self)."; + LOG_DBG << "No peers to random send."; return; } else if (connected_peers == 1 && ctx.peer_connections.begin()->second->is_self) { - LOG_DBG << "Only self is connected."; + LOG_DBG << "Only self is connected. Cannot random send."; return; } diff --git a/src/p2p/p2p.hpp b/src/p2p/p2p.hpp index 81ba3f09..48b31bef 100644 --- a/src/p2p/p2p.hpp +++ b/src/p2p/p2p.hpp @@ -20,7 +20,7 @@ struct proposal uint64_t time; uint8_t stage; std::string lcl; - hpfs::h32 curr_state_hash; + hpfs::h32 state; std::set users; std::set hash_inputs; std::set hash_outputs; @@ -81,8 +81,9 @@ struct state_request // Represents state file system entry. struct state_fs_hash_entry { + std::string name; // Name of the file/dir. bool is_file; // Whether this is a file or dir. - hpfs::h32 hash; // Hash of the file or dir. + hpfs::h32 hash; // Hash of the file or dir. }; // Represents a file block data resposne. diff --git a/src/p2p/peer_session_handler.cpp b/src/p2p/peer_session_handler.cpp index 8fe9f4f9..c3def4d2 100644 --- a/src/p2p/peer_session_handler.cpp +++ b/src/p2p/peer_session_handler.cpp @@ -12,7 +12,8 @@ #include "p2p.hpp" #include "peer_session_handler.hpp" #include "../cons/ledger_handler.hpp" -#include "../cons/state_handler.hpp" +#include "../state/state_sync.hpp" +#include "../state/state_serve.hpp" #include "../cons/cons.hpp" namespace p2pmsg = fbschema::p2pmsg; @@ -20,191 +21,195 @@ namespace p2pmsg = fbschema::p2pmsg; namespace p2p { -// The set of recent peer message hashes used for duplicate detection. -util::rollover_hashset recent_peermsg_hashes(200); + // The set of recent peer message hashes used for duplicate detection. + util::rollover_hashset recent_peermsg_hashes(200); -/** + /** * This gets hit every time a peer connects to HP via the peer port (configured in contract config). */ -int peer_session_handler::on_connect(comm::comm_session &session) const -{ - if (session.is_inbound) + int peer_session_handler::on_connect(comm::comm_session &session) const { - // Limit max number of inbound connections. - if (conf::cfg.peermaxcons > 0 && ctx.peer_connections.size() >= conf::cfg.peermaxcons) + if (session.is_inbound) { - LOG_DBG << "Max peer connections reached. Dropped connection " << session.uniqueid; - return -1; + // Limit max number of inbound connections. + if (conf::cfg.peermaxcons > 0 && ctx.peer_connections.size() >= conf::cfg.peermaxcons) + { + LOG_DBG << "Max peer connections reached. Dropped connection " << session.uniqueid; + return -1; + } } - } - // Send peer challenge. - flatbuffers::FlatBufferBuilder fbuf(1024); - p2pmsg::create_msg_from_peer_challenge(fbuf, session.issued_challenge); - std::string_view msg = std::string_view( - reinterpret_cast(fbuf.GetBufferPointer()), fbuf.GetSize()); - session.send(msg); - session.challenge_status = comm::CHALLENGE_ISSUED; - return 0; -} - -//peer session on message callback method -//validate and handle each type of peer messages. -int peer_session_handler::on_message(comm::comm_session &session, std::string_view message) const -{ - const p2pmsg::Container *container; - if (p2pmsg::validate_and_extract_container(&container, message) != 0) - return 0; - - //Get serialised message content. - const flatbuffers::Vector *container_content = container->content(); - - //Accessing message content and size. - const uint8_t *content_ptr = container_content->Data(); - const flatbuffers::uoffset_t content_size = container_content->size(); - - const p2pmsg::Content *content; - if (p2pmsg::validate_and_extract_content(&content, content_ptr, content_size) != 0) - return 0; - - if (!recent_peermsg_hashes.try_emplace(crypto::get_hash(message))) - { - session.increment_metric(comm::SESSION_THRESHOLDS::MAX_DUPMSGS_PER_MINUTE, 1); - LOG_DBG << "Duplicate peer message. " << session.uniqueid; - return 0; - } - - const p2pmsg::Message content_message_type = content->message_type(); //i.e - proposal, npl, state request, state response, etc - - if (content_message_type == p2pmsg::Message_Peer_Challenge_Message) // message is a peer challenge announcement - { - // Sending the challenge response to the respected peer. - const std::string challenge = std::string(p2pmsg::get_peer_challenge_from_msg(*content->message_as_Peer_Challenge_Message())); + // Send peer challenge. flatbuffers::FlatBufferBuilder fbuf(1024); - p2pmsg::create_peer_challenge_response_from_challenge(fbuf, challenge); + p2pmsg::create_msg_from_peer_challenge(fbuf, session.issued_challenge); std::string_view msg = std::string_view( reinterpret_cast(fbuf.GetBufferPointer()), fbuf.GetSize()); - return session.send(msg); - } - - if (content_message_type == p2pmsg::Message_Peer_Challenge_Response_Message) // message is a peer challenge response - { - // Ignore if challenge is already resolved. - if (session.challenge_status == comm::CHALLENGE_ISSUED) - { - const p2p::peer_challenge_response challenge_resp = p2pmsg::create_peer_challenge_response_from_msg(*content->message_as_Peer_Challenge_Response_Message(), container->pubkey()); - return p2p::resolve_peer_challenge(session, challenge_resp); - } - } - - if (session.challenge_status != comm::CHALLENGE_VERIFIED) - { - LOG_DBG << "Cannot accept messages. Peer challenge unresolved. " << session.uniqueid; + session.send(msg); + session.challenge_status = comm::CHALLENGE_ISSUED; return 0; } - if (content_message_type == p2pmsg::Message_Proposal_Message) // message is a proposal message + //peer session on message callback method + //validate and handle each type of peer messages. + int peer_session_handler::on_message(comm::comm_session &session, std::string_view message) const { - // We only trust proposals coming from trusted peers. - if (p2pmsg::validate_container_trust(container) != 0) + const p2pmsg::Container *container; + if (p2pmsg::validate_and_extract_container(&container, message) != 0) + return 0; + + //Get serialised message content. + const flatbuffers::Vector *container_content = container->content(); + + //Accessing message content and size. + const uint8_t *content_ptr = container_content->Data(); + const flatbuffers::uoffset_t content_size = container_content->size(); + + const p2pmsg::Content *content; + if (p2pmsg::validate_and_extract_content(&content, content_ptr, content_size) != 0) + return 0; + + if (!recent_peermsg_hashes.try_emplace(crypto::get_hash(message))) { - session.increment_metric(comm::SESSION_THRESHOLDS::MAX_BADSIGMSGS_PER_MINUTE, 1); - LOG_DBG << "Proposal rejected due to trust failure. " << session.uniqueid; + session.increment_metric(comm::SESSION_THRESHOLDS::MAX_DUPMSGS_PER_MINUTE, 1); + LOG_DBG << "Duplicate peer message. " << session.uniqueid; return 0; } - std::lock_guard lock(ctx.collected_msgs.proposals_mutex); // Insert proposal with lock. + const p2pmsg::Message content_message_type = content->message_type(); //i.e - proposal, npl, state request, state response, etc - ctx.collected_msgs.proposals.push_back( - p2pmsg::create_proposal_from_msg(*content->message_as_Proposal_Message(), container->pubkey(), container->timestamp(), container->lcl())); - } - else if (content_message_type == p2pmsg::Message_NonUnl_Proposal_Message) //message is a non-unl proposal message - { - std::lock_guard lock(ctx.collected_msgs.nonunl_proposals_mutex); // Insert non-unl proposal with lock. - - ctx.collected_msgs.nonunl_proposals.push_back( - p2pmsg::create_nonunl_proposal_from_msg(*content->message_as_NonUnl_Proposal_Message(), container->timestamp())); - } - else if (content_message_type == p2pmsg::Message_Npl_Message) //message is a NPL message - { - if (p2pmsg::validate_container_trust(container) != 0) - { - LOG_DBG << "NPL message rejected due to trust failure. " << session.uniqueid; - return 0; - } - - std::lock_guard lock(ctx.collected_msgs.npl_messages_mutex); // Insert npl message with lock. - - // Npl messages are added to the npl message array as it is without deserealizing the content. The same content will be passed down - // to the contract as input in a binary format - const uint8_t *container_buf_ptr = reinterpret_cast(message.data()); - const size_t container_buf_size = message.length(); - const std::string npl_message(reinterpret_cast(container_buf_ptr), container_buf_size); - ctx.collected_msgs.npl_messages.push_back(std::move(npl_message)); - } - else if (content_message_type == p2pmsg::Message_State_Request_Message) - { - const p2p::state_request sr = p2pmsg::create_state_request_from_msg(*content->message_as_State_Request_Message()); - flatbuffers::FlatBufferBuilder fbuf(1024); - - if (cons::create_state_response(fbuf, sr) == 0) - { - std::string_view msg = std::string_view( - reinterpret_cast(fbuf.GetBufferPointer()), fbuf.GetSize()); - session.send(msg); - } - } - else if (content_message_type == p2pmsg::Message_State_Response_Message) - { - if (p2pmsg::validate_container_trust(container) != 0) - { - LOG_DBG << "State response message rejected due to trust failure. " << session.uniqueid; - return 0; - } - - std::lock_guard lock(ctx.collected_msgs.state_response_mutex); // Insert state_response with lock. - std::string response(reinterpret_cast(content_ptr), content_size); - ctx.collected_msgs.state_response.push_back(std::move(response)); - } - else if (content_message_type == p2pmsg::Message_History_Request_Message) //message is a lcl history request message - { - const p2p::history_request hr = p2pmsg::create_history_request_from_msg(*content->message_as_History_Request_Message()); - //first check node has the required lcl available. -> if so send lcl history accordingly. - const bool req_lcl_avail = cons::check_required_lcl_availability(hr); - if (req_lcl_avail) + if (content_message_type == p2pmsg::Message_Peer_Challenge_Message) // message is a peer challenge announcement { + // Sending the challenge response to the respected peer. + const std::string challenge = std::string(p2pmsg::get_peer_challenge_from_msg(*content->message_as_Peer_Challenge_Message())); flatbuffers::FlatBufferBuilder fbuf(1024); - p2pmsg::create_msg_from_history_response(fbuf, cons::retrieve_ledger_history(hr)); + p2pmsg::create_peer_challenge_response_from_challenge(fbuf, challenge); std::string_view msg = std::string_view( reinterpret_cast(fbuf.GetBufferPointer()), fbuf.GetSize()); - - session.send(msg); + return session.send(msg); } - } - else if (content_message_type == p2pmsg::Message_History_Response_Message) //message is a lcl history response message - { - if (p2pmsg::validate_container_trust(container) != 0) + + if (content_message_type == p2pmsg::Message_Peer_Challenge_Response_Message) // message is a peer challenge response { - LOG_DBG << "History response message rejected due to trust failure. " << session.uniqueid; + // Ignore if challenge is already resolved. + if (session.challenge_status == comm::CHALLENGE_ISSUED) + { + const p2p::peer_challenge_response challenge_resp = p2pmsg::create_peer_challenge_response_from_msg(*content->message_as_Peer_Challenge_Response_Message(), container->pubkey()); + return p2p::resolve_peer_challenge(session, challenge_resp); + } + } + + if (session.challenge_status != comm::CHALLENGE_VERIFIED) + { + LOG_DBG << "Cannot accept messages. Peer challenge unresolved. " << session.uniqueid; return 0; } - cons::handle_ledger_history_response( - p2pmsg::create_history_response_from_msg(*content->message_as_History_Response_Message())); - } - else - { - session.increment_metric(comm::SESSION_THRESHOLDS::MAX_BADMSGS_PER_MINUTE, 1); - LOG_DBG << "Received invalid peer message type. " << session.uniqueid; - } - return 0; -} + if (content_message_type == p2pmsg::Message_Proposal_Message) // message is a proposal message + { + // We only trust proposals coming from trusted peers. + if (p2pmsg::validate_container_trust(container) != 0) + { + session.increment_metric(comm::SESSION_THRESHOLDS::MAX_BADSIGMSGS_PER_MINUTE, 1); + LOG_DBG << "Proposal rejected due to trust failure. " << session.uniqueid; + return 0; + } -//peer session on message callback method -void peer_session_handler::on_close(const comm::comm_session &session) const -{ - std::lock_guard lock(ctx.peer_connections_mutex); - ctx.peer_connections.erase(session.uniqueid); -} + std::lock_guard lock(ctx.collected_msgs.proposals_mutex); // Insert proposal with lock. + + ctx.collected_msgs.proposals.push_back( + p2pmsg::create_proposal_from_msg(*content->message_as_Proposal_Message(), container->pubkey(), container->timestamp(), container->lcl())); + } + else if (content_message_type == p2pmsg::Message_NonUnl_Proposal_Message) //message is a non-unl proposal message + { + std::lock_guard lock(ctx.collected_msgs.nonunl_proposals_mutex); // Insert non-unl proposal with lock. + + ctx.collected_msgs.nonunl_proposals.push_back( + p2pmsg::create_nonunl_proposal_from_msg(*content->message_as_NonUnl_Proposal_Message(), container->timestamp())); + } + else if (content_message_type == p2pmsg::Message_Npl_Message) //message is a NPL message + { + if (p2pmsg::validate_container_trust(container) != 0) + { + LOG_DBG << "NPL message rejected due to trust failure. " << session.uniqueid; + return 0; + } + + std::lock_guard lock(ctx.collected_msgs.npl_messages_mutex); // Insert npl message with lock. + + // Npl messages are added to the npl message array as it is without deserealizing the content. The same content will be passed down + // to the contract as input in a binary format + const uint8_t *container_buf_ptr = reinterpret_cast(message.data()); + const size_t container_buf_size = message.length(); + const std::string npl_message(reinterpret_cast(container_buf_ptr), container_buf_size); + ctx.collected_msgs.npl_messages.push_back(std::move(npl_message)); + } + else if (content_message_type == p2pmsg::Message_State_Request_Message) + { + const p2p::state_request sr = p2pmsg::create_state_request_from_msg(*content->message_as_State_Request_Message()); + flatbuffers::FlatBufferBuilder fbuf(1024); + + if (state_serve::create_state_response(fbuf, sr) == 0) + { + std::string_view msg = std::string_view( + reinterpret_cast(fbuf.GetBufferPointer()), fbuf.GetSize()); + session.send(msg); + } + } + else if (content_message_type == p2pmsg::Message_State_Response_Message) + { + if (p2pmsg::validate_container_trust(container) != 0) + { + LOG_DBG << "State response message rejected due to trust failure. " << session.uniqueid; + return 0; + } + + if (state_sync::ctx.is_syncing) // Only accept state responses if state is syncing. + { + // Insert state_response with lock. + std::lock_guard lock(ctx.collected_msgs.state_response_mutex); + std::string response(reinterpret_cast(content_ptr), content_size); + ctx.collected_msgs.state_response.push_back(std::move(response)); + } + } + else if (content_message_type == p2pmsg::Message_History_Request_Message) //message is a lcl history request message + { + const p2p::history_request hr = p2pmsg::create_history_request_from_msg(*content->message_as_History_Request_Message()); + //first check node has the required lcl available. -> if so send lcl history accordingly. + const bool req_lcl_avail = cons::check_required_lcl_availability(hr); + if (req_lcl_avail) + { + flatbuffers::FlatBufferBuilder fbuf(1024); + p2pmsg::create_msg_from_history_response(fbuf, cons::retrieve_ledger_history(hr)); + std::string_view msg = std::string_view( + reinterpret_cast(fbuf.GetBufferPointer()), fbuf.GetSize()); + + session.send(msg); + } + } + else if (content_message_type == p2pmsg::Message_History_Response_Message) //message is a lcl history response message + { + if (p2pmsg::validate_container_trust(container) != 0) + { + LOG_DBG << "History response message rejected due to trust failure. " << session.uniqueid; + return 0; + } + + cons::handle_ledger_history_response( + p2pmsg::create_history_response_from_msg(*content->message_as_History_Response_Message())); + } + else + { + session.increment_metric(comm::SESSION_THRESHOLDS::MAX_BADMSGS_PER_MINUTE, 1); + LOG_DBG << "Received invalid peer message type. " << session.uniqueid; + } + return 0; + } + + //peer session on message callback method + void peer_session_handler::on_close(const comm::comm_session &session) const + { + std::lock_guard lock(ctx.peer_connections_mutex); + ctx.peer_connections.erase(session.uniqueid); + } } // namespace p2p \ No newline at end of file diff --git a/src/pchheader.hpp b/src/pchheader.hpp index 8926a262..72be1a20 100644 --- a/src/pchheader.hpp +++ b/src/pchheader.hpp @@ -30,6 +30,7 @@ #include #include #include +#include #include #include #include diff --git a/src/sc.cpp b/src/sc.cpp index e7b9e985..b9dd01c9 100644 --- a/src/sc.cpp +++ b/src/sc.cpp @@ -56,13 +56,12 @@ namespace sc LOG_ERR << "Contract process exited with non-normal status code: " << presult; goto failure; } - - if (stop_hpfs_rw_session(state_hash) != 0) - goto failure; } else if (pid == 0) { // Contract process. + util::unmask_signal(); + // Set up the process environment and overlay the contract binary program with execv(). // Close all fds unused by SC process. @@ -106,6 +105,7 @@ namespace sc ret = -1; success: + stop_hpfs_rw_session(state_hash); cleanup_fdmap(ctx.userfds); cleanup_vectorfds(ctx.hpscfds); cleanup_vectorfds(ctx.nplfds); @@ -254,6 +254,8 @@ namespace sc int fetch_outputs(const contract_exec_args &args) { + util::mask_signal(); + while (true) { if (ctx.should_deinit) diff --git a/src/sc.hpp b/src/sc.hpp index 2c1e1b79..73601c0c 100644 --- a/src/sc.hpp +++ b/src/sc.hpp @@ -1,5 +1,5 @@ -#ifndef _HP_PROC_ -#define _HP_PROC_ +#ifndef _HP_SC_ +#define _HP_SC_ #include "pchheader.hpp" #include "usr/usr.hpp" diff --git a/src/state/state_serve.cpp b/src/state/state_serve.cpp new file mode 100644 index 00000000..bcad23c6 --- /dev/null +++ b/src/state/state_serve.cpp @@ -0,0 +1,252 @@ +#include "../pchheader.hpp" +#include "../hpfs/hpfs.hpp" +#include "../hpfs/h32.hpp" +#include "../util.hpp" +#include "../p2p/p2p.hpp" +#include "../fbschema/p2pmsg_content_generated.h" +#include "../fbschema/p2pmsg_helpers.hpp" +#include "../fbschema/common_helpers.hpp" +#include "../cons/cons.hpp" +#include "../hplog.hpp" +#include "state_serve.hpp" + +/** + * Helper functions for serving state requests from other peers. + */ +namespace state_serve +{ + constexpr size_t BLOCK_SIZE = 4 * 1024 * 1024; // 4MB; + + /** + * Creates the reply message for a given state request. + * @param fbuf The flatbuffer builder to construct the reply message. + * @param sr The state request which should be replied to. + */ + int create_state_response(flatbuffers::FlatBufferBuilder &fbuf, const p2p::state_request &sr) + { + LOG_DBG << "Serving state req. path:" << sr.parent_path << " block_id:" << sr.block_id; + + // If block_id > -1 this means this is a file block data request. + if (sr.block_id > -1) + { + // Vector to hold the block bytes. Normally block size is constant BLOCK_SIZE (4MB), but the + // last block of a file may have a smaller size. + std::vector block; + if (get_file_block(block, sr.parent_path, sr.block_id, sr.expected_hash) == -1) + { + LOG_ERR << "Error in getting file block."; + return -1; + } + + p2p::block_response resp; + resp.path = sr.parent_path; + resp.block_id = sr.block_id; + resp.hash = sr.expected_hash; + resp.data = std::string_view(reinterpret_cast(block.data()), block.size()); + + fbschema::p2pmsg::create_msg_from_block_response(fbuf, resp, cons::ctx.lcl); + } + else + { + // File state request means we have to reply with the file block hash map. + if (sr.is_file) + { + std::vector block_hashes; + std::size_t file_length = 0; + if (get_file_block_hashes(block_hashes, file_length, sr.parent_path, sr.expected_hash) == -1) + { + LOG_ERR << "Error in getting block hashes."; + return -1; + } + + fbschema::p2pmsg::create_msg_from_filehashmap_response( + fbuf, sr.parent_path, block_hashes, + file_length, sr.expected_hash, cons::ctx.lcl); + } + else + { + // If the state request is for a directory we need to reply with the + // file system entries and their hashes inside that dir. + std::vector child_hash_nodes; + if (get_fs_entry_hashes(child_hash_nodes, sr.parent_path, sr.expected_hash) == -1) + { + LOG_ERR << "Error in getting fs entries."; + return -1; + } + + fbschema::p2pmsg::create_msg_from_fsentry_response( + fbuf, sr.parent_path, child_hash_nodes, sr.expected_hash, cons::ctx.lcl); + } + } + + return 0; + } + + /** + * Retrieves the specified data block from a state file if expected hash matches. + * @return Number of bytes read on success. -1 on failure. + */ + int get_file_block(std::vector &block, const std::string_view vpath, + const uint32_t block_id, const hpfs::h32 expected_hash) + { + int fd = 0; + pid_t hpfs_pid = 0; + std::string mount_dir; + if (hpfs::start_fs_session(hpfs_pid, mount_dir, "ro", true) == -1) + return -1; + + // Check whether the existing block hash matches expected hash. + std::vector block_hashes; + if (hpfs::get_file_block_hashes(block_hashes, mount_dir, vpath) == -1) + goto failure; + + if (block_id >= block_hashes.size()) + { + LOG_DBG << "Requested block_id " << block_id << " does not exist."; + goto failure; + } + + if (block_hashes[block_id] != expected_hash) + { + LOG_DBG << "Expected hash mismatch."; + goto failure; + } + + // Get actual block data. + { + const std::string file_path = std::string(mount_dir).append(vpath); + const off_t block_offset = block_id * BLOCK_SIZE; + fd = open(file_path.c_str(), O_RDONLY); + if (fd == -1) + { + LOG_ERR << errno << ": Open failed. " << file_path; + goto failure; + } + + struct stat st; + if (fstat(fd, &st) == -1) + { + LOG_ERR << errno << ": Stat failed. " << file_path; + goto failure; + } + + if (!S_ISREG(st.st_mode)) + { + LOG_ERR << "Not a file. " << file_path; + goto failure; + } + + if (block_offset > st.st_size) + { + LOG_ERR << "Block offset " << block_offset << " larger than file " << st.st_size << " - " << file_path; + goto failure; + } + + const size_t read_len = MIN(BLOCK_SIZE, (st.st_size - block_offset)); + block.resize(read_len); + + lseek(fd, block_offset, SEEK_SET); + const int res = read(fd, block.data(), read_len); + if (res < read_len) + { + LOG_ERR << errno << ": Read failed (result:" << res + << " off:" << block_offset << " len:" << read_len << "). " << file_path; + goto failure; + } + } + + goto success; + + failure: + if (fd > 0) + close(fd); + util::kill_process(hpfs_pid, true); + return -1; + success: + if (fd > 0) + close(fd); + if (util::kill_process(hpfs_pid, true) == -1) + return -1; + return 0; + } + + int get_file_block_hashes(std::vector &hashes, size_t &file_length, + const std::string_view vpath, const hpfs::h32 expected_hash) + { + pid_t hpfs_pid = 0; + std::string mount_dir; + if (hpfs::start_fs_session(hpfs_pid, mount_dir, "ro", true) == -1) + return -1; + + // Check whether the existing file hash matches expected hash. + hpfs::h32 file_hash = hpfs::h32_empty; + if (hpfs::get_hash(file_hash, mount_dir, vpath) == -1) + goto failure; + + if (file_hash != expected_hash) + { + LOG_DBG << "Expected hash mismatch."; + goto failure; + } + + // Get the block hashes. + if (hpfs::get_file_block_hashes(hashes, mount_dir, vpath) == -1) + goto failure; + + // Get actual file length. + { + const std::string file_path = std::string(mount_dir).append(vpath); + struct stat st; + if (stat(file_path.c_str(), &st) == -1) + { + LOG_ERR << errno << ": Stat failed. " << file_path; + goto failure; + } + file_length = st.st_size; + } + + goto success; + + failure: + util::kill_process(hpfs_pid, true); + return -1; + success: + if (util::kill_process(hpfs_pid, true) == -1) + return -1; + return 0; + } + + int get_fs_entry_hashes(std::vector &hash_nodes, + const std::string_view vpath, const hpfs::h32 expected_hash) + { + pid_t hpfs_pid = 0; + std::string mount_dir; + if (hpfs::start_fs_session(hpfs_pid, mount_dir, "ro", true) == -1) + return -1; + + // Check whether the existing dir hash matches expected hash. + hpfs::h32 dir_hash = hpfs::h32_empty; + if (hpfs::get_hash(dir_hash, mount_dir, vpath) == -1) + goto failure; + + if (dir_hash != expected_hash) + { + LOG_DBG << "Expected hash mismatch."; + goto failure; + } + + // Get the children hash nodes. + if (hpfs::get_dir_children_hashes(hash_nodes, mount_dir, vpath) == -1) + goto failure; + + goto success; + + failure: + util::kill_process(hpfs_pid, true); + return -1; + success: + if (util::kill_process(hpfs_pid, true) == -1) + return -1; + return 0; + } +} // namespace state_serve \ No newline at end of file diff --git a/src/state/state_serve.hpp b/src/state/state_serve.hpp new file mode 100644 index 00000000..e3e1efc5 --- /dev/null +++ b/src/state/state_serve.hpp @@ -0,0 +1,23 @@ +#ifndef _HP_CONS_STATE_SERVE_ +#define _HP_CONS_STATE_SERVE_ + +#include "../hpfs/h32.hpp" +#include "../hpfs/hpfs.hpp" +#include "../p2p/p2p.hpp" +#include "../fbschema/p2pmsg_content_generated.h" + +namespace state_serve +{ + int create_state_response(flatbuffers::FlatBufferBuilder &fbuf, const p2p::state_request &sr); + + int get_file_block(std::vector &vec, const std::string_view vpath, + const uint32_t block_id, const hpfs::h32 expected_hash); + + int get_file_block_hashes(std::vector &hashes, size_t &file_length, + const std::string_view vpath, const hpfs::h32 expected_hash); + + int get_fs_entry_hashes(std::vector &hash_nodes, + const std::string_view vpath, const hpfs::h32 expected_hash); +} // namespace state_sync + +#endif \ No newline at end of file diff --git a/src/state/state_sync.cpp b/src/state/state_sync.cpp new file mode 100644 index 00000000..d1dbb6b4 --- /dev/null +++ b/src/state/state_sync.cpp @@ -0,0 +1,429 @@ +#include "../state/state_sync.hpp" +#include "../fbschema/p2pmsg_helpers.hpp" +#include "../fbschema/p2pmsg_content_generated.h" +#include "../fbschema/common_helpers.hpp" +#include "../p2p/p2p.hpp" +#include "../pchheader.hpp" +#include "../cons/cons.hpp" +#include "../hplog.hpp" +#include "../util.hpp" +#include "../hpfs/hpfs.hpp" +#include "../hpfs/h32.hpp" + +namespace state_sync +{ + // Idle loop sleep time (milliseconds). + constexpr uint16_t IDLE_WAIT = 50; + + // Max number of requests that can be awaiting response at any given time. + constexpr uint16_t MAX_AWAITING_REQUESTS = 1; + + // Request loop sleep time (milliseconds). + constexpr uint16_t REQUEST_LOOP_WAIT = 20; + + constexpr size_t BLOCK_SIZE = 4 * 1024 * 1024; // 4MB; + + constexpr int FILE_PERMS = 0644; + + // No. of milliseconds to wait before resubmitting a request. + uint16_t REQUEST_RESUBMIT_TIMEOUT; + + sync_context ctx; + + int init() + { + REQUEST_RESUBMIT_TIMEOUT = conf::cfg.roundtime / 2; + ctx.target_state = hpfs::h32_empty; + ctx.state_sync_thread = std::thread(state_syncer_loop); + return 0; + } + + void deinit() + { + ctx.is_syncing = false; + ctx.is_shutting_down = true; + ctx.state_sync_thread.join(); + } + + /** + * Sets a new target state for the syncing process. + * @param target_state The target state which we should sync towards. + * @param completion_callback The callback function to call upon state sync completion. + */ + void set_target(const hpfs::h32 target_state, void (*const completion_callback)(const hpfs::h32)) + { + std::lock_guard lock(ctx.target_state_update_lock); + + // Do not do anything if we are already syncing towards the specified target state. + if (ctx.is_shutting_down || (ctx.is_syncing && ctx.target_state == target_state)) + return; + + ctx.completion_callback = completion_callback; + ctx.target_state = target_state; + ctx.is_syncing = true; + } + + /** + * Runs the state sync worker loop. + */ + void state_syncer_loop() + { + util::mask_signal(); + + LOG_INFO << "State sync: Worker started."; + + while (!ctx.is_shutting_down) + { + util::sleep(IDLE_WAIT); + + // Keep idling if we are not doing any sync activity. + { + std::lock_guard lock(ctx.target_state_update_lock); + if (!ctx.is_syncing) + continue; + + LOG_INFO << "State sync: Starting sync for target state: " << ctx.target_state; + } + + LOG_DBG << "State sync: Starting hpfs rw session..."; + pid_t hpfs_pid = 0; + if (hpfs::start_fs_session(hpfs_pid, ctx.hpfs_mount_dir, "rw", true) != -1) + { + while (!ctx.is_shutting_down) + { + hpfs::h32 new_state = hpfs::h32_empty; + request_loop(ctx.target_state, new_state); + + if (ctx.is_shutting_down) + break; + + ctx.pending_requests.clear(); + ctx.candidate_state_responses.clear(); + ctx.submitted_requests.clear(); + + { + std::lock_guard lock(ctx.target_state_update_lock); + + if (new_state == ctx.target_state) + { + LOG_INFO << "State sync: Target state achieved: " << new_state; + ctx.completion_callback(new_state); + break; + } + else + { + LOG_INFO << "State sync: Continuing sync for new target: " << ctx.target_state; + continue; + } + } + } + + // Stop hpfs rw session. + LOG_DBG << "State sync: Stopping hpfs session... pid:" << hpfs_pid; + util::kill_process(hpfs_pid, true); + } + else + { + LOG_ERR << "State sync: Failed to start hpfs rw session"; + } + + ctx.target_state = hpfs::h32_empty; + ctx.is_syncing = false; + } + + LOG_INFO << "State sync: Worker stopped."; + } + + void request_loop(const hpfs::h32 current_target, hpfs::h32 &updated_state) + { + // Send the initial root state request. + submit_request(backlog_item{BACKLOG_ITEM_TYPE::DIR, "/", -1, current_target}); + + while (!should_stop_request_loop(current_target)) + { + util::sleep(REQUEST_LOOP_WAIT); + + { + std::lock_guard lock(p2p::ctx.collected_msgs.state_response_mutex); + + // Move collected state responses over to local candidate responses list. + if (!p2p::ctx.collected_msgs.state_response.empty()) + ctx.candidate_state_responses.splice(ctx.candidate_state_responses.end(), p2p::ctx.collected_msgs.state_response); + } + + for (auto &response : ctx.candidate_state_responses) + { + if (should_stop_request_loop(current_target)) + return; + + const fbschema::p2pmsg::Content *content = fbschema::p2pmsg::GetContent(response.data()); + const fbschema::p2pmsg::State_Response_Message *resp_msg = content->message_as_State_Response_Message(); + + // Check whether we are actually waiting for this response's hash. If not, ignore it. + const hpfs::h32 response_hash = fbschema::flatbuff_bytes_to_hash(resp_msg->hash()); + const auto pending_resp_itr = ctx.submitted_requests.find(response_hash); + if (pending_resp_itr == ctx.submitted_requests.end()) + { + LOG_DBG << "Skipping state response due to hash mismatch. Received:" << response_hash; + continue; + } + + // Now that we have received matching hash, remove it from the waiting list. + ctx.submitted_requests.erase(pending_resp_itr); + + // Process the message based on response type. + const fbschema::p2pmsg::State_Response msg_type = resp_msg->state_response_type(); + + if (msg_type == fbschema::p2pmsg::State_Response_Fs_Entry_Response) + handle_fs_entry_response(resp_msg->state_response_as_Fs_Entry_Response()); + else if (msg_type == fbschema::p2pmsg::State_Response_File_HashMap_Response) + handle_file_hashmap_response(resp_msg->state_response_as_File_HashMap_Response()); + else if (msg_type == fbschema::p2pmsg::State_Response_Block_Response) + handle_file_block_response(resp_msg->state_response_as_Block_Response()); + + // After handling each response, check whether we have reached target state. + hpfs::get_hash(updated_state, ctx.hpfs_mount_dir, "/"); + LOG_DBG << "State sync: current:" << updated_state << " | target:" << current_target; + if (updated_state == current_target) + return; + } + + ctx.candidate_state_responses.clear(); + + // Check for long-awaited responses and re-request them. + for (auto &[hash, request] : ctx.submitted_requests) + { + if (should_stop_request_loop(current_target)) + return; + + if (request.waiting_time < REQUEST_RESUBMIT_TIMEOUT) + { + // Increment wait time. + request.waiting_time += REQUEST_LOOP_WAIT; + } + else + { + // Reset the counter and re-submit request. + request.waiting_time = 0; + LOG_DBG << "State sync: Resubmitting request..."; + submit_request(request); + } + } + + // Check whether we can submit any more requests. + if (!ctx.pending_requests.empty() && ctx.submitted_requests.size() < MAX_AWAITING_REQUESTS) + { + const uint16_t available_slots = MAX_AWAITING_REQUESTS - ctx.submitted_requests.size(); + for (int i = 0; i < available_slots && !ctx.pending_requests.empty(); i++) + { + if (should_stop_request_loop(current_target)) + return; + + const backlog_item &request = ctx.pending_requests.front(); + submit_request(request); + ctx.pending_requests.pop_front(); + } + } + } + } + + /** + * Indicates whether to break out of state request processing loop. + */ + bool should_stop_request_loop(const hpfs::h32 current_target) + { + if (ctx.is_shutting_down) + return true; + + // Stop request loop if the target has changed. + std::lock_guard lock(ctx.target_state_update_lock); + return current_target != ctx.target_state; + } + + /** + * Sends a state request to a random peer. + * @param path Requested file or dir path. + * @param is_file Whether the requested path if a file or dir. + * @param block_id The requested block id. Only relevant if requesting a file block. Otherwise -1. + * @param expected_hash The expected hash of the requested data. The peer will ignore the request if their hash is different. + */ + void request_state_from_peer(const std::string &path, const bool is_file, const int32_t block_id, const hpfs::h32 expected_hash) + { + p2p::state_request sr; + sr.parent_path = path; + sr.is_file = is_file; + sr.block_id = block_id; + sr.expected_hash = expected_hash; + + flatbuffers::FlatBufferBuilder fbuf(1024); + fbschema::p2pmsg::create_msg_from_state_request(fbuf, sr, cons::ctx.lcl); + p2p::send_message_to_random_peer(fbuf); //todo: send to a node that hold the majority state to improve reliability of retrieving state. + } + + /** + * Submits a pending state request to the peer. + */ + void submit_request(const backlog_item &request) + { + LOG_DBG << "State sync: Submitting request. type:" << request.type + << " path:" << request.path << " block_id:" << request.block_id + << " hash:" << request.expected_hash; + + ctx.submitted_requests.try_emplace(request.expected_hash, request); + + const bool is_file = request.type != BACKLOG_ITEM_TYPE::DIR; + request_state_from_peer(request.path, is_file, request.block_id, request.expected_hash); + } + + /** + * Process dir children response. + */ + int handle_fs_entry_response(const fbschema::p2pmsg::Fs_Entry_Response *fs_entry_resp) + { + // Get the parent path of the fs entries we have received. + std::string_view parent_vpath = fbschema::flatbuff_str_to_sv(fs_entry_resp->path()); + LOG_DBG << "State sync: Processing fs entries response for " << parent_vpath; + + // Get fs entries we have received. + std::unordered_map peer_fs_entry_map; + fbschema::p2pmsg::flatbuf_statefshashentry_to_statefshashentry(peer_fs_entry_map, fs_entry_resp->entries()); + + // Create physical directory on our side if not exist. + std::string parent_physical_path = std::string(ctx.hpfs_mount_dir).append(parent_vpath); + if (util::create_dir_tree_recursive(parent_physical_path) == -1) + return -1; + + // Get the children hash entries and compare with what we got from peer. + std::vector existing_fs_entries; + if (hpfs::get_dir_children_hashes(existing_fs_entries, ctx.hpfs_mount_dir, parent_vpath) == -1) + return -1; + + // Request more info on fs entries that exist on both sides but are different. + for (const auto &ex_entry : existing_fs_entries) + { + // Construct child vpath. + std::string child_vpath = std::string(parent_vpath) + .append(parent_vpath.back() != '/' ? "/" : "") + .append(ex_entry.name); + + const auto peer_itr = peer_fs_entry_map.find(ex_entry.name); + if (peer_itr != peer_fs_entry_map.end()) + { + // Request state if hash is different. + if (peer_itr->second.hash != ex_entry.hash) + { + // Prioritize file state requests over directories. + if (ex_entry.is_file) + ctx.pending_requests.push_front(backlog_item{BACKLOG_ITEM_TYPE::FILE, child_vpath, -1, peer_itr->second.hash}); + else + ctx.pending_requests.push_back(backlog_item{BACKLOG_ITEM_TYPE::DIR, child_vpath, -1, peer_itr->second.hash}); + } + + peer_fs_entry_map.erase(peer_itr); + } + else + { + // If there was an entry that does not exist on other side, delete it. + std::string child_physical_path = std::string(ctx.hpfs_mount_dir).append(child_vpath); + + if ((ex_entry.is_file && unlink(child_physical_path.c_str()) == -1) || + !ex_entry.is_file && rmdir(child_physical_path.c_str()) == -1) + return -1; + + LOG_DBG << "State sync: Deleted " << (ex_entry.is_file ? "file" : "dir") << " path " << child_vpath; + } + } + + // Queue the remaining peer fs entries (that our side does not have at all) to request. + for (const auto &[name, fs_entry] : peer_fs_entry_map) + { + // Construct child vpath. + std::string child_vpath = std::string(parent_vpath) + .append(parent_vpath.back() != '/' ? "/" : "") + .append(name); + + // Prioritize file state requests over directories. + if (fs_entry.is_file) + ctx.pending_requests.push_front(backlog_item{BACKLOG_ITEM_TYPE::FILE, child_vpath, -1, fs_entry.hash}); + else + ctx.pending_requests.push_back(backlog_item{BACKLOG_ITEM_TYPE::DIR, child_vpath, -1, fs_entry.hash}); + } + + return 0; + } + + /** + * Process file block hash map response. + */ + int handle_file_hashmap_response(const fbschema::p2pmsg::File_HashMap_Response *file_resp) + { + // Get the file path of the block hashes we have received. + std::string file_vpath = std::string(fbschema::flatbuff_str_to_sv(file_resp->path())); + LOG_DBG << "State sync: Processing file block hashes response for " << file_vpath; + + // File block hashes on our side (file might not exist on our side). + std::vector existing_hashes; + if (hpfs::get_file_block_hashes(existing_hashes, ctx.hpfs_mount_dir, file_vpath) == -1 && errno != ENOENT) + return -1; + const size_t existing_hash_count = existing_hashes.size(); + + // File block hashes we received from the peer. + const hpfs::h32 *peer_hashes = reinterpret_cast(file_resp->hash_map()->data()); + const size_t peer_hash_count = file_resp->hash_map()->size() / sizeof(hpfs::h32); + + // Compare the block hashes and request any differences. + auto insert_itr = ctx.pending_requests.begin(); + const int32_t max_block_id = MAX(existing_hash_count, peer_hash_count) - 1; + for (int32_t block_id = 0; block_id <= max_block_id; block_id++) + { + // Insert at front to give priority to block requests while preserving block order. + if (block_id >= existing_hash_count || existing_hashes[block_id] != peer_hashes[block_id]) + ctx.pending_requests.insert(insert_itr, backlog_item{BACKLOG_ITEM_TYPE::BLOCK, file_vpath, block_id, peer_hashes[block_id]}); + } + + if (existing_hashes.size() >= peer_hash_count) + { + // If peer file might be smaller, truncate our file to match with peer file. + std::string file_physical_path = std::string(ctx.hpfs_mount_dir).append(file_vpath); + if (truncate(file_physical_path.c_str(), file_resp->file_length()) == -1) + return -1; + } + + return 0; + } + + /** + * Process file block response. + */ + int handle_file_block_response(const fbschema::p2pmsg::Block_Response *block_msg) + { + // Get the file path of the block data we have received. + std::string_view file_vpath = fbschema::flatbuff_str_to_sv(block_msg->path()); + const uint32_t block_id = block_msg->block_id(); + std::string_view buf = fbschema::flatbuff_bytes_to_sv(block_msg->data()); + + LOG_DBG << "State sync: Writing block_id " << block_id + << " (len:" << buf.length() + << ") of " << file_vpath; + + std::string file_physical_path = std::string(ctx.hpfs_mount_dir).append(file_vpath); + const int fd = open(file_physical_path.c_str(), O_WRONLY | O_CREAT, FILE_PERMS); + if (fd == -1) + { + LOG_ERR << errno << " Open failed " << file_physical_path; + return -1; + } + + const off_t offset = block_id * BLOCK_SIZE; + const int res = pwrite(fd, buf.data(), buf.length(), offset); + close(fd); + if (res < buf.length()) + { + LOG_ERR << errno << " Write failed " << file_physical_path; + return -1; + } + + return 0; + } + +} // namespace state_sync \ No newline at end of file diff --git a/src/state/state_sync.hpp b/src/state/state_sync.hpp new file mode 100644 index 00000000..599b7323 --- /dev/null +++ b/src/state/state_sync.hpp @@ -0,0 +1,83 @@ +#ifndef _HP_CONS_STATE_SYNC_ +#define _HP_CONS_STATE_SYNC_ + +#include "../pchheader.hpp" +#include "../p2p/p2p.hpp" +#include "../fbschema/p2pmsg_content_generated.h" +#include "../hpfs/h32.hpp" + +namespace state_sync +{ + + enum BACKLOG_ITEM_TYPE + { + DIR = 0, + FILE = 1, + BLOCK = 2 + }; + + // Represents a queued up state sync operation which needs to be performed. + struct backlog_item + { + BACKLOG_ITEM_TYPE type; + std::string path; + int32_t block_id = -1; // Only relevant if type=BLOCK + hpfs::h32 expected_hash; + + // No. of millisconds that this item has been waiting in pending state. + // Used by pending_responses list to increase waiting time and resubmit request. + uint16_t waiting_time = 0; + }; + + struct sync_context + { + // The current target state we are syncing towards. + hpfs::h32 target_state; + + // List of state responses flatbuffer messages to be processed. + std::list candidate_state_responses; + + // List of pending sync requests to be sent out. + std::list pending_requests; + + // List of submitted requests we are awaiting responses for, keyed by expected response hash. + std::unordered_map submitted_requests; + + std::thread state_sync_thread; + std::mutex target_state_update_lock; + bool is_syncing = false; + bool is_shutting_down = false; + std::string hpfs_mount_dir; + + void (*completion_callback)(const hpfs::h32); + }; + + extern sync_context ctx; + + extern std::list candidate_state_responses; + + int init(); + + void deinit(); + + void set_target(const hpfs::h32 target_state, void (*const completion_callback)(const hpfs::h32)); + + void state_syncer_loop(); + + void request_loop(const hpfs::h32 current_target, hpfs::h32 &updated_state); + + bool should_stop_request_loop(const hpfs::h32 current_target); + + void request_state_from_peer(const std::string &path, const bool is_file, const int32_t block_id, const hpfs::h32 expected_hash); + + void submit_request(const backlog_item &request); + + int handle_fs_entry_response(const fbschema::p2pmsg::Fs_Entry_Response *fs_entry_resp); + + int handle_file_hashmap_response(const fbschema::p2pmsg::File_HashMap_Response *file_resp); + + int handle_file_block_response(const fbschema::p2pmsg::Block_Response *block_msg); + +} // namespace state_sync + +#endif \ No newline at end of file diff --git a/src/util.cpp b/src/util.cpp index 2b5e020c..c08ebd7b 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -207,23 +207,61 @@ namespace util pthread_sigmask(SIG_BLOCK, &mask, NULL); } - // Kill a process with SIGINT and wait until it stops running. - int kill_process(const pid_t pid, const bool wait) + // Clears signal mask from the calling thread. + // Used for other processes forked from hpcore threads. + void unmask_signal() { - if (kill(pid, SIGINT) == -1) + sigset_t mask; + sigemptyset(&mask); + pthread_sigmask(SIG_SETMASK, &mask, NULL); + } + + // Kill a process with a signal and wait until it stops running. + int kill_process(const pid_t pid, const bool wait, int signal) + { + if (kill(pid, signal) == -1) { - LOG_ERR << errno << ": Error issuing SIGINT to pid " << pid; + LOG_ERR << errno << ": Error issuing signal to pid " << pid; return -1; } int pid_status; if (wait && waitpid(pid, &pid_status, 0) == -1) { - LOG_ERR << errno << ": waitpid failed."; + LOG_ERR << errno << ": waitpid after kill failed."; return -1; } return 0; } + bool is_dir_exists(std::string_view path) + { + struct stat st; + return (stat(path.data(), &st) == 0 && S_ISDIR(st.st_mode)); + } + + int create_dir_tree_recursive(std::string_view path) + { + if (strcmp(path.data(), "/") == 0) // No need of checking if we are at root. + return 0; + + // Check whether this dir exists or not. + struct stat st; + if (stat(path.data(), &st) != 0 || !S_ISDIR(st.st_mode)) + { + // Check and create parent dir tree first. + char *path2 = strdup(path.data()); + char *parent_dir_path = dirname(path2); + if (create_dir_tree_recursive(parent_dir_path) == -1) + return -1; + + // Create this dir. + if (mkdir(path.data(), S_IRWXU | S_IRWXG | S_IROTH) == -1) + return -1; + } + + return 0; + } + } // namespace util diff --git a/src/util.hpp b/src/util.hpp index a0162f11..032654cd 100644 --- a/src/util.hpp +++ b/src/util.hpp @@ -7,80 +7,87 @@ /** * Contains helper functions and data structures used by multiple other subsystems. */ + +#define MAX(a, b) ((a > b) ? a : b) +#define MIN(a, b) ((a < b) ? a : b) + namespace util { + // Hot Pocket version. Displayed on 'hotpocket version' and written to new contract configs. + constexpr const char *HP_VERSION = "0.1"; -// Hot Pocket version. Displayed on 'hotpocket version' and written to new contract configs. -constexpr const char *HP_VERSION = "0.1"; + // Minimum compatible contract config version (this will be used to validate contract configs) + constexpr const char *MIN_CONTRACT_VERSION = "0.1"; -// Minimum compatible contract config version (this will be used to validate contract configs) -constexpr const char *MIN_CONTRACT_VERSION = "0.1"; + // Current version of the peer message protocol. + constexpr uint8_t PEERMSG_VERSION = 1; -// Current version of the peer message protocol. -constexpr uint8_t PEERMSG_VERSION = 1; + // Minimum compatible peer message version (this will be used to accept/reject incoming peer connections) + // (Keeping this as int for effcient msg payload and comparison) + constexpr uint8_t MIN_PEERMSG_VERSION = 1; -// Minimum compatible peer message version (this will be used to accept/reject incoming peer connections) -// (Keeping this as int for effcient msg payload and comparison) -constexpr uint8_t MIN_PEERMSG_VERSION = 1; + // Minimum compatible npl contract input version (this will be used to generate the npl input to feed the contract) + // (Keeping this as int for effcient msg payload and comparison) + constexpr uint8_t MIN_NPL_INPUT_VERSION = 1; -// Minimum compatible npl contract input version (this will be used to generate the npl input to feed the contract) -// (Keeping this as int for effcient msg payload and comparison) -constexpr uint8_t MIN_NPL_INPUT_VERSION = 1; - - - -/** + /** * FIFO hash set with a max size. */ -class rollover_hashset -{ -private: - // The set of recent hashes used for duplicate detection. - std::unordered_set recent_hashes; + class rollover_hashset + { + private: + // The set of recent hashes used for duplicate detection. + std::unordered_set recent_hashes; - // The supporting list of recent hashes used for adding and removing hashes from - // the 'recent_hashes' in a first-in-first-out manner. - std::list recent_hashes_list; + // The supporting list of recent hashes used for adding and removing hashes from + // the 'recent_hashes' in a first-in-first-out manner. + std::list recent_hashes_list; - uint32_t maxsize; + uint32_t maxsize; -public: - rollover_hashset(const uint32_t maxsize); - bool try_emplace(const std::string hash); -}; + public: + rollover_hashset(const uint32_t maxsize); + bool try_emplace(const std::string hash); + }; -/** + /** * A string set with expiration for elements. */ -class ttl_set -{ -private: - // Keeps short-lived items with their absolute expiration time. - std::unordered_map ttlmap; + class ttl_set + { + private: + // Keeps short-lived items with their absolute expiration time. + std::unordered_map ttlmap; -public: - void emplace(const std::string key, uint64_t ttl_milli); - void erase(const std::string &key); - bool exists(const std::string &key); -}; + public: + void emplace(const std::string key, uint64_t ttl_milli); + void erase(const std::string &key); + bool exists(const std::string &key); + }; -int bin2hex(std::string &encoded_string, const unsigned char *bin, const size_t bin_len); + int bin2hex(std::string &encoded_string, const unsigned char *bin, const size_t bin_len); -int hex2bin(unsigned char *decoded, const size_t decoded_len, std::string_view hex_str); + int hex2bin(unsigned char *decoded, const size_t decoded_len, std::string_view hex_str); -int64_t get_epoch_milliseconds(); + int64_t get_epoch_milliseconds(); -void sleep(const uint64_t milliseconds); + void sleep(const uint64_t milliseconds); -int version_compare(const std::string &x, const std::string &y); + int version_compare(const std::string &x, const std::string &y); -std::string_view getsv(const rapidjson::Value &v); + std::string_view getsv(const rapidjson::Value &v); -std::string realpath(std::string path); + std::string realpath(std::string path); -void mask_signal(); + void mask_signal(); -int kill_process(const pid_t pid, const bool wait); + void unmask_signal(); + + int kill_process(const pid_t pid, const bool wait, int signal = SIGINT); + + bool is_dir_exists(std::string_view path); + + int create_dir_tree_recursive(std::string_view path); } // namespace util diff --git a/test/bin/hpfs b/test/bin/hpfs index 0c51e627ba183042330e5f50429b0a282e81b3ee..57bb1edd0b288ec3807b92591b8186181938d460 100755 GIT binary patch delta 33012 zcmZtP4SbK)|M>CGxpreT4BHH28*>xG#@ye{{b(2s%k4-)VwjSzY;H3q%QfE8Xr!8w zRvSZNBvfd$q!J_PQz(^$w9o%_uIppe@Be)~^gQQW=Q`Ip=Q=mn=lXp3t~*(2-N{Os z#TD0x;n4qE^U~M*t0kOPR-{~WtNPY|-i={jHkt5p;j-3!>NVcI{miL0ztzcKXCzfp zOU>K9V-2q#%@LK~Afl>_Rol&R{CBZ=ph`=_QEdKRrBAK12dqLJ-ge=RbzHH&ql)9E zIk;*~b=jO;^-ZDNMaG&lLJQfJMmYBg1X*{|BzT4^8KMI_ibzIn(l z;%oCjwZ661<=M&NJ{nlXK03z?t6o!On;ojJqMT2vk5%i;hikM{iRNQ9#;TF|yKAhf zQ>$IBeOA%>`c}ZlUC0;axRy0*`A)VgUHqS;d(4+Os#5HurT;k^Wgci*vv%of`{?!m z91S>O{@${MiZR0?YetoRVQ2Dt<=>Zgl-TJ(|4DD3KRNOz->P#dJ9cGd4~s54^2c?_ zR_c6ySyvxr1ZL*@#LPDQPOc159vsY&| z)%=71b~Gb2uRFemliq9&P@`?OP!n!WpUF={P07x!teelS1m6RtioFZSu6UN(dK zc2LvJVf?q$%X5^%KW<8Fik>zw^{cFSR9q*o;Y))HC_}|7R$*A^+gKB17HCUzED4lKRxFu{=^m znZ1@btkk-%-IfN8%vsAXB%WzVlhS)1ja||EjBesZJ6;^SawvaDPd}}be;|2rQOamL z>4-C+k#$-rrkfQ4Ml+vMRg8;nJyVd^9WFu_Mk6X17%V>ZCbz zRb;ilbjFoS{H%;eoR#XDYgg50D(_x3i@D!Cvo9lPc4mx9&EJ{%u~I9{d8_NI@#d?m zo7BEl>)yqK$}g_`xcEHt!s_6lbJpP{k^0!mB{5c(BhKoz%>c7yV0-RH_q}>zJaF;e z<_L3?a+-zafF3KnZdR-Bpu_6Lt_ZkNwZ0>Eh1Xe4hlBGT>b_S`?VmPtKFP;*a{7`c zb|aJBEg5On%L-9RX4kCI>T@$Y>qYgJ*=$X;>R=|VsT1@V^;~pIRb9MX*K0*Uzbe)h zI-2I1vDMxv(!FL4M|s2LavG+`+DB?n^rdO->ghHwnPWAo%cDAJS8X#XyPmmpZOH*qybVRJ_?%x>WJ%s2nRijo8N-++MfL1u|Y8FfU1DtDH(? zpAl@8o?~~}y?avmfYpOV?jGE^Sx37}H~XwhGk&RP9$go%cA3}KwN_bX*vnylclKM^ zpLJ({%^dRbP2*~X{Qm17QR-E5?JFOtk>-f(X?;`nx?ER0V$(n37Tha5Xk&TP-+5IB z^}*}zqIy1X!oA{u++bGSP`BD7GCRN6z!?h;t>dqE-My$rgjSLdjQ%(q@mAN+}zzOzT1 zopw|3iZy3+J*-J-tqBk9(G@&MN{)N()zJB@3`d;DcUfT$9dTaSsdFB2MwyFq8mIwg zZcgLg-$mNff8~0YYe`Ychs+_X!xt6RU~xF$K04h#`s%-rPIez1Z2E7C_CHG=yAns7 z&O#mbddwWP=`}Tb_2zn^u{-Q*Z`Zlfm&EGfxx$7T-CApQ-CVD=&&#?Jv*S6^UT5IQ zUj??K?eh}t^DH>}urfwC(P+v5ia;?!WHdBhCpr zVVFH2?w#SYR@d>&m3DSMtI*D-dc8*V(lzB{%RarP+359f@BYFt*li=3Y@U5RlRfL{HzHLn$_MMsQQ{oZ~A*>nl9HTGH#UeH&?z{ zIkm^#Fo*MrEM5D6wzt9xC?^w481pt7Cu97!D6lYGC%ofXl(iq`&=%EW4~D|ceSxK)4YDBwiI_z zmCUg>D;URCnK`qA%$!|b>goJjxgV-Z-!@iOU0yCT54_zPbf(K*MPsFC5cW%^)i=Hoaqy%xGR5xpyxhJokZ_8Yr z+ghL2<^PjcMFmZNmy`~@omE#CvG;T!`z@V4#%%DubPwFZjlZrq(2E1?s7IXBnGkkT z6iL>7Yov={y1R2MvZl;Aoq4JG_xlahC^IBKEHyzVyR&@rjy{9Ur+k-Q>^{0rFVFha z$7sS&uj`2WDtD}N*y`Q!XSUmAtnB@jdk$A$YF*(BJD_5q-b+4vLT5Dh=Le(~>0!HK zXz2qN{@YH{#ol{jIAPUg=z~cg=q6f49&vtaU&dUK z$R=M}_fKEXx~bdW)WztV?~G0V#=7||Rc~>|`SvQASnVXY76!(###TR6TS2 z?j9k#KGkRG)$d*xp1Iz1vscGHXy>;)ruls`>n$+Ge+%)$_mqV4_mf%>EyStAEVtAI|cw|2)(2Bbk<} zTFbjVRrUPl`<_-t>8AYb{ZA-0#WW7g@?7;C4Op;S)v~9^_VJa?tp}R6dev&|5od=S zm#bg;A-mB>oSk2Fx#H3<>L(aRep3$K>z($d*TDv<%b0^#+T2Tv*l4%i>M?6CSg(uK z{kbCE&o0wSo~(P?-Oabx{`Vwn9=VThC@+$GVx8O9U8Ek&-gM0oXR!Is!Rcy_S@)xN z)lu{KM+KgXp5dO%I2mGYITUVmen!ve0Q3Bz$*Q*5@8c%bcD|@Dsk@jvd&GI?6?5Uo z&nRE>_Q&t5$MUxwo~4W*X6A<-^)`IIOV%|Veao!&d4N6oe)JGt(`Qv1v2Px!ny z#jcl(dqsttsm0#a{>;>k(p_y2wbka@;)m61W>N7Y#_h>w<&&e-E9R7w!##UWc6ZLD zli|kQNp|Oioa$%XS(!iO)GFoq(350%wWM=?hmvV3VESEq&A5`GuV}AR`^~*y*5{Mx z`7dj$$>yCer>dvSCr^i|!)E$vJ&t#tZf9J2%l!3pH!9Hbt4_wH)#lW%TB}xO&Q}ks zCFZAJh5GN*_x<0?#b^0tUyb+RljOK>Yu1WL((U=kdC1b502RkE0aEf``nIW3{qpyo z9i}=MBdt>ON|5UwAl<9dUvo#>XICGiway++(0Ona@kNIcN16wg)RN>%%FCR5?s4Yg z@pG+IdwS3MIl(S<;n4C@j}UaW%KzfBsXALh!b{E%X%UTJL9pJR@DvxaBCL98RH zt!iZca3#mPDiwFmHCJ3c;oXS?&Ns}l*G^P;av;G|ac0KVQD){ZGb)$^h?hNLCS7ap zeN@l9CT6E!oBI93?0I0t^EPMy+TCa1BP6nDo7eXR<)8l5$3w-L=YM}z?K6k}(KxhG ztaXZWpQYU){V04|-}`5oJOB8B(V2Z?x<`jTF3H=WnwV|>e7{vhPx7o7n)$$xdFn6D z-O}4GvvVJl&%n>p_9l`7Eu_3v}4UH*x?zZt4`ev#{WrM{A` zRI%2ej`lTww6v}n(TQt{OjByHYA=5%^{jeJo-$P9TGcz)ncuRsGXF}@IYQnx)FR{V zV2Sil&!}ngriW@=`*b@y_cR`b-TjkK&{?k?H@J=&GR05UhpWnkp`L1(p3)ws?p5q1f!o0pd`hoJNOW-du8P@&SRs>LNjYD$3*U zruHb16hHNThhG|V=D&x*l5Z$sVpF^1GE1vddNw0{`@^I!IyCA2%2D`yHAQ#sDV$M5 znMyShuK=}HJtx@#^zDU)b}`R2(ptsrBj|j%@N9s}Fw{Yr9HiE%8e-IABh+;F3dw5mfRFNC-o`C=XQ zzABOK!HkNy5WDzGmR4mRrW9wE%m`M~)hM|Tta=*Bb?yG{T32;6g58nTbyY88Vkeou zTh;emAI!q?+qRnWLX--YsCp_VY^y!UnM~G0(|R54@X_p^_g0N``MsV>slBll=_`g_ zk4-O*RYkF?IQBrPr(OcuNLq;MRCRhA>$xK?{g=4(AD!>Wry;7g$}9XKL|rqC9)aTL ztAaw_SW2DqgY33EqD!3y9lcjxo73mU$ONU@yxkDRPVv37EkdBR2i)tA) z%FhhbXJ!;W*I0RY80)IojlIii?5tQ#XZpyw$D`o5oNEe`qf{$Z zXS}yn?!Rujx31vr3i4qqb=6}5!`rIaN3E&ZW*^S2TtU|?P^w0&0MBF!V7!G(Tr~Ia zbhup4Qnk#_|L`hY8m;asRb3XfRb9MGyg0F2;o-J=I?j+i?bTc#hlh3TTe9yX)kL0Z z$D?1-QBV1%yQ)?AN(c3U2W!OeF6up%Ar-q)#=x>NmveVvTvwH-)TzSPyD47FY?0q$ zRF^6f|K`}q+htbE_52SqlS?YgxbAAO|B$~(wg(gMY>qg8zg6ZkFAVe7SNhwtB{7Nm zkebhE(80hheHzB)Ohvvz}toV>i0_>D{K3x zCmVDl>0gCj{-eCm|NTj!m;aE^zAD;C8zTb-t6Hj!B==PjAuZYX$_wX3>k;RRdYrmn zZ*-Io`l?lRyBJoclsM-TkJzCeuiW$0vtRnhy%%4y&ah_kO z^Urm;-A{cOa?(|1y>J}jyv+lOe)SkjuHo*Ii{;DyYPi226Ome0ec;FriFriz?q68B zyvgN7kL2Z!`#xXi#ovEhy_i$%RzFx?Bl-Lh73RH#6P#7$<|C??S}&dAR9zJ(kHx80 z^<%m{&?2jMU)2-*ev4w|oj4W4X!<2iO>5ljAFEwe7q8Ll;UM>jUHk&E2U<~I77b8s zjVr&%2LsfY4h>imdN00aHFTJ}3yxeZBh9{R)@92o{h!WwO@<6q?NakOa_@EMy?Zm9 z)7_nM^_uRCBfS^jw1z;MzIADwG9bO|zZVVTtO4nN{_DC!t*m;qJlJo4T`3Iy*Znz(h*yKF?Po1?4`D0$NqI3|MGl=$^1miS!=KB0Vh@%6@30t1 z{`-@`D(R1(%GhhFY-ijjK$@ADS= zDi2jHLsqqY;NsSJ_(ktb?o8`t*2tEj>UNE9saE%NzS&yBXI^U&dR_8{seX-K zWG&L2JH@%v9$qO)aVy^8366r6gt+rAm5||TlK&1>K9lYp86mF>R}Iu!**{!`MU5iE zz5cdp>hXPf+tywBzx^E{)e==elM@$pAAf&uD&DJMP2C!M|B2&dXQ2#CR6XnV9PqD> z3ANth+8uN2d)c0-T7>@E>VclIrqDpg{hqmVLGC81(Z+dIIPx)7Q`PWhANhCZ6w8tk zs-NF(Rz`h$7wd1Qo!_;Wb0gG1HAk9{R7w8tvSPU>H_!W4pC-trkt#%G$$^n7Y}yDi z{Hr(A`SRXaTl%lwu9{P!2ydu1Lw_y6sj0QLcQ=k%v6=MISXBc(m%MJeGe`W`=y!qOj9g z9#y>6S<0PS_;QkJrqpKnc$`Z19{UYxpB8o+uRc~4vd$23ot6p{)Nb#RuZVP$!xPjr zpS;s$E-tlOIzOQrRxmljt0wckMgC^-(HgSq2^C&-Qwdpm-(B?lUDxb~oUaugd_sj7 zsk2DsC46*OTJfBtrS2!kq1I8NE8BcCh3UX0dfZR?)ScAwe$pOyQtY5Jk@_rYjcxtEZnKiJtzet=-cv$hk=>+N0_9GErlAE81eR`oZ%zN|pDrd1+XB$V;N8 zs7}g9o}QvcsQ2XP6jfiP$dxIoZncWs!}6g=p}nNmQ>vjFE8U*r<4ZLe@s#SPM#z?@ zREOy543Q5Qg03so9+=>FzoUosmwDD?ra?^x+@Ax>q}o*VNVU%Q*ZAgC%(+gUo~pW4 zjPGE#;Gpc7s+!eWtM^3rH%RezI8Ar{G`T%hHLddJ6OQ}ea5^RGY1OKxI;JoF() z2aS>Q(^XjK&Q?9Gk67vSi+&;HZcTlT{A-D*c0cPFvN}JJ&dI8I_}OR%+gh%{w0^Rs zu3(!_NMc;{(=n>}^(T5-hZeq*%nB9x^E|uG@u{86U2!;n;J6NdK99HVKinGu05f=ckz}Tpf`rvtYBdo~7y=s}9S_qdZrq%u<0~bC2Ac z4dI8PsNkTZRs};NtO~wI5*0Lbj0&zkT)ysj7XCg4 z>rUCXZkxP3TXm{fI-36E{AKdpZ1rfxE$(Q9ba+OMs_Xi=%$0dX*>BudjCL|vgYQk& zN9FxzR7*8dE?ztk32GmPm%j&{v6(1^$?He)MFmG2g=L^i)wmO zqmN|nbLuH3+vVrzO&@79SG7QKGo7**3Ppc^ZOH%>H z&!w_DP4#Lzv7x(^^h!KK@Ks%C!3X+^LtnLqq~k)~n|v zO?On`LyHtYK${8W)aWZ468t1>7CEj{vT>hlAuvliiL_z;7Rhg=Sy!e9Z;eDup5?adX z7gS8&M)k_a?+^A$GfA&#^ljVEby-K8+2p2e%V_d49luK5iD$Anr@FKDE0D42Dya2e z`rVz!sNA8~GBrGDVd0g}5+y;!l&BX+wRNw;+7a$gyft~#Z@ zrfYEjDsk^||3*Ks04?IU`-YZRr7@Y@_jK?3I#|A34gLS*zHPO|THIbFx1Nc57J6E< z(2G6ur#z{F41xd$}4`Z%};5=Da zuu^##YJluo#gpndIlD^LQ5)peD%Cq+cmVy|sv2i{8kD+rD<`tLS7xrSY!gosGrymXF<`!aQy=VN z=!~_jPlf?q?To)x*ZI3FZo`V6FYm8bKdP~X>rJ)8gC!|m)GW{Lukm8&z?nL7QPgSA zxXnC&TzEr;m|Z4TldJ1gkcVS)nYn3paAAX&)#EC(3K^^h*x$$1(L1>P5PoA5RURvE ztXIh%^Ec`z#hxGY_(|qx>N-}a2 zHGYX7ShN^lQ$p*js*=4)jr82GUcddazb5Ujco^QS5z}%go>Yjw?)gQ;Fe| zzM970$SgBgUL7gZ-cmCv?|6xn&`kHKe3w+{`L+u1i28duN&o(WF6eLGBFY^D>HiQ^pjibLyyc*`bn^$o3uTY}E3FWd8Rz5^w%aos1=RA?q!zLe3L( z_LmQKswp1l7r86Ytx&bDv3h|nvbnos4?$eHO{M}wvmO+uM@PB@+vW&N9%X=TGz`{+3RfhiZ ztHn_@FdjO^pL( zuC(W`I8yw{9j5nGvZTr#KMOS^u_MwD+s)ABuQ|?`<6YcKK z@3%R2eP@D8|Zw?G9GCCbJO&LYGBLuTq8{vy0VM6 zuJKDWmZXc{P!p;+#Ew4izU(Y%^_iMbqoF%`n4|niuj&!6CbIQ2RX6pP-YPh4t$o1v zh}W&JLmhOuoy#+CBr^?n zT&ZODVSVM0-p5qksD)E?6P8@p7t^nFx=bdgv-Af#(g;${Lr<@wU>3xjGq)oG0S@E#f0QvQps#)Wt`@pdB0}Z6sarJQZ z%}@R3%AZKWai)64)9w!f{xWK{s@(4lJx#`OHXB;2Q}p!8HYQ1IX#P6JHMdx8=u%l{ zY-J6OfQoj;JIp8Abeu%zeEIRXY8|$Y8_U9Ti>_^=Pu4fu9go%VInwC_?=1vFVyWaXG?t;zRG7=J(Rqe~bfS5By|T_?7v8nT!RI~-L*dhv7R;oXPz zOaH3-(C!IwD;u``EYvYHc4hPV_-Oa^fBL20jC=S8Yo;9d$HV#6BpLg;8tv!MeU$!t zdf9$GEPkq1cBAo#*MFMvyl^k9?M8e8e~x@o=4#<+(L#Rv zg0<-2Nj{K8@5pw!>aKUWhVW++e{@{`x&1&`$E&{LawYO-JYjj~qzWob`a%^OsS)GL zTnP;vjqtLWF<;kbj5*fFCl)fG=*CX+WAPhl43unXMwHdum#@2{utE4 z;aHEIF&9VSDV&F;n2k$TmAQ)0#8Nzlj+PEb5Km^IxS9vISlqIjdpC)Izggy5hW#-Y z6Y(fcMJMh-KVF=aU?eu;=`bFXF&P(QCgxxs?!{ufjMve74d+KW98EA1hY7bDvr`Go zBd{H_@hI-a-|;LCT*p1bIT*~Mc$uf)&KS(Iu>hp6ysaFdT0x3j~Mh7tH4#$gdo#fzAMq3?1&M&mIYhgWer`gUQOVg&BSIJ}Hg z@vnE8{}}{&<}qz>Fc#x#ypFG-Uss3YSB%DLyoE@>$v6jh<2w8ncVX-I8THs5@8Tc~ zX01=d&Zu8TjKcjm59{ZrGWZD0%4e|kWV9DhQ+#a)HO0||jOtztMqX*1#g{&0Q1#|1 zWIq+b8b>Gy^Dr9^f5PC!)}J$W`Y@JGGM1LH*`QB4OFQ1>dKFGlu^85aR05p|EWuIu zKF-6Nn1jAwFsiWymSQ40y0g1tC@#Sm%)=zyhiUi?=HN{%!s;cQk1f#AgRz34m|8?2 zhQL`&!dsYz{$El6Mq&}Zfu(p5Q|OsXn1us*_f~+@u>_yT8@K}f>9Kb(8joNCeu*jg z7iRTh{s(@=l1P_L;q_t>F2N>r-5VH-t-fZuVH_^QNtla^@hI-Xt5|}*eVLvZf#Kh< zP+>1j#Zz>lcaEpX^42or1VS9W56LB-n!GpLCBhOL*4#hK=gtu_~ zIl8nT=V1)Kg-KY9X?P8D@Gcf%nC*5|&~HdY3ZIFanQZ zAH0fl(eDxV5RAl&7>~XeXgNk)WOKqK%)?wP#+{gYoq&^oU!23?`91dvV=)oOT;h=n z7vdIN_X8*5hj<-7L%#tICN$?^G$!CAOu^Ndg?oQwjNnBq!P-v72sUvt|NRH@_ryqu zM*a5}6EF)?@F-^CMJ&L(Sc3I_q5#xiqWBMTI7VPJzJLk122=1;T!+_j7Y6>!EyGB> zh4K2?EhwJd=nD57U%bja!}XYkZ(t7kUE@gzyW%D6kDi0s$1xm>FcyEoN$B$ngB3?% zE^fx7_$fN^68a2bn@=Usgn;L-6o3&p3DYnGPvJIPdYyZY^MB)>V>bFe%6^WKSo?SG zIkv@QT#8wkg$1||OYj)pz;Dn$f%)$w5KUmvA1pvP2It`x%to&pj8^o=Qrv@{Ls=;? z9Dm1N*!52;j>~ZwzJa;ud6O}NiFh5iq2Dlv<2#HT#{6&k7d0oLJtkuUX5#y}3%ze~ z&u|3Z!VC-=?r{8w?eQii;+WgqGR(sqEXE>ib%$Gqm(eGY!TmSu17>4fBJ=+ufvF@^ zx=WX1IBvr@JccvzDrTVXV@xxQz`YoU4gO)UVJpnSIaq)tSb~@F2DT`p%SZ6ojnTLa z6H*Bj6G*`z7X@HfEWmgy!EgBm#SQ!!{YP@I`DsEl#$f`k!xTJLy_(reW8bE>|vA4|2JRu{~bL%NR7C^Z2Q7XUxM< zcopa2zS=HVHl9f(u$O>;9hd7YF2=jK5rZFRcf-zj1C!7Q=E7JHbFd>8;WKqoj{ZbVOG4=kR@{LdwDorFE;_aqAoM&ezJ z$EJMwm5lM2iAS*jZ(<3?G@;_Sksr(lP2z#1nadS}%Ohwp9&XNgSc-Xg2aB<33$8mU zl{K1x-(*^bk$3{*@hm3e@JOZ=PR0UUf+bjlH}D(upTaG|X!MU_T4GB~!J(Lii?9GU zVhQfU8>s}oCgA@R_ox*GV0}!$&X|IuFbki>0^EQlcnojgRrH_AAZ*R~7>Wtl5mRt2 zuERXsg#$E73cw6JgW>$x%ySx#SQw7c z`mbzY3{JxFxD4lGF0RL;=-GjaW4#WpRG0sB3LqgGcVQy7>BK0jKTVtgi)A=^KlEV#6$QgUc#Tza|W}ZGxI;3K*cUDS1+uI6L2Oj!}l>4 z6S`7!+}Dko<6r1MlLv_yDvn(-0TVF=voH%McITF18kS%--oRbxKZ_nr?MclEbj2i0 z#5A0bIXIZ#JRile=)_a#o5FV8hYRCmOu#vqf{QT=GqC{oVhQS>d*8q^^qj{{2` zXPL$r`z-Un%OeaH5}w3mI2&{E0v^L(@hVn|qvdm02QUJin241IFi7zZW~2UFU`KHw zI`I(tJjWo%Cg?GU3*#u9`W*9r3V{p~PGcUfk7p^yF@rgAE?tXFa5=`}Wt@usLpTpx z;5JOeV>l77;!^beAJ2XmfkhaH5s$L2;0Vl2C6GcO4`0M$EWqn{2mR*J z_E>cUU5zm~2Pfh>Oc+U5<56_tMf9OZ?_v||G>RU>{-{^lwYUs7V=itVqkKx!0d*KkA zfG^@=JcL{D3?9PEsRS+&a7>_p#jN2Njsvk5F2)I1jEnIiZo#TgPylwoOZX^yF5$x~ zhT{v^3pe8gyg8As$1zVbSh49OrX{9!B5;F13i>alrWlR6n1EF$)Ajf;uESoq3;RrA zmSFX#xbO?~AckW$_QJPt0-nRgScY3Ld@2>kJiPP*^ZyV5&t<$yd743k^)VKo!AW=u zmtiU9;;?B9I-G$K={(^~r=qC;T4y#!%_84&o)u@)Li}qE<*#6Sp3C-*O`c~K;nMk3 zYz2d^)j}S@NNBs1X||Fs!BLon^C<8P9>p8zM86jpq!b*1O)wT?aUV{?Z*Upj#2b`b zeHlHQ!8(D_I35#m9?nT6u#CVu%)(t*hG(%}I!h)_#9%5k2j}5Y+>6gHrwcJ1JzsP< zaxff&GgxS_D^A5k%)qB`8|L6K%*WKL1dbE%UBz=hM&MnH!x}3o06Sm?Ucv(0@*)+) zZ}2W&!{AIF+gH(p7=xp5GS0&{FdO&cUObIwarR5h|GNaHujYi+%v+p?Cor3q{erVqg~MQ(icB#6uY4O?ZiV&ZhXjuSR8xYqEt3*+%1#!*fQPQ}ZZfj4j)HrU85 zz#*vwt`aCfUkW;l5$Hre3i=CEFf)gmW3$anA8h*?6~~Xzb1h>hmq8_btj0O`J4TZp zyp`F3Jun3)VHPgL0!)2_Kna0ecmvzM#j_v<#A16~g`?In?cQd(;3V9FHMVi#m+9(v zxeykkANo*v&zgY%jH83ZEoS*b9a3+}_c_zj-L zn|K$if5^S$!ppETZoxzN5njSm=u5$4_OWE+Tuj5Yn1i3{8eZC*mZ03o~#RZo{isjG;x`b4);-ec2)?C;ngGjTQM;W;eE+8=Q~ zc0|9_4SbF#5Q#G}9+w?rbYm7~V-D`cN*}XXVS99Jq(T^qtq!wUVLT?`R7}HVn1g5V z5MIVh=y!yczsmZ8so?~c5QxJpoQmII21b9%Ai=qK2G`*&d;^1W91isvV+03cB96j2 zI0x6^X)MB~$EY~YN5>}CehkHf$C&>y1S%b;<~Rh?a3SVk0T!X-1O;FtbZlmU!BCu! zF_@1@_zkAvw9nZCa2+1S#l;LhEJWYe_{tX}FyUk>T~1*BN$we5#7y)(#XZ9yEXMJ8 z9rvQ&7Cu5@Bv$%@^D!8c@j9-m!~B+S8OxCe9bDjvn!U$bf91oU~6F8_v>qZ8vWHQ-zB zIe|#b!2Y-mr{FPMf>-fv^v$J66oQfkb0}o>! zmSQnB`hk|?J`8?`bnJ|uU*?vh&yU>lR06>Sa$AQ3GbR90mAksgUinmo?RV0IN^B1U#yUN1b{LC`3C$Ewj zLHGU_Okfm$V&uSW{sL7ixu6Lbbw`DG-04{+b*31#yv~xNvvem-ZIyA>pU+c4AWGU?wUtroMx3Q(2SH#nvHVcu4)?i z=jKpHojR18NkzjcT>q&q>K}f)948O|L(Vwqr`acyb>fHel8(J4?`i7DNljh(P19bg zmBFJDRi+vnBNL?_)${iEVge+}v_6ER2}fJuK*=jpKE{Q3*;A$(siktEOf@k6d{n$$ zs$PYH;bpGL4u||Y-UyHwm-00J87}<;joOl=BW3)hbYCM-a+KjIX*%WBa9Qe7jXaVc zE0cGVjoO~{$y7P+;-}ro@|%lVjbBsEXskllbTC5Hur-enE?Bd`XrQ*OdEIDW92;R( z)@yME7Pd7%k>{9HRM12o1|wpmjOj*SdvXI>$z+|>b!2%Zd}#VsWi$@EIGXQB=)v+^ zW-PMqnZ9KioZ4T0(}fO{Y6iqelwmYBGRH_2FWpfUc{CX#^V{leYgBWF~;RFy6*Li63g0N`e%y=WweJ8Qvc!@zDjwpKC|p0 z6iQnAO04x`WTS^sU$vD39`r&fe+NdF)GA5#dDGdUo`$DKizKT{Ye@_-)l__x*5Npmo+C&@rBqmf6C z$F0WIl1wkI7fq~Fx=~B=h^fW0LuXz8xOF|buG5yupSqNJ65vhuZGK$e2b%56XH*&O z&5;xGJQ%kgmpv1xQDS8#%j3F9^$e$FZSQrYU3y%~Nb~rGD~8OX2-1FeT&9H>ftB<; z(7z652VljWY=m!s9$i>F!b z_Hq(lB?Ekokg)V=WiERD{v_&sf8A?RgP(NBO0ucd@{VS<9M>evHD4|pDAg(xt1eMs z{4-rP4(6Wt`a2vC$qb!zXS%GeY}7ZxlclUPL%g^$4VaQF_55i-o*(PUb)7OQSr3ES z;#Y;pm}Hrvh(zkhbgvtPH_BsGsc^1+npbonpQPkvos>Vr9v>+r?Ve#Z z(L0Oqkr~!a@h%{|V}{IUI94bje13*|%-$e!a)ylXGwP}1@|+)KIXTDObRXku{>>AP z8JDPrUaqrhvenASq_1dpr_uG1HzGUN?yFZ%S0>CXSZpG;uP{->4~h)eRqyf$nVHDYaPN zDQmc{CE=u*GQSJq0>V$*<$9G69w{AaP)3T>)nz2tV6Z=7XZ4D%&4ZIpuj7?K@Nwzl zZ-jWyA-KXWz@LFVb?y+gLg#Ft@^!sJ_}eE7h*kJ$CSP{5V-50TkuLIQ zIMkYDtt65*gh5qGibtJUWm0vqQP+D*9X>A4vTxQQ!d>jJ*CoQuq+b9X)!CgC&eqXX z(n(UCWUD?hf0k@-&YfM%kpo@I#Pys}tHKr{KNCrPipz%vGS_a%!-1Uiz4X&ulF49% zt=8Sd#;J3(kgb7?%Ymz<;}cAhWp(vZzNr@dkxQ_TRIbScj$dt$9WN(oJ!AlBMwivr zeDG>gkGBcJzx7bZE0$ned55DsBpnYjf;~xSA=h;FrmL-G&g&>ijih-HdFJVYf;~g{ zL@7^`;E}>*R;93Sla#E>u@16>>)ey%8W{Vmasn$hVQGk`y-!K=T1GSeuIgj8$TUcv ztHq|?N4G%#Rrdp$)p9{&>Pm*F1rny2F5NXVWSr(HS)h4LHfcu60nH}4P@BRwO2s-x z%sGOec(i6P<`Pjb~HeWlyo zg!6jXNflyAid|!0-)j<)-g2DdDq5~-Vx@8@{mb(d7)fjFy(u}As|_T=Q;B2}VQ*Ta zGsLZNub++iz~ByN5l&nqC-jp@zN!X3p^vE!DBg9NPUh`$`H(w4f;ry ze5lEl?=;2o4}|SoTjq+Q!^U+Jz1k#U-b zWq~G0HfbW{fTodL(1eO=2=ygQ6Dr*`^<d zIWnB&P{Ky!df6C8&&TKkkaM$GACQgnsK#aA|BhaS+r901ks01 zWy^X7pZGN)@{RkXNF6E3wns`l5s!oVBtElAGBIY=>?RDrz1jN7sGfI1OCBM!Wei)Y zcM9Qp8)PHT>0Vic-++3yvII)x!)!9kHp(w7(h}do@btPxro%G&VMfqF{<@@| zx?V*QJk-mEhiTK#a)3kJ&8{x(gvKbeQkbm?kNm4h07G~w>2OO_ktsIWmWwcH?m1A|FSNqniF0z3nwO2lj;LP9k znRV7SrqtXDJks%Tyb$hGD)u8&k}0jG7Wj_z+C!~ZpFEO`01(fiYZmJGFM zg45*X7L@VKYwo8Kl9J_XlK3}XZtElS?IYeL9GUZ)wGMjSAUsz#wbV}!Ee#*9Nv}Ov zv4^}mw{v{uvp~JUO6Q zC>Jy-qN3ouglSGlcg;B&r@1Z*G{Le-!wlARl?$2?qFO%Yf=Z^Tgq_?FzRfTvUYTIE=l#Ja%=ABi_%8(lZT8ebI7k`t2=_jQ+GH|<9(AqYX(~(oGazb@%0Qnz=ph?4r{zO1?!GVQ+VXL2l6bWvIwW6SWOtF6 zc3gdCzHHYwV^llJ8zxV*qda4W-RH#}_27PoB>MbIO-K2M64>ABwkLL2dbj814&h%& z(z{i7H~qw%-k#2zR7l7Fk588qNUOceKE9Z+zx>2;R^f^rs7acA#QPFQ=I*lhJpF6` z6q($Cf-`p6Po}Xk*5U0nX(PdB}3wdhFZh{=8e#Lzmg1Gv`&3$H1t!-)cvN=$^bE+wcAyn%9{Q zkCxq?DQd!g>p8${9T}#`Kl+?J&bjwV(CZB8o25$^(l=UJ>v;C!C96EIQ{Ly_6RAY^ zCz7;FKJ3C8#7j7i@aUtuLN5vHO5%L!-IdyV%hO=gJ|xq+8ucpi`07aP%_H*1_Dz~Y zxQTqnc@64+%kLz)cudYyU5n+WB0qoGemkxmu_4uSw@5L%*R%5d$sSwt6ka9 zjmv)^2Xuyadvp|zCn>4%SfaZu?&>R%i-6Bl$p>)@L#y?yM zhh>4LNH*CV(0m{lGzUcW05&t5?wZpwPIE{WX!gmb9@O)ut`ZLvJs33C4_h;RZBLek z+d4B(6gvE)9gZPve0kXZ7@njLoVO38=>x|PTW`}OM<2LsA1EU1QF7Rx98#*YEj=Rj zhVVq%i(K2ps}~PoK@!!A3I)mFUJQ;)NA0C5lN?4d5&b$KPe)E2wcm~v>&OW!5-3-D zF|Iz9KXpMq64sl74C$`%k#W6gV4^JP%{X{R-XYOA{Fzmt_p&&?_5aLjidQb-Q&PDP zhmU<`?X*r(&dJF>Y+ybE_##A}<{*E=os^mtLm48mq;8dW`cVEUovM5MH_bVz*4H9R zb4&(nKIC8IgSX|SzRaA9pOwk~^n14AzVv}1U+QCv<IXRzrpcD>{V40KJWb3v{kb*Ly<_>5T>QDc(&@iy za9WP{V`3~QwigP&{@kkR#qNb-6z$t59s1L8HDp|W3au;)Gx>HZN_Ql$O6qo*`(Pc z2Q&q8L9<=d0IFZ1%&_p!tSsV!Z$gA(dspi_6Eyi11Wu3i5@|L_Unb5ffTg7 zM3$3DiUwMH$|pLPd}%M!euK!c`b!y6jSP`GQt+jhSaC*KX?w(gXQ@myfwSM~+s<0&WhE4_NM^b_&u_*I#_GM~2{(ZhMs z@|FF5CV_C8eTr8K;koj4JcDB~C+e><3UpSfHJBvM*r%ViZ;&*Nv-QJn@?csvRh}D6 zH40>d&UVmU*qlWDk@q{DwDT)_S>I(P)g8iJx+qvJj|ne`pC*hjh1|- z{g9(P|9+=4#eQoQRbkf%UW$Lqr$?Slypm{AbLo=6gVXz@Ss$v?66n8ozqRg3jt&=n zYZWhh6KL~c`8dI7>9P1Mk68>&Q8Gael*ya3S4Ni5Nyz&PkhLMoS}`Q}v3NrZa5( z&b~s9j%0jiSF1=zmVRfim8Cke@H^|xz}n$6Dj-M^VVlUuUL|H$vY&m{T)waJpH1J&`U+N$Dp!AX)4HI%`Ln5h$Q`pK{fQ+ z{+=dCPHKGRH!$v;w_5IXjC?nAn~n9b9Ndi-FIlwy4-l0$#%L)Z18 zr9aq1h5vU+Pc=pY#rivZnM5}c> zHAXTxgEKl4tuc`?4j7j0=e+mm@d}9Xt=y@yHDK-sa3){Jt zxUE=&LH-ItjnQX~jHVS-)%Tq>+}ad;X@XJGwGj)t;4q!tA&cv&KJ#iS(c50giXKp+ zx5~ExFG_5Ik>qu+<5s`NJ_Sw%B=wpuiv*0+ocsLqa|$J)9i?^xKT<`N2na?>3B(@NTM|-VIkI0niwKA0!vi)2?*ecmq%Dsm5 z?9Ju*mBCi^o^rrk=_thN#nP)}nIx`*G|4O_$0ZM9Oo);(RcPfab2wI5&)!^~4@F^G zHcExoq0}wuSVu?ug*PI9T8+JbL1K#}7h!KGIwD&~!0UlGS(d`%xVpNC;l5Rl6k*DJ zxu|5hJk+#>{LLt&MKXTHMEH^9D{7NkB^Ts~lFM@5LbsG$5@#{wV@WAyc#Ilp+AaQK zM#P|zvL5$JK!Gn-mK6b-6fBKZU$M+7#=-r3G~iqmEgUT8HhTax10D~6c0l(4xD9v! zn3|wnWRt@&e|$u~PQu3bu`x~R*JI8lIlbOg>)%Rl$U`NcOKb_mD;XsicSiC{ko<|% zD=C)KN=8eECDNIsG-6{J(c-ZjD5F)l{VXB;gvw}Fw{$2g zPQqkk+$C?I#B5BD;4V*3g)Uf@!s7s~8e*IoFJtr${MAZ5(fqrnd?7cLL`hi5Gm^ZK zBA$7cUKmHkdRaQY2kE$HBa!i_^#2X3-`cQpNbYFbOxTj@jYgWw&DSGJ7U1@RDz4Qj z1;D`Ah_0P-cO$xT&{R%E(`1U0c3A{*-6NYhY$t5hwSEIU(ItjxeF z`~+b>znSQ{jIdiWHc?Ku)XcfJSWg;G!k^0UTD?i3c0n3K6r4F{BaGP z>*>wRPjB;`7ai|1SJ7_JkU=27qVu4L0SYL3Fd*5as1u}pO4>m5pHDVZ!EpI%Gupf; z!?vL4=CsIx|DwT{)%==M7Qu2>NwJdONdqM1QX0QQ{JvAacM7m}JLkW0MWye{R}fdi zg8H4RXJN z2KiLJt-z~|l30nme?LXeZXl#BsU+#2l2?hN`liVCN{(To)l#emei5>hlc)3ijsQ81dM8a~4;G1bJWNtssA zng-yIcX~CW{PpMUc8Uvfg9K1HB&?bK0gqrBZ{D!_ z@zGhCUBk%j^Vzzk?k27dfcb!L0Hz)VcFNds6d45Emb*3dAfF5i(E00SdVmV9X4$?g z4dAXzpkKz~u0}=ovn)@Z0vP>SQdQ3|3<211enw^{w{NwKH)ilDrZ^+nd##S2{Q^v{ zJp!Bt=y3JM?ZIY%X{-Cz|!GzlO#@i!b;L5c_*GbIMXaWUi>>5I)|iiC+)CiRzzMH ziJtmyNdU0;=hUw+n3T(vofKaS@9RliGr}s7VL?i7mW&|9h9pPHbtwxn3GJ67igw6( zC7;VJC7+0M7iILlfQoUH+qsX|IAq=~#%z4H72E>5@b`#p%im23QnM{Bq)kzRBJJW` zATBLZb}^}FxPNOmsV-8@sQ9#^ZtD2}{bjDA;iL^=6(N2_TIw94k^~ej2I1*=`@{m4 zk(158_!?up1r7M#F4hwSl*`@S7|{aHY>0Mvr)}W54R~LET!*mR~mRi(VZqbYyi*~(f(fEH^)bT?zn`3UsPPadM zKk%9RnUeUfQDZnlEeGEO!r*#M5ty>1nkE2obd ZGot;y7WB(Qg@JvBS!s+>%Ail8^bnZ8^bURbIdu1VKf?+<4BUwFp-aJ&c?>L##0(bk|bec zlo&Zx4pXa~8>!w7)ku9*+WY@_UDw;_^Zot5w_Bh4>-D-`*Xw+}uIqih-}YT|uKb#F zN?ZdMZa@W`s z%c~K_pFSh)SAH~xRMK` zvWnPc9QNxOn6%SM7T3{g6|JLjMp)(QD%ogTc{$~LR(YgKG8$BAs-leNtBh1Fa`#tR zUL&y5R_m;SwYANFx2ur%jZsaj2i_QGHM;PBj;=ITaP-1d>uB-+94%)YZdyHP-%9If z$^RVv`LuDbX%ppZghf?vw(~11lkZ#quD;Nf9`HZu-nlPF-Snx{o3djvDtxs<(;*{w zREKP(KFlrc=&kHOF3~$LQXpgZ*ZjF~3I}wa56RN4PSK>pj}4 zmyO__ZB-{@5dXW=$n2S>mKs4Z?d-P~T`9Y)A^4y7odblf< zn>XZ#r&Lt#+OZ=%?6=3{{`JaG50#udX!5x7sz>gj|Jaq9n0sVafn6QVotLt_yjp8i zSrVnn8QqrDE$`OT>dW7Cjj2nnCG4zAm(seQj9J?Kyzb&hR=hAKV<7*Lp7xDS{)FWD z1iyf&Be{^e7-_8VvRWCcyj?5qao?#(==)Cj|=T=2Wk2>shF1VE3 zY2M%VY0;UCK!onXf`ZxgmwL>h74sgbO7oZKT9;OxT(P!|0!JGkuNvuhrhv3GbFm%T z;&eJ18jV)>@n7ac!&}!fM{S1L z|Fl)MrSA5a7g(nrCB}O)THQ2!)r=0Jo;0z*-^g5D z%h;0LTV)$}v#Ye|yuPeEUvJRCPbsFvRm>{~oX*>BR@$SYE3G$LudC^qNoL32#*lSG z+&X7FC3UI_4r~&$v^$rO?rJ>`D)Z3ek*YyzM z%D2GCAz!7nM*gNfEXIj%H)06he7knJ3yrM?-ZYNB9bUnQ60Dw{a2zpyf4fPY2S1t1 zr&1QvqPLS1j={Rfj0L~CrpP8nkIixF4?{L5R9=_Ie7$7OmmaPYj&wu4Q>9(hYTerN zw_96B%wopqtgfkjRF*gSTxr(nos53(gnLdAruvF8^W~da5`{E;*ICt;Rz^>fII=O@W^-(_a($(96UM;mMH}t(vl=>?7>Ncg+x!mISKXS7N?2>|7Dj+xSLl4#9lbw1%4$~`kgd;I& zXJXR6a~#)+^WG@7B?oYzST}dIaeZg&N`trR+~(d} z0egR2Jx2PZaYmor(xu~OmOzQ(KsOF>M?2w=y^oTbkz~$iKV1YDuPev<g3i@^NpUlVJWk8vMY;yht9(0Uv`(=?mBuV&H1d-vW6A!m?Bu8S z*Hl@??fsGJm=W;Ngs{Z9x{R_eeCVqu#9TN%IO5oqYJB%`kn9Xr?#6!-E2wV9xsQIf z*PCM$d>r8EGlv}0bFY4ED7D&{@=194->X=i*y`%UXPp*Bn6K>Kn{tmHdQoxJ zHat94d~d%6OP*1#OV&67YX)RubR7oaqo+vN{oJ|277!s$?5#e$eCJO)^AePjO(WY+=q~) z_m%2K%A9IOqtmf&S0*}T{YF*K$T>aKeshxX?5U2%(`N!a)WmxeM^IXhcjFlH4t^Q*Z+chS7w3UnVoG7c6tP+u61 z!aiPM)OHJu=TOq6<1o#ZdYmkRUYFWWmsX<{$>K7EtY>1rhYMYeG|#bzWPmTyU+VZ^=~`T z{K4NoZTB&ZtZ!SWA;!sX8?^sGFS?MJrEhGFS=x6e+sM9{rPB&@dA-uiP4wz&UH`v! zm)~>)$RQtYXM%njStWPS`Q~oTFAq0!>XWy+7Tj-s3~PKM>1m%j?wFbvVY5+OV%Z~& z0!MXY`53R<^Ov4dYI5$4%Y#(2hC}~PFKgqb|`pV;(`rFUCvPUS>x>ckQ}+UoIiz()S^`-&|j3 zSJ!frf1n9#2XMmDflu~im%hKbo4E z8pV@owQf}pKT$>Cvqp#8ty{D=m*10D z|MuBtn@SknM;CiaicYew!3xuicW>7Z>OJHC@r*F}S$$Dt{@l>0eCI8%)^RjuQ$O)_ zS2c{YcXoTdz+uOkzDCrqjU{A{@-RO7`7N*I{Yd;P_ED}b#=c)BdwtxO#CoyhY|cm1 z7k+K%TX(YAS-l$c0&QZf`?ZUDE%&=$>$~y3!0Wf^>PI8xxB7woVqAk_Y7I&+3R|6f z=r?y|U(rKq)=(V``|r){qk2e24ek^pe&4O)4ZlCCho0(gojj(8);#$=f)0P3KJVx_ zBXv(eZsH%u)YF~1kn-<|+wmJY^PjOwc(z+v30VXk&aP$&i9@>#lU+fot~}d{{x5kD zr*e!-52o8IcQ)extXF%c&c@QKY!+iSU@$?)5nbK9qiu!`mmNn{pfP)qr}6Qh{_1_B z@XvSDNaMA?n%P6<8}I!!famsyf5qB;IvY9DIvHtyXV^P-TA3Uqvv(^`dG%e@*m&__ zrhQ~bf} zS1L{=N)5Z3uD%q*uIh)j(;4a0m!@U`)hW|4muGL?C)v?KMT)1JinE6gkOVjNnp!Pq z-BkU+Qr#rZT&;6jRbL|L=#gK^T`jk(y|TemO;RiJD|)F24||iw^02lFkT@UZ5xAkL zb#CjZvU6t1c_JteuKTBU|c`AVvO(5I2sSwkC_os~n-@l>Sr zsigL(cu~G8$zG?CxpE}iSM9b}i;y@!RV{E?L+jjnjmpk_g`ne`{1iVGXt!VQVx0T4 ze*WPq>J_E>N#m+&wc030tE&2S%hj_^-DqkK(T@Zj{p*rG?`RCGrps{S@Pa_e@lzi8 zO{%HUc6Ccu1gJIYX=xb9SiN1_D&pxnWwoagbllD#AE+8B&u2OA_(TjRlUNWOq zEs3b9!qpntSW^Yc*qW-H>ZZfP)iQC`RCU~sP?1b7uc}LjTIxX9Icu?VNt;h&M|8C9 zr_-K!wA1X8pb(X;j?3~872q|zhIKFW{VGUKhNv11rKS1M6Pxx+Y}!wbuUTEU1AC<% z<6&CIdoI?`Kx|+q9Ov^ZgsPwIYP|eiTh$7>6kv60YG7Hn_7QZ{36NKPRGs`mbyQ>3 z=)Wu$9;sZVx1_EoOMiD>S?O&89zEfC$=Pt#qVfQL%GGz?!^IxD`g>K4)Ok(inYyZl z>Mqmks{7T_s&RV8#BX_C;hul>P^f+=_)F&0Q^D$8Szk{z@tdJbvDSPAayw?om3nG` zIwn!|m48U9Ca%`qb2YG28rfY>*xy%?4fRy5&=aY;k+&Zu=T#{yu~$_}%*)?cU%9#2 z3o4qIO~^y`eMd7to#|7@XQt*#tXP@!*L}^w+1p6vD8D=(GFd(J);)}uU6HC--ISG$ zRg7o(O5~rGU))$l*|}i+qEvPNQ%_mNj?rhjHrD+1QL2Jc&7^lT72`F49F9-QNm#V1 zmj7WhRYTQW;;E|{SQ3*~7^4dGXV=5Up1Qs;Z*HZ_KW!@2TBwiJd-7!qs{NcqM@tEf zR*~*&IMZBcM$`utBsp3IsnY!A(TZ)UoxI;lb?|)XLGG9Ge`%$DQ}#E5^_3~*+cLlr z`NP|)Fn1LrDIL@ia+59`m@w&TpQ$Xq#g{HdR^7GYH)sOSM`o^ z-}MJaczTtC-BeS%*8};rn~Df^@ALquhTjIQnJsZ z+V?mPZg;L$S64qON67r|B)O@Wt zz!An{2&W{+kZZ6j*8};dw;Jr3cQBIEj~?$O^>DCO9fnWSJAIn%5rJ z>4~11LwG`8w2yn1Bwxp>&MHN!^;MJVKlhK>ze@90={;uNu*|4;Op)`9CM?4i9n-#jkftNwcQLthOGRn3pF9&PloU>P z4T}FSdQeVupZ|wBJDY~_@`X{G;LX)n&|oA&#^>U~3epH)>Whj7TX4Dxt|U9`7-HQ_LB*c_N;0U@^bqpM$24LC(Auv)X8%0S@lxzhQF*e(93axhdI3lIt6e*6E<{cGfZyqwLVYH)_DA3Hh2 zQR9|8-Cwn_?~0e%JyoC@E6e+<$dE)H$^N~d4|zTA$SZQOzgk{%(gUj_v5wU|km`<< zU6)C>WOo$^Yt2Am3)VCT+*f|J$3cO#li#0HtL)`}l=OI2yY^N0 z$E`7sFZi*nH9<7Yu|NvrRWJKf{iVi02C6_>4pdD;@>@MohdJZbD2O`Db#_GN4OB7e z6FD_d{aMA04s=Q5o23Pbu2p_o&JR+(p4!XCqK9uhyG!I?meUUDIarPJzoN?4e2D9e zx$@~?RYx6^p9ZV2p7Y7@Xb8;;8+~8a=z<&ncL?LjZw;Zojvkfyhdi60s`+(tqv6&) zBAIg?-^ptU>KXNle3ZcE9eP8)Nl;BftG9Tf!#R2&U$Z^#aP8|7{=6D)_p;~DdtOyn zRhqDO|GUo*B&^_RX5kGA4DRP|FiGH|F$^#7J^#C5UPcXfQ0d^S{t zr~`6yC>@_mhJSUu!FOdHKUn;)j+<>9K>o)ae^lmu=|7B)cVJU?b$klvI&$en*#^QT4TF!q~uCCXvL;>;xU7=j8M~!^@P;De5Wtc^DIyB;Lc-uc}=BgW){y z+IL@+&Tpxv`7exA3Cgpk&IpQc!U_`WjIgz zNdAi|ywYbyoYMW_yf+>?r#hBtS9(~s!a&Vlg8@{VOjh^t8IUgNboAa@rB<3ST z$2j@JbJl$LCiCg`Y!d(T_e$kceqMq3Tsokn)EKobPzJrMesB*w$54jt36kf>tEW|z ztQxO|s0&guUe#`zebri;S^BMh`HxwE&YSIN_3lF`@j zi+nMh%Ec9Gu3Pi)CX*)sn@f-7a zV;{(fI0|Id1l6hBjJ8%sZ^@Mjs!`xUeV226zcj-Nr|NN7EnyQ?!-_w@XnXusWwAUz zQ8lmL;1tW^)0^Y;&3@^E&aR8=E7>$r)lZq%#wz}I?hJgT7o(3p{z~tdt|9Houk=pq ze6IQzS>YUA|2yo(=KEn+1?DJ}exYk7yO||{>^BdauMo>xp=L?1NvgFvDsv{O8Eia% zO;XWz&odJBs;XbLaSIxkv5Gz2xpV;^%=HEFo=kXEg;w2n`q7ZSRCZ{yY&5?g?C~Y5ST}W2P-OtHC=ct1lkh1N;fwNy(sfpl<~Ccgjx5YD1CW&^Zgb# z1zycJ)K#Q?dWgBPIey@Ot>h}7>Ey#pyWh}XI_vjts<6^tTcy1P`uJJLGcsigpEi2R z!6|%VIVyfr`3&-_w94!iAFcn zF&h2RakJ6+GTk3vEhuNb{57YtNDfa^PnSDC+}bgAipOi} zxpLXAXuOPiO%1cxKQ4z}V}tyUTzpM6Rr93$bXB14$QRSqCeP_d8Q|+Ob%tuLev&OS z)O*oM2w)HhVCphll?X)fTg)u|r+_`q6(pUbCj@KGyLUY^Y(?Dw*5wyNjl ze~`wc9FW(Csb=Ay)MKQ2az4`=5N3|lHj)^rze25%TJo_qQrrWoDOD@8+l)ez)q->>5?{V6a>=I$V^e z=Bh#JHyJ&b*AD-c)pHrUdva{98d@cbdyDyP(Sn_&`T^jCV}5>%d5T~2tdLgoRYbXt zan_*rkeBAGA5|ymw?HMTR`THjHnlAIbAf8@mA98xh3t{m3sq}-tx!o?$SBN{Hy5f= zRX+MsH=b{8${u^RNrgpft7#`ojPf<%wNnqv$i$q@pvDz$85jWm#e4r&E#>`X;QdYEi2ctkJX=k z@=}_r#nM@rrk+muLRa}XZ!$IA9{$7z@jl00b5d6Kvu1P5?c~O8-LkUn-jyTo&;+ReJ% zzf{%bjo5cfRh_EWnLh5Ey2!FSXGf`?&W74io=vBx(`8aRZ%7(R!wl8Y9v&#;Gt{{9 zy?feh_bQDz?D#YPa)xrVt7%faT-B+WR?T&_CwrMU>f1AD@ILFuM#m&HQ+4M@RSbIb zatvoiU%fN&gQ%*FWM(E0sQI!!Q#Iqsw=h$MvGDF?@?6gAlNIa%&r9qI)z-hM-hlqK zE<%n=#YMai+_^&aR3FHL6{2I0i;t0>ynyc$&nU$lyvU{wR{u@vq|l>SJ=2pkHL$MM{kaKE51%ukRRTm+6;-=!c6s*nOoFzY_Dgws1!G^ ztWrZI2b$j#o9mGmZfMXC6?t02C>=_0GQsmY#KR*)Q%U*UZf?bfW$QtJ8l zw*~=!^J?v_4692mmX&pB20_R7`BQeNH|_RbOXTbpN-a28F~8+5m9N|$EGm`5wO9+D zx$G3DR@@kZjWc?zNB-r_3ka+-QvoH$%lIUv+Wwn(u1b zOZlorm5=7=5+A?c9Wqbfv6|#x%2)3xH}9NMSrN?#Vf`-L9y}-i^Zm-x&aQdxV^yn$ zN2*opI#aXO>jWKrQ%WVtN7c@cJfPg|HN*d-tFYdD9xisXUVXajwf)g7D%)Mv@ZD2E zUOmV+2-C##5U)>Hzd@{SXDN|4K2rhhpPt4|-TZk?6PondW$LfUk~;Xa zzQh?C=d0%PcqQ{8-pi_Z{PFI$l_R)7&V8o3)_D4GsWbJB+qPtX3W(JOtT3192}e(9 z^0^93`QZ|2LGHS|!!c=J=@-vwXW0N9Ny(4uewra@%Ws(};n#1-Ju}H1mG?PQH{xk4 zT3hEJI*h2RZg9!^JR3%))oEDYbBDrU3neL`qZ{;eT@Q9OhU#Oa$OZ4u~X$P$XJ|{dvFNWHf1KaH}-0_Ql=A^@l3pBZy=h1ne z+#%6zTIf=bX{612U{8K(smFNv_>>BJDuxt(Dp+Z$M@RYTl&aa`rnlA1t&9TgFL&HL zN!{jmSdZEmYPIn*SI*gGIr~VL)2cz`&tLhUrrecTr+LKAlZ~fU-Ck$(MUloy++T)# z$yjEz{Fo$eMD2BqAv$IDr9(@dvAH?_RhwBEYwL^|9Zr+z*dmqAs1{)b%pKdx0|qZb zpRDK76_3&J9C_gkuMyhI@-wOvA8*c{QFX%BE-A}op0wm~+*_YCQ>vU*9a~ijNu?%R zm%&{I^-BA;%fK%2u^Dy8ei3RL7?aU>2R_#&?Vnz0f5bNUA!gnk@0i2?xH-IDmgQ&F zaNj6B3TgM!N)PEdzS$-J;aT?QHrM&+QuTPLvmO8S<3AnO|Ft;zIIQD6iBI6avwWs$ zVr$Yw!irS2{Cns4;1c~ww$oX6tSZwx$H)`HA&x zOvZhfg%2_2GgKJ7)iyaF&59`1iXO{urK$( zfRqL{+av;Qa4{xe#5}fG?1niw0gvKryn$o5z4|n?**?NZ?9A;q9*5x!%)xBjfd{Y{ zFYDvnn;&8-1~;dTEL?j=&}olF-hrBnu01A9h{F+~Ej(fHN_m ziOu#ow#V~04DHKlIQn8X?!f~56pQg8+M3#IF`1>#P&~4d>k*4qGKb>`bTdkwi*Y7y z#l?6Mv(bUK(3ckv?pd@P2VorMU=rqGCKlpOyn}^UjYr55Y=yqfY_;E=^Y!X^*rscQ@i*YyFTG?zrVJND1N}ZkYDNMxP zn2LS3aFyUVEWqaPvRJVny0>Qkz;IlI-EcLI!5g@^HP?Udt;{(I!|)`|MhCu!-fdVU z7=gcF42HhP&|wrV#@Dwoa+rmua4+7$V)SXt;(VWx!$^$99ykG~V|ogKbp*EK7kCOU z;2o^9odVj~Y%MVo`(rGo;{;rX>39;iV-cRhKkyFve!%%pv!h}pCSokEzzO&rrlb@2 zlfZWDwSxk14Bo*h=+oY2+li5Q4P&tcCt&3dX#mFIb{vDJa0TAM4d~N>%M>H=CdOj5 zot%&LcXIuw6PQZEPMm{yAR4=1JiduD@FuRor*^Yn;0t&eC*wm* z!C?0Ko!A}^;xPObXW@X{6c!(Ww{ux+&#>C}&{F&gV=*J2Ro#un$g8Bwc=!_*Rd+_< z5Dmc^CnyN_VK$!mjKzyjpJnazV4pb0T3XCw9eSs6@Bfm9QC}qt?#voWA<&*c1`b30 z+GiG)Vh;L$#j3_uSd58i>%!DvC}v=1+=q#H98>W+=3prnU{Dd~V=J_Ewb`~~D5jhw z(3!vuOhnt)6oA2)gKe+?cVIDoj>(M79n8W(ya>y~=~#pd@ID&o&xq~8XgrJYcmh}G1J__F?!$mf+*^94PyvAlBqU%!4F4tr+u(9cz->4KKgTuLhA*V{;cz^U9ET1}$BM;VW*Cj9 zupi#Rx#-)6jR&Lf7RI6fHF}QG*SR_2INXUlu@DbpN(q4y0=}_4!+pWs+3Vr+9Y~DX|9=6ALOu}Sbi&=Q|C)Nnw!Xm8gV2xmugX`bFAJ;7j z(U^hpn2pI;h*@|G^U(b!1z+d-(Og)I0m=k zHav-6q66=s_W+yiuM`3i1blv_0F1_QxERy%B5ucJCCoW4yvv;9X7qWE4eK6rj%aMbX*mghZ#KFq=MSb**RV#@Fr^iE)L|IPk@n=v+l>;D#k2_#g0 z$dF?sZpU~$g){LEW}(mXTxJ-FM==%~{lj9zc9?~!n1`3K2!Fx**s7EvAHwqmMq?(% zrw}+#AQ?lQ6o5T24->EmzvuS{_wgS34`r_TbwD)6V?5?yG8SSM+WARB9(KecjKll5 z9Rr53rmzj($CLyDGx&i=D(=M`yp07|nV()1;}-NDZnHJumoZT|8sl&WCSe4>GFpR6 za37vT{l(Tje1N4GFoMT#zD;U_>oEa8_Hm{-XArnU!Wyi`FSYhzZ9I>y>~VaRzR}HQ0*Zj}_p4ev#(DztB68A?G)DQMei7@CYX1YgPC{B$L1ae)_i) z(Nc^rZda4iOmqQ(5!vkgw+XQYX^22-&|b*FPH1_d~sh1eNO@D~gi&3XJNwms?} z#}32$I17&lIi1;9oI>CLf#4cW=VeUChqwiUU*PVB?eQTdV)BwSp=3z<=0!0LR;eAX%{};KOFdD1XrsX)G4!^3wowyax z;7R-_-08f7_PS1|&satTBXK6i;$obDg_w@v^%!aFjfG>m{&x~6A>j!6zQo3YQRvR^ zP2#XQCSd|*Vj<>XDHdT&1P#Y64H)uqJdiYUIy>XaNP3KC8gm{xa3|WEP%$>gl5r{Q z(FA;7re_$1-(Vcxz$6?Q#bt$)F%L7a2v6dDypI0knIeqF;AUKw*cy{@IA-Be%)>2M zgvarI3V~|`{9j>?no|Ha#CYt6$v6hHFa`7Q9W25kypQ+Me*%k;9~nhs1jb`mOvd%N z2KV7UyoTrTS4??8;2#136L}twW|d=0Ou$Jv183nH?8h&-_hA;E$4Gw0V{tq%KSuF2*9G3XoJZ(1hX(1^KdQ};WE6BYtjETMh2ttGREUAOvVv0oR2dw4_9IlR^yku4{q-TSauNO%<&;~TgYZ{jKZFW$k*vGjZf`v68_2_|6WzARF-_hU=NSMek+ zK?j~f@7G!67=cecOT{<_C%n$}KZQU#30H9^Zi-_o#uC zTj6$0#8Ws4@8B}@`47*27>Oq_7NehIU%}CsnL=O|ft|P-3-JJ!pgo?AX%<6{?ePE( z!}kZWMsPpo;7KgN%UFyNgSa~0;PS&z%*4*P7ZY*wU@FGln6i^V#Pd{)?Xd*=qVH^O zpV$Vg4`HY=24~r%cMw4KhOA9d+i?B0>jHUpL!BiZFId}w*Vj;}|@T^YIpL#_BIp0CvF}I27F% z@ZlB1aXEIwZ8!!?$1?Od_9Yf8HXp}ji7C$zxKCgf`Y)uV7>zqI9;?61&|@=PgR!^| z`;F%+!Jt>Dco8Fr;kX&Q;cgs*KjM7!oIt}d5|83Oys?Pu{}ciD#k@+H$fCi97=yEM z9A3o5=)kQwViJoEU&qKao^Xa9hha9(qQGK2i4W0%0gG6q6da8a7>6-<9LM2xT#Ti7 zpK^m1GqUOI6Bv!-F#+e}j1&Ty1lC|S?nBQsh8FALL!5-cG$<8kVIdyClqC!yu0Z!? zyd%bN3{PjH!5%mP6EPj9;&$AMr|@G;xkKP<0zS)m?#D=U&!7OTffKL`rsEyV!|lsx zAYR9Z_&Ww?^4Pwd5yTiAhLdp??!at3iU;s2UdA_8aQ#0dFnuK_tmL}IS@;cR)3ZM? zcm*G)4GPBQI1J-(7EZ)$T!aU36JEwY@FCX9;(W@B#P-+`vvDdOz=YLY|2GKeZ_s=U zh8iPrKE`IzQk;No-eLrB9d5^V+3c7&9ZPV+Iu_R|-gaReevYw}a~UV#FPM%GaXU6z z&lKR06ase$96%omx`B~cg1!{w#AIBPL(8$nCN3ZB_%;p4FVKB8YiBEqO88ifGw?5r zCO!N;t`6*t$(V#$xCHYsWe0&G0tI*S@_3%r3B(T9S^9%ReLxtNOUF$cfIqxb-CVC6$};4QBIXaeDH zahJkwI0?t$ZcN7l+>ZCL5F-kha~y`g**vOY6mB`p++*`2+~08kX5w1hi9cc?*8Y_9 zu`BwftmAV$fhe4bahQ3O)s5Mhja%^mRzAkf3Ol21Jq^N8Y4C$Nx&B)o;0=>H{ihM`!9 zG`49vv&xD&TvAs)dJyo$c>F?CnD zd~q}mdyngX7J*qLY{G2Zi3c$JJ9>^Ai)r~b=Ik0h$J!W!V{ja<#>MyzZp9hb={YV! z2Y!V<@7ruYV+0j_vW=ADMFW{fQ}0ArMX=2m50Ij>KZzhweML9Xr^dupxHC zfSc^)SP!%Cty^5C_#PIa?KWFE`l0`aynw(qI1>|aJI=rYOj$$VB7uE)8_(lk_yAko z;X2>R;=wkUj|q4IXJGT6nKJB*2k>pYjGy2`JdeSIE)- zhF8gq^P~R@CNPZulH~56{A;Vx;`bNCOEYk5*sWChjJF4RTqef^nXC_8kQHEW*j@T{ zvIlvzSM0TNP9KSqyP5^!_cx?UGtFe_uQ?-=HQ&k#%~!HZV;8@Nsz%*mb}q%|`QnL6 zY+Wk)F0?;W!Sg>pUb>mOd9yg)US|zagQS@zM*3^o%4E%dWQFFC?9wcibDA@9_n~T7 zt?t{Qwi-1kE0cDHQ<(m*t2FyZ)#o3dJpT_lXUXfDU*t`l__G|=v5Rs|(??V(^p!Bp zIO(F9C!E~t-^7QxMfj&X5*0WcWf>PyeALEJ?>xf@}xmBv_@wY1coT`q! zZoIT|s#;z*2A4Wt<_RU_1$$LVa`L0k#tAaBnmtG|b)-px%=V$YLS=WCojN5VfxjiD z>bd1SUn<`v+k@O0lx+EjGF8s1FnfqvxvGo3zKUD*GU0`*(g@#JwaZ>dU0(IIy^j6i z5VNr!^S!uHNT4FqLuso?kmtc|*w9kxG=r&jXV^!}o21z%4J~VhH=Vzr?DfMckKj8J z#;|On?DNc-(i4@=sWT_!u+jnwU~Wp_6Xs`r45_h>@rC!6rZS_vr6jcugt(C0K9 zDQDd5wfPsWcPY{Cn<&-XNgkajzj!jxaqf0^`-nu@(SS%25q@tv&7F)36Rn92e~Nop zqSWd^%exVtZKhX~5_h||eL|viY-F$HxtIe#TPJvIC45Obc+iXcG92uoqhz*+y`Fu_ z3v$tmY6?8;?rvjVD3w>z?155Dj9(M}pmToyf?27AdXlzJ>UvVhcIocP=zaNuo`xWg z2R>{=lA({3$_HSN7$djF(7J>QTq$F8w`$n~OoKevkmf%|ns|{`YmAgOW;w;ZV0ZUC z&jAu@Rp-xB>Z>7@AqyQN%e-i3y|J>_iy@1V3tp7pZme0e=lqI1YL3-q1$k^H+)>(k zb9kmJ=>|#j<#m$yn=fzbBPIMZM#k8yTO~euy<|_$|z@L!0Kz6sDlg#E+6<;5Yy()n|oV;O@xo*5pR_4yso%XDcAWzgncC8d9voIr2kAx_oT1wOF#M za(kX^mhYKI_{d~);dmAiKIKZkPxuUBeQEhuVS2etkdLS}9)1q}?Q< zJL#E4obq=50hmF3(d2LaHT0S-j~(R5?X@lA9>H0!3daQcHA?*I#2ymtU8kk z`%kg<>`*_Z%_b}S?bW3-VY_{b)sZ4HJ#^J|pYW~8GU7Si-~he`m?GQ!sH^4_`CQw_ zI>#fKW8I}(W$K;gDkhJlB-+5e!lQ`r2zi;q+;-A*t_*T93Lepc+~}-E6j!D}v*o7F zl`Z9}kSlWveFaN=;So>J zkcs~G5YHI|KemdgyN2+4Q~v*V)O{TPT$d1{KGtW~;h)VhRw3#K*+tANa4O${b0hUZp-{Z?Ab^(5hhmFBkX z;UL^w-l$GV-B+3yi$_E)-ZMzEAO<#uU`M$UU=P-RN4MQdbL;b1Ot_WQ4B(vRuB4MB zHId;YvDHvouzLvGudE}%ZPv&Q zWO$&x5xZ_0NBQSt?*($-@25MXx465Swc-~9Z|Sy%s5I%XnJ$wxugeO}RQ@CxjFNMj z7v!$yocPtCurtyY8t<7=>f|BP=Kk5_$9q<6hFiM(JP)^8c60LzrBVFNR@;yKSt*uD zmh936h+L>aasKkBPTVFngNgkp9fCR5NM6*j;kt=6Bqx}qGJGlspRD-xw6Z47IVjMI^JhWXSYNvAO59H2@%y$ey_y@&LD$6HHv$c{SCv~fp}1KmpD>*$Ql^JxO9~_Lde{6mDvK1 zhi2xX9CWE9xPSs5fJl)46)5%b~2T?$Cy z-^v~%ms|X*NBeAjE;v`jGmA*#Dk;^^9qk#iA<`?1lE=%$FzT5t8Jd;yfhJo{Tl}oq zDHUqdy7Q~e*45p|Qa!%9)Y*(-FMDR3VBPGP1zTcWc7K^zo5Blag{DOI)Micd1Wq~X zs_@j@NS&0%btv`p8eOW@3y(xP&X!b%b56+$4)g4|OLJe&X-eg;##{Wtp`tX?)RX=i zf0?YQFDo?7WS1sX&S|3MuBM6j)rAOYrfDest=MEuJz1e?A-gnTa;`3oXs#=&A-?q( z+%{G?N{2h@@@hyNVSAgkawd%VPckEKS-W?7J$9uySyhjzZz5gl>v?X>wkD$pdX_Zc zwR*Oc*5m0VR>JC2;SilEga-jKgi!CC`cywdAFCk+^{HlIwzaB@btE-gnndyx5W(e@ zmt86;$y{Ec#OxQd>8vByUFRg~$mMKv`Xp6HKF_wEUUPKhfO!s2WD(qQj%Mqp zs#>1$O?hpTEz9(+IGJ$0b#jJ>e2*-`A4!7-Ox?=$`U~M`p4`>3V)1JTx1^coy!6)`m&uwuS<#T8`%UMp;n|I) zcAesMWhD)%%zMKVl`ZCI_YF_1qOBxEONT~OwO2+pqPX2MN3%mVl4$>CgEZmREya!K zygf&11v8Y9OlzrlL{e+CG>#>TqU$>TwDo9zS1&_^!H2l~iK>r{_odV5rU z)~EiZPpu(QjVZawM(M{SNt}+1-e^5)CFw}=Mrp_Mt7H;!`*0&y>n!GPCozxp{80TR z>C(dPFQ*&ZD{--0X-tj2@{dK$CiJSGbkXqVCYn$gBkpYve%o9T9{w%$ZgRINqZUo@ zwYSaX;2BSN_S@D&P%_~euJkOzsid=kdFByb=*m-MrRyh3kNbq@%DJXIOyxASdwV3k z{bUoL@@U`6X8TYd4^iuN%^~{l7eoP1U{Ngm10266{dG!^OxASO2lQ3BOEXN)Y39pa z%@XnB4`1>u)lBn~^w)eNlQloe3e5xAr3sgF8m?_k5AkabqotW9Uivp@rqkawAF?E~ zIitSzUETjjpTJ4sezvDMH;j_4vX<%a8rPOrpzpb!>=zTMBQ;x4SVQU1g1KESFM|E< zHuICOM?gD%|0Eyi46WoeN8A>FV7*%r-)O4)`~$O8iP90r2j(M)#OcVr56WIGxJwc# zZdW-|Kl|`E7ipHatcqrNb=;vpCb{M zhpV(W9h*SxSzUIL&b`G|Xr_+5?J9Jqj^w!tEz~C+%9vukGQ> zm`swItr(2Yg#Sk+`r>_&m|KtC)*Z(qpo4z$`amD*EvGeCz;?D2#i!-_sP`Nt$W;N`Fmz znXG9jE1o9HZrQInF4vx>g|>s%=jq_~j9#hKZO;?H@q@0<(?wl%!7sPx%d<7sI2d3uT_qR zUt2~>`NpyB>dfUJ`wLD(oXtzGRv9>ZZkNfo9h|SgLygjc!)jP zkT5|`ccQ{i<*q(ukN9RA-ciooVcrpu>QQc@$sLbic`nq%SiT$NOpL71C^%z~OBNtDaYhrbG z!rz$TY7*R?<@BZ0?M`97(!V=}m6ypHUs=(eekRJE?ks>ka)m_unJ=t|;>EH2nm}sy zAnD>4<|XBMlBA+9tQ)X{@QNyD0{3u;4Mrn#zb)a1(Ho~-{{UzE!Gyvti1!-$j@j~EW6OWhdy^11ZavF~NF z=Ao?6+>%|I8*&cZY-g;|^_cZ6-`R>^FAi;&W}2z_p&##-#9Dl2eYy8LHp`i zYjcVrd|s~TOjqTfUiSL-w8F>hSvvP-_NEuQwv1tP@wmL)n_;XWD|&NIRoMl0_b;Cq zmtp<+a!u~)!+WmMSsmLnXI`XD>Q|&OLIWZX%5O=%|7vqg)P!dvr_tNX3J!Y z6`EAprI{q>G)LsF=79M1g}u_OFFmRJrQQjvNm5^yMn`$QFGG;}rBsY(?G^N8?R@=} zwUCQ-9_5+xqiVnibF2C`IPA;F(9*t;kv@MTAQ^iKXl@i#~@-T0d&Y zEYd5hn)TkIP#3haNIuf*y`&%Q7$JY^TpPZ&_V3Qml40%F<|`pd)R6;UTX)@59Xb8= z<3>piDQ+Ks&EBJ5fQ99p+obgarLS{xj#e9VN}&V@Wzl*xdK`5jiS-2QYcJ zWa0qb@bhKE=*_na`v=gqI&y^^cE1bOWGsH3%kF}duR^1?6231To}++J84hk;FSyF~ z9>T>b>-3?1^2u}drgoo;R!?Sd*CkMYw3HN$g;zwpa!ZW?1Hb zXAPM52>qt1%MiAe2!gzK*TEQqW90Dj1ji9fwK6OwIMd3omEbE9rZb!*IK<5B=^*&p zchYVwGwMB(B`50=n2CJ(1ndRh>31(d9^*JtApaz=BL2`t=)I(irmBq6RFXLwn}q46 zL?-ITk;6Kryj;_SiW&m``ndl3u**>1p^@vMZkqlIZ;qzCY}ACxVNI-D({vOy6#7V* zW}rSbL_I5`G`(bw<{8&Ic_=*eG^ghZLF z86qn*{1NP7^htPhR_MBFO=P%5_ZBrlGn?Cz2_k`}%ryEFlEK_43>e~#qR_Wj3o zS|VlmxFU%}Bs|d`EU7xwO^0%H2HQU}B7m!`Kt~3-au@5!05eie9wxFZ)5K#GWqaM0 z=A-OQ{=cTqJgBSdisSI!ZvjOMb&0qy#6%;4N?ocL?Jybbj7t({)U-xXTnIr>5kXNv zF#?HETkF>)TARUC6BjaL#xWgZI<;0iE@(|_rY^+c51EM#w$xE7($9JCM<#!r_c{06 zd(S<0dE0%zA9wMYAa{jZO4A=iq9c-W#H z3%UO2@YJ6dbqlE`@bm{WuE{h_|1(Wb#V?}Wx72MBSq47+{^eMVJMbxYbaw67kZsT0 zCTBUl!hyF~Y##e}Nx>vwkWMe6%BT9;#7I3c5vRl~0`_amV=uuoO_Zn{rBm1H&Ers7 z4zD*_Jr9GkIKHGttlYD$k=w|f=x%7pVy4VrB4#&G{ZZMdm`qhBYIVd6eovWp&tow- zrpCq8ovMT-hzlCG1iL$$i*T+z^G*=qKrYt~&mxz4BH13mw{XgZ<9Y+}G~bRwLDuy5Ur^-oomO&bbz#KbaPM7VEsh*VLKZ|9oWw@8|;Atsh+ zD#GQLqdkv;%~;f`3Z0W0(sY~%rLAClm46(b#c@rDrldRFcIDtnqhfQIPYactLusX& zhH!7j61rDt(qqiiwAFO;V%oHHJ}Zp{V3V%p;D588m{_U-ZnQuEOTp;W) zx}Z-@T+uZXH}ue_gk0#lMwmFKS94j)`gUp4Kf}!Z?YU0CiSMEVYnTJI1``tWehyFv zL@(A?W*&OJP2Yv+gNg$7O0`1`@GL2kat>%|hX&vgFuwFCr`iLH1a5o9HrCC zX~}hcv)uD2IuB8=0VeKhvWXulGmmzCt{r(;{z9istkyRsk`$ZonG7*;NK;L4DQjYo zs`8yt+y|Y3oR&elQ@|bfD!frEf70QGaH^C67(BX$H+Gu5f?%7@hHKEi%ZO1qx{m1D zcmWj^C`wXZAD&};xq@gs)1Oz+e|^oz=XR~6N8=Q`5>0OvaTQ!J9t=RMYB&q`UCs$F&I;7FZTAE!RzSyjY2mC-5a`H#B;3k=u*OT zM(<(hJmpw6DRK^x8 z!0T9c5Cwv}(M(bDS{z)dX(o;;7vUbk%vv1mQEo%(n4ffcE#AyY@jX=hI^y-Fx~)Um zfYMv3FJONM_`_H--v?T>ew|nDeuVo`DvKPCZ$lgTLXP9RoVNoUZ&Rx!??>gWT8GiS zr}PqXuGPE}YH&vSt|IxP;e*`+UyIvX*PRBzM+GGBJB2Yw7#L)70W2|AGuO@f?} zBmFgL0l@R##p{XPu2Gs;#8cO<6>NJNcfKI>Reu2=*abxCymCvKuz{aM+)H|gzf1b#?lLZ){;d3gE@B88{8tTn%@vr2T;3@N(9go$XW)G1D6HQhaQe!1vwwL{k`9>th7&VDYxXw9i>e)Jh3%6*{kyCawvChe=qY z^a?6Es>}*%xT$gzcl5goPD3B)no+eLnz*Zk&4|kyfp7y8D5)!@-+Gr%N3>!yKJ-E7 zEqAq>32slw>x(+0VUT&FCZqn4#YAjz^et!DpiedGNScLSGmTotQ|4?a->5rj6$ zC|6?@4v$f%YQz+cs75(ZuNo~k%U;@EjZZ!G8%xR7XQa4OGyK7Gs;Sy}#fh68Y;&VN zi=OLe}+v+Uoq*^(tI=t73&ustC`A>cs&4mJI+sMDn|0>_-_WzZSE%mwmpX480 z_Bm>zmn|y3&ga_5pT*ZjWQ-E`Ii*hB)l+--Id8^BZ_|?+C#6H(Ro}esszx4iMt7)> paQ#)SgkzKb?x!c|(?iaf*wiOJrs|>L%y%8H)|sb9eB>;3{tvBq1rq=O diff --git a/test/local-cluster/cluster-create.sh b/test/local-cluster/cluster-create.sh index 4ac5d43a..40d150c0 100755 --- a/test/local-cluster/cluster-create.sh +++ b/test/local-cluster/cluster-create.sh @@ -117,14 +117,16 @@ do mkdir -p ./node$i/state/seed > /dev/null 2>&1 - # Load credit balance for user for appbill testing purposes. pushd ./node$i/state/seed/ > /dev/null 2>&1 + + # Load credit balance for user for appbill testing purposes. >appbill.table ../../../../bin/appbill --credit "705bf26354ee4c63c0e5d5d883c07cefc3196d049bd3825f827eb3bc23ead035" 10000 - popd > /dev/null 2>&1 # Copy any more initial state files for testing. - #cp ~/my_big_file ~/hpcore/hpcluster/node$i/state/seed/ + # cp ~/my_big_file . + + popd > /dev/null 2>&1 done