diff --git a/dependencies/user-cgcreate.sh b/dependencies/user-cgcreate.sh index 8d798f8..a539a3d 100755 --- a/dependencies/user-cgcreate.sh +++ b/dependencies/user-cgcreate.sh @@ -23,6 +23,7 @@ fi # Read config values max_mem_kbytes=$(jq '.system.max_mem_kbytes' $saconfig) +max_swap_kbytes=$(jq '.system.max_swap_kbytes' $saconfig) max_cpu_us=$(jq '.system.max_cpu_us' $saconfig) max_instance_count=$(jq '.system.max_instance_count' $saconfig) @@ -33,6 +34,11 @@ if [ "$max_mem_kbytes" != "" ] && [ ! ${#max_mem_kbytes} -eq 0 ] && [ "$max_mem_ ! instance_mem_kbytes=$(expr $max_mem_kbytes / $max_instance_count) && echo "Max memory limit calculation error." && exit 1 fi +instance_swap_kbytes=0 +if [ "$max_swap_kbytes" != "" ] && [ ! ${#max_swap_kbytes} -eq 0 ] && [ "$max_swap_kbytes" -gt 0 ]; then + ! instance_swap_kbytes=$(expr $instance_mem_kbytes + $max_swap_kbytes / $max_instance_count) && echo "Max swap memory limit calculation error." && exit 1 +fi + instance_cpu_us=0 if [ "$max_cpu_us" != "" ] && [ ! ${#max_cpu_us} -eq 0 ] && [ "$max_cpu_us" -gt 0 ]; then ! instance_cpu_us=$(expr $max_cpu_us / $max_instance_count) && echo "Max cpu limit calculation error." && exit 1 @@ -61,7 +67,7 @@ for user in "${validusers[@]}"; do if [ $instance_mem_kbytes -gt 0 ] && ! (cgcreate -g memory:$user$cgroupsuffix && echo "${instance_mem_kbytes}K" > /sys/fs/cgroup/memory/$user$cgroupsuffix/memory.limit_in_bytes && - echo "${instance_mem_kbytes}K" > /sys/fs/cgroup/memory/$user$cgroupsuffix/memory.memsw.limit_in_bytes); then + echo "${instance_swap_kbytes}K" > /sys/fs/cgroup/memory/$user$cgroupsuffix/memory.memsw.limit_in_bytes); then echo "Memory cgroup creation for $user failed." has_err=1 fi diff --git a/dependencies/user-install.sh b/dependencies/user-install.sh index 5b0a0e0..4cfe2b3 100755 --- a/dependencies/user-install.sh +++ b/dependencies/user-install.sh @@ -5,12 +5,13 @@ # Check for user cpu and memory quotas. cpu=$1 memory=$2 -disk=$3 -contract_dir=$4 -contract_uid=$5 -contract_gid=$6 -if [ -z "$cpu" ] || [ -z "$memory" ] || [ -z "$disk" ] || [ -z "$contract_dir" ] || [ -z "$contract_uid" ] || [ -z "$contract_gid" ]; then - echo "Expected: user-install.sh " +swapmem=$3 +disk=$4 +contract_dir=$5 +contract_uid=$6 +contract_gid=$7 +if [ -z "$cpu" ] || [ -z "$memory" ] || [ -z "$swapmem" ] || [ -z "$disk" ] || [ -z "$contract_dir" ] || [ -z "$contract_uid" ] || [ -z "$contract_gid" ]; then + echo "Expected: user-install.sh " echo "INVALID_PARAMS,INST_ERR" && exit 1 fi @@ -69,7 +70,7 @@ dockerd_socket="unix://$user_runtime_dir/docker.sock" echo "$cpu" >/sys/fs/cgroup/cpu/$user$cgroupsuffix/cpu.cfs_quota_us) && rollback "CGROUP_CPU_CREAT" ! (cgcreate -g memory:$user$cgroupsuffix && echo "${memory}K" >/sys/fs/cgroup/memory/$user$cgroupsuffix/memory.limit_in_bytes && - echo "${memory}K" >/sys/fs/cgroup/memory/$user$cgroupsuffix/memory.memsw.limit_in_bytes) && rollback "CGROUP_MEM_CREAT" + echo "${swapmem}K" >/sys/fs/cgroup/memory/$user$cgroupsuffix/memory.memsw.limit_in_bytes) && rollback "CGROUP_MEM_CREAT" # Adding disk quota to the group. setquota -g -F vfsv0 "$user" "$disk" "$disk" 0 0 / diff --git a/examples/client/lib/hp-client-lib.js b/examples/client/lib/hp-client-lib.js index e894d09..73daeb6 100644 --- a/examples/client/lib/hp-client-lib.js +++ b/examples/client/lib/hp-client-lib.js @@ -1,10 +1,11 @@ /** * Hot Pocket javascript client library (for NodeJs and Browser) + * Version 0.5.0 * NodeJs: const HotPocket = require("./hp-client-lib") * Browser: window.HotPocket */ -(() => { + (() => { // Whether we are in Browser or NodeJs. const isBrowser = !(typeof window === 'undefined'); @@ -21,7 +22,7 @@ TextDecoder = util.TextDecoder; } - const supportedHpVersion = "0.5.0"; + const supportedHpVersion = "0.5."; const serverChallengeSize = 16; const outputValidationPassThreshold = 0.8; const connectionCheckIntervalMs = 1000; @@ -51,14 +52,16 @@ contractReadResponse: "contract_read_response", connectionChange: "connection_change", unlChange: "unl_change", - ledgerEvent: "ledger_event" + ledgerEvent: "ledger_event", + healthEvent: "health_event" } Object.freeze(events); /*--- Included in public interface. ---*/ const notificationChannels = { unlChange: "unl_change", - ledgerEvent: "ledger_event" + ledgerEvent: "ledger_event", + healthEvent: "health_event" } Object.freeze(notificationChannels); @@ -180,6 +183,7 @@ // Subscribe for unl changes if we have to maintain the trusted server key checks. subscriptions[notificationChannels.unlChange] = trustedKeysLookup ? true : false; subscriptions[notificationChannels.ledgerEvent] = false; + subscriptions[notificationChannels.healthEvent] = false; let status = 0; //0:none, 1:connected, 2:closed @@ -547,7 +551,7 @@ if (connectionStatus == 0 && m.type == "user_challenge" && m.hp_version && m.contract_id) { - if (m.hp_version != supportedHpVersion) { + if (!m.hp_version.startsWith(supportedHpVersion)) { liblog(1, `Incompatible Hot Pocket server version. Expected:${supportedHpVersion} Got:${m.hp_version}`); return false; } @@ -690,6 +694,10 @@ ev.inSync = m.in_sync; emitter.emit(events.ledgerEvent, ev); } + else if (m.type == "health_event") { + const ev = msgHelper.deserializeHealthEvent(m); + emitter.emit(events.healthEvent, ev); + } else if (m.type == "ledger_query_result") { const resolver = ledgerQueryResolvers[m.reply_for]; if (resolver) { @@ -1141,6 +1149,24 @@ outputHash: this.deserializeValue(l.output_hash) } } + + this.deserializeHealthEvent = (m) => { + if (m.event === "proposal") { + return { + event: m.event, + commLatency: m.comm_latency, + readLatency: m.read_latency, + batchSize: m.batch_size + } + } + else if (m.event === "connectivity") { + return { + event: m.event, + peerCount: m.peer_count, + weaklyConnected: m.weakly_connected + } + } + } } function hexToUint8Array(hexString) { diff --git a/prereq.sh b/prereq.sh index cc124b3..4d5d36f 100755 --- a/prereq.sh +++ b/prereq.sh @@ -30,6 +30,11 @@ else fi fi +# Install slirp4netns if not exists (required for high performance rootless networking). +if ! command -v slirp4netns &>/dev/null; then + apt -y install slirp4netns +fi + # Check for pattern # And whether Options is *grpjquota=aquota.group or jqfmt=vfsv0* # If not add groupquota to the options. diff --git a/src/conf.cpp b/src/conf.cpp index a62b631..18d6d80 100644 --- a/src/conf.cpp +++ b/src/conf.cpp @@ -63,9 +63,10 @@ namespace conf cfg.hp.init_user_port = 8081; cfg.system.max_instance_count = 5; - cfg.system.max_mem_kbytes = 1024000; // Total 1GB RAM + cfg.system.max_mem_kbytes = 1048576; // Total 1GB RAM + cfg.system.max_swap_kbytes = 1572864; // Total 1.5GB RAM cfg.system.max_cpu_us = 5000000; // CPU cfs period cannot be less than 1ms (i.e. 1000) or larger than 1s (i.e. 1000000) per instance. - cfg.system.max_storage_kbytes = 2048000; // Total 2GB + cfg.system.max_storage_kbytes = 5242880; // Total 5GB const std::string img_prefix = registry_addr.empty() ? "hotpocketdev" : std::string(registry_addr); cfg.docker.images["hp.0.5-ubt.20.04"] = img_prefix + "/sashimono:hp.0.5-ubt.20.04"; @@ -242,6 +243,7 @@ namespace conf const jsoncons::ojson &system = d["system"]; cfg.system.max_mem_kbytes = system["max_mem_kbytes"].as(); + cfg.system.max_swap_kbytes = system["max_swap_kbytes"].as(); cfg.system.max_cpu_us = system["max_cpu_us"].as(); cfg.system.max_storage_kbytes = system["max_storage_kbytes"].as(); cfg.system.max_instance_count = system["max_instance_count"].as(); @@ -324,6 +326,7 @@ namespace conf jsoncons::ojson system_config; system_config.insert_or_assign("max_mem_kbytes", cfg.system.max_mem_kbytes); + system_config.insert_or_assign("max_swap_kbytes", cfg.system.max_swap_kbytes); system_config.insert_or_assign("max_cpu_us", cfg.system.max_cpu_us); system_config.insert_or_assign("max_storage_kbytes", cfg.system.max_storage_kbytes); system_config.insert_or_assign("max_instance_count", cfg.system.max_instance_count); diff --git a/src/conf.hpp b/src/conf.hpp index 8ba375d..3e132f7 100644 --- a/src/conf.hpp +++ b/src/conf.hpp @@ -77,6 +77,7 @@ namespace conf { size_t max_cpu_us = 0; // Max CPU time the agent process can consume. size_t max_mem_kbytes = 0; // Max memory the agent process can allocate in KB. + size_t max_swap_kbytes = 0; // Max swap memory the agent process can allocate in KB. size_t max_storage_kbytes = 0; // Max physical storage the agent process can allocate in KB. size_t max_instance_count = 0; // Max number of instances that can be created. }; diff --git a/src/hp_manager.cpp b/src/hp_manager.cpp index ee3d86d..22bf961 100644 --- a/src/hp_manager.cpp +++ b/src/hp_manager.cpp @@ -56,6 +56,7 @@ namespace hp // Calculate the resources per instance. instance_resources.cpu_us = conf::cfg.system.max_cpu_us / conf::cfg.system.max_instance_count; instance_resources.mem_kbytes = conf::cfg.system.max_mem_kbytes / conf::cfg.system.max_instance_count; + instance_resources.swap_kbytes = instance_resources.mem_kbytes + (conf::cfg.system.max_swap_kbytes / conf::cfg.system.max_instance_count); instance_resources.storage_kbytes = conf::cfg.system.max_storage_kbytes / conf::cfg.system.max_instance_count; contract_ugid = {CONTRACT_USER_ID, CONTRACT_USER_ID}; @@ -147,7 +148,7 @@ namespace hp int user_id; std::string username; - if (install_user(user_id, username, instance_resources.cpu_us, instance_resources.mem_kbytes, instance_resources.storage_kbytes, container_name) == -1) + if (install_user(user_id, username, instance_resources.cpu_us, instance_resources.mem_kbytes, instance_resources.swap_kbytes, instance_resources.storage_kbytes, container_name) == -1) return -1; const std::string contract_dir = util::get_user_contract_dir(username, container_name); @@ -819,13 +820,15 @@ namespace hp * @param username Username of the created user to be populated. * @param max_cpu_us CPU quota allowed for this user. * @param max_mem_kbytes Memory quota allowed for this user. + * @param max_swap_kbytes Swap memory quota allowed for this user. * @param storage_kbytes Disk quota allowed for this user. */ - int install_user(int &user_id, std::string &username, const size_t max_cpu_us, const size_t max_mem_kbytes, const size_t storage_kbytes, const std::string container_name) + int install_user(int &user_id, std::string &username, const size_t max_cpu_us, const size_t max_mem_kbytes, const size_t max_swap_kbytes, const size_t storage_kbytes, const std::string container_name) { const std::vector input_params = { std::to_string(max_cpu_us), std::to_string(max_mem_kbytes), + std::to_string(max_swap_kbytes), std::to_string(storage_kbytes), container_name, std::to_string(contract_ugid.uid), diff --git a/src/hp_manager.hpp b/src/hp_manager.hpp index 7abe6ec..4300b54 100644 --- a/src/hp_manager.hpp +++ b/src/hp_manager.hpp @@ -50,6 +50,7 @@ namespace hp { size_t cpu_us = 0; // CPU time an instance can consume. size_t mem_kbytes = 0; // Memory an instance can allocate. + size_t swap_kbytes = 0; // Swap memory an instance can allocate. size_t storage_kbytes = 0; // Physical storage an instance can allocate. }; @@ -84,7 +85,7 @@ namespace hp int write_json_values(jsoncons::ojson &d, const msg::config_struct &config); - int install_user(int &user_id, std::string &username, const size_t max_cpu_us, const size_t max_mem_kbytes, const size_t storage_kbytes, const std::string container_name); + int install_user(int &user_id, std::string &username, const size_t max_cpu_us, const size_t max_mem_kbytes, const size_t max_swap_kbytes, const size_t storage_kbytes, const std::string container_name); int uninstall_user(std::string_view username); diff --git a/test/vm-cluster/.gitignore b/test/vm-cluster/.gitignore index 0cffcb3..c0ca2e1 100644 --- a/test/vm-cluster/.gitignore +++ b/test/vm-cluster/.gitignore @@ -1 +1,4 @@ -config.json \ No newline at end of file +config.json +hp.cfg +contract_fs +ledger_fs \ No newline at end of file diff --git a/test/vm-cluster/cluster.sh b/test/vm-cluster/cluster.sh index 1467a48..9c6ae55 100755 --- a/test/vm-cluster/cluster.sh +++ b/test/vm-cluster/cluster.sh @@ -5,6 +5,7 @@ # ./cluster.sh select contract # ./cluster.sh create 1 # ./cluster.sh create +# ./cluster.sh createall 22861 # ./cluster.sh reconfig # ./cluster.sh reconfig R # ./cluster.sh reconfig 1 R @@ -16,11 +17,22 @@ # reconfig - Re configure the sashimono with given "max_instance_count" in all the hosts (Only update the sa.cfg, Reinstall the sashimono if "R" option is given). # lcl - Get lcl of the hosts. # create - Create new sashimono hotpocket instance in each node. +# createall - Create sashimono hotpocket instances in all nodes parallely. # get-unl - Construct the UNL of all the nodes (Useful when creating cfg for contract upload). # docker-pull - Pull the latest docker image from docker hub. # start - Start sashimono hotpocket instance. # stop - Stop sashimono hotpocket instance. # destroy - Destroy sashimono hotpocket instance. +# ssh - Login with ssh or execute command on all nodes via ssh. +# sshu - Login with ssh or execute command on all nodes via ssh under instance user. +# attach - Attach to the docker instance output. +# ip - Show ip address of nodes. +# updatecfg - Update the hp config using the local file hp.cfg. +# statefile - Send a local file to instance contract_fs/seed/state/ +# umount - Unmount instance contract/ledger fuse mounts. (Used to cleanup orphan mounts) +# backup - Downloads contract and ledger files from the given node. +# restore - Uploads previously downloaded contract and ledger files. +# syncwith - Manually syncs the entire cluster with the given node. LOCKFILE="/tmp/sashiclusercfg.lock" trap "rm -f $LOCKFILE" EXIT @@ -29,12 +41,16 @@ PRINTFORMAT="Node %2s: %s\n" mode=$1 -if [ "$mode" == "select" ] || [ "$mode" == "reconfig" ] || [ "$mode" == "lcl" ] || [ "$mode" == "get-unl" ] || [ "$mode" == "docker-pull" ] || [ "$mode" == "create" ] || [ "$mode" == "start" ] || [ "$mode" == "stop" ] || [ "$mode" == "destroy" ]; then +if [ "$mode" == "select" ] || [ "$mode" == "reconfig" ] || [ "$mode" == "lcl" ] || [ "$mode" == "get-unl" ] || [ "$mode" == "docker-pull" ] || + [ "$mode" == "create" ] || [ "$mode" == "createall" ] || [ "$mode" == "start" ] || [ "$mode" == "stop" ] || [ "$mode" == "destroy" ] || + [ "$mode" == "ssh" ] || [ "$mode" == "sshu" ] || [ "$mode" == "attach" ] || [ "$mode" == "ip" ] || [ "$mode" == "updatecfg" ] || + [ "$mode" == "statefile" ] || [ "$mode" == "umount" ] || [ "$mode" == "backup" ] || [ "$mode" == "restore" ] || [ "$mode" == "syncwith" ]; then echo "mode: $mode" else echo "Invalid command." - echo " Expected: select | reconfig [N] [R] | lcl [N] | get-unl | docker-pull [N] | create [N] | start [N] | stop [N] | destroy [N]" - echo " [N]: Optional node no. [R]: 'R' If sashimono needed to reinstall." + echo " Expected: select | reconfig [N] [R] | lcl [N] | get-unl | docker-pull [N] | create [N] | createall | start [N] | stop [N] |" + echo " destroy [N] | ssh or | sshu | attach | ip [N] | updatecfg [N] | statefile [N] | umount [N] | backup | restore [N] | syncwith " + echo " [N]: Optional node no. : Required node no. [R]: 'R' If sashimono needed to reinstall." exit 1 fi @@ -46,7 +62,7 @@ fi configfile=config.json if [ ! -f $configfile ]; then # Create default config file. - echo '{"selected":"contract","contracts":[{"name":"contract","sshuser":"root","sshpass":"","owner_pubkey":"ed.....","contract_id":"","docker":{"image":"","id":"","pass":""},"vultr_group":"","hosts":{"host1_ip":{}},"config":{},"sa_config":{"max_instance_count":-1}}],"vultr":{"api_key":""}}' | jq . >$configfile + echo '{"selected":"contract","contracts":[{"name":"contract","sshuser":"root","sshpass":"","owner_pubkey":"ed.....","contract_id":"","docker":{"repo":"","image":"","id":"","pass":""},"vultr_group":"","hosts":{"host1_ip":{}},"config":{},"sa_config":{"max_instance_count":-1}}],"vultr":{"api_key":""}}' | jq . >$configfile fi if [ $mode == "select" ]; then @@ -89,8 +105,10 @@ fi shopt -s expand_aliases alias sshskp='ssh -o StrictHostKeychecking=no' +alias scpskp='scp -o StrictHostKeychecking=no' if [ "$sshpass" != "" ] && [ "$sshpass" != "null" ]; then alias sshskp="sshpass -p $sshpass ssh -o StrictHostKeychecking=no" + alias scpskp="sshpass -p $sshpass scp -o StrictHostKeychecking=no" fi function updateconfig() { @@ -152,7 +170,7 @@ if [ $mode == "reconfig" ]; then # If reinstall specified, show warn and take confirmation. if [ ! -z $reinstall ] && [ $reinstall == "R" ]; then - echo "Warning: you'll lost all the sashimono instances!" + echo "Warning: you'll lose all the sashimono instances!" echo "Still are you sure you want to reinstall Sashimono?" read -p "Type 'yes' to confirm reinstall: " confirmation /dev/null; then printf "$PRINTFORMAT" "$nodeno" "Error occured pulling $image." else @@ -293,7 +326,7 @@ if [ $mode == "docker-pull" ]; then exit 0 fi -if [ $mode == "create" ]; then +if [ $mode == "create" ] || [ $mode == "createall" ]; then # Read owner pubkey, contract id and image ownerpubkey=$(echo $continfo | jq -r '.owner_pubkey') if [ "$ownerpubkey" = "" ] || [ "$ownerpubkey" = "null" ]; then @@ -329,13 +362,17 @@ if [ $mode == "create" ]; then if [ "$1" != 0 ]; then peers="" for ((i = 0; i < $1; i++)); do - hostinfo=$(echo $continfo | jq -r ".hosts.\"${hostaddrs[$i]}\"") - peerport=$(echo $hostinfo | jq -r '.peer_port') + if [ -z "$2" ]; then + hostinfo=$(echo $continfo | jq -r ".hosts.\"${hostaddrs[$i]}\"") + peerport=$(echo $hostinfo | jq -r '.peer_port') - if [ "$hostinfo" == "" ] || [ "$hostinfo" == "null" ] || - [ "$peerport" == "" ] || [ "$peerport" == "null" ]; then - echo "Host info is empty for ${hostaddrs[$i]}" - exit 1 + if [ "$hostinfo" == "" ] || [ "$hostinfo" == "null" ] || + [ "$peerport" == "" ] || [ "$peerport" == "null" ]; then + echo "Host info is empty for ${hostaddrs[$i]}" + exit 1 + fi + else + peerport=$2 fi peers+="\"${hostaddrs[$i]}:$peerport\"," done @@ -365,13 +402,28 @@ if [ $mode == "create" ]; then fi } - if [ $nodeid = -1 ]; then + if [ $mode == "create" ]; then + if [ $nodeid = -1 ]; then + for i in "${!hostaddrs[@]}"; do + createinstance $i + done + else + createinstance $nodeid $peerport + fi + else + # Create all instances parallely with specified peer port. + peerport=$2 + [ -z "$peerport" ] && echo "Peer port is required." && exit 1 for i in "${!hostaddrs[@]}"; do - createinstance $i + if [ $i == "0" ]; then + # Create first instance sequentially so others can get its public key for their unl. + echo "Creating first instance..." + createinstance $i $peerport + else + createinstance $i $peerport & + fi done wait - else - createinstance $nodeid fi exit 0 fi @@ -531,3 +583,246 @@ if [ $mode == "destroy" ]; then fi exit 0 fi + +if [ $mode = "ssh" ]; then + if [ $nodeid = -1 ]; then + if [ -n "$2" ]; then + # Interpret second arg as a command to execute on all nodes. + command=${*:2} + echo "Executing '$command' on all nodes..." + for i in "${!hostaddrs[@]}"; do + hostaddr=${hostaddrs[i]} + let n=$i+1 + echo "node"$n":" $(sshskp $sshuser@$hostaddr $command) & + done + wait + exit 0 + else + echo "Please specify node no. or command to execute on all nodes." + exit 1 + fi + else + hostaddr=${hostaddrs[$nodeid]} + sshskp -t $sshuser@$hostaddr + exit 0 + fi +fi + +if [ $mode == "sshu" ]; then + + function sshwithuser() { + hostaddr=${hostaddrs[$1]} + nodeno=$(expr $1 + 1) + containername=$(echo $continfo | jq -r ".hosts.\"$hostaddr\".name") + username=$(sshskp $sshuser@$hostaddr "sashi list | grep $containername | awk '{ print \$2 }'") + + ssh_command="cd /home/$username/$containername ; sudo -u $username bash" + sshskp -t $sshuser@$hostaddr $ssh_command + } + + if [ $nodeid = -1 ]; then + echo "Must specify node no." + exit 1 + else + sshwithuser $nodeid + fi + exit 0 +fi + +if [ $mode == "attach" ]; then + + function attachdocker() { + hostaddr=${hostaddrs[$1]} + nodeno=$(expr $1 + 1) + containername=$(echo $continfo | jq -r ".hosts.\"$hostaddr\".name") + username=$(sshskp $sshuser@$hostaddr "sashi list | grep $containername | awk '{ print \$2 }'") + + echo "Press ctrl+P,Q to detach." + ssh_command="sudo -u $username bash -i -c 'docker attach $containername'" + sshskp -t $sshuser@$hostaddr $ssh_command + } + + if [ $nodeid = -1 ]; then + echo "Must specify node no." + exit 1 + else + attachdocker $nodeid + fi + exit 0 +fi + +if [ $mode = "ip" ]; then + if [ $nodeid = -1 ]; then + for i in "${!hostaddrs[@]}"; do + let n=$i+1 + echo "node"$n": ${hostaddrs[i]}" + done + else + echo "${hostaddrs[$nodeid]}" + fi + exit 0 +fi + +if [ $mode == "updatecfg" ]; then + + function sendcfg() { + hostaddr=${hostaddrs[$1]} + nodeno=$(expr $1 + 1) + containername=$(echo $continfo | jq -r ".hosts.\"$hostaddr\".name") + + username=$(sshskp $sshuser@$hostaddr "sashi list | grep $containername | awk '{ print \$2 }'") + originalcfg="/home/$username/$containername/cfg/hp.cfg" + + scpskp -q hp.cfg $sshuser@$hostaddr:~/ + sshskp $sshuser@$hostaddr "jq -s '.[0] * .[1]' $originalcfg ~/hp.cfg > ~/merged.cfg && mv ~/merged.cfg $originalcfg && chown $username:$username $originalcfg && rm ~/hp.cfg" + echo "node$nodeno: Updated $originalcfg" + } + + if [ $nodeid = -1 ]; then + for i in "${!hostaddrs[@]}"; do + sendcfg $i & + done + wait + else + sendcfg $nodeid + fi + exit 0 +fi + +if [ $mode == "statefile" ]; then + + function sendstatefile() { + localfilepath=$2 + filename=$(basename $2) + hostaddr=${hostaddrs[$1]} + nodeno=$(expr $1 + 1) + containername=$(echo $continfo | jq -r ".hosts.\"$hostaddr\".name") + + username=$(sshskp $sshuser@$hostaddr "sashi list | grep $containername | awk '{ print \$2 }'") + fspath="/home/$username/$containername/contract_fs" + seedpath="$fspath/seed/state" + + scpskp -q $localfilepath $sshuser@$hostaddr:$seedpath/ + sshskp $sshuser@$hostaddr "chown $username:$username $seedpath/$filename && rm -r $fspath/hmap && rm $fspath/log.hpfs" + echo "node$nodeno: Transferred to $seedpath/$filename" + } + + if [ $nodeid = -1 ]; then + for i in "${!hostaddrs[@]}"; do + sendstatefile $i $2 & + done + wait + else + sendstatefile $nodeid $3 + fi + exit 0 +fi + +if [ $mode == "umount" ]; then + + function unmountfuse() { + hostaddr=${hostaddrs[$1]} + nodeno=$(expr $1 + 1) + containername=$(echo $continfo | jq -r ".hosts.\"$hostaddr\".name") + + username=$(sshskp $sshuser@$hostaddr "sashi list | grep $containername | awk '{ print \$2 }'") + contractmnt="/home/$username/$containername/contract_fs/mnt" + ledgermnt="/home/$username/$containername/ledger_fs/mnt" + + sshskp $sshuser@$hostaddr "fusermount -u $contractmnt ; fusermount -u $ledgermnt" + echo "node$nodeno: Unmount complete." + } + + if [ $nodeid = -1 ]; then + for i in "${!hostaddrs[@]}"; do + unmountfuse $i & + done + wait + else + unmountfuse $nodeid + fi + exit 0 +fi + +function downloadNode() { + hostaddr=${hostaddrs[$1]} + nodeno=$(expr $1 + 1) + containername=$(echo $continfo | jq -r ".hosts.\"$hostaddr\".name") + + username=$(sshskp $sshuser@$hostaddr "sashi list | grep $containername | awk '{ print \$2 }'") + contractfs="/home/$username/$containername/contract_fs" + ledgerfs="/home/$username/$containername/ledger_fs" + + echo "Downloading from node$nodeno" + rm -r contract_fs > /dev/null 2>&1 + mkdir contract_fs + scpskp -r -q $sshuser@$hostaddr:$contractfs/seed contract_fs/ + + rm -r ledger_fs > /dev/null 2>&1 + mkdir ledger_fs + scpskp -r -q $sshuser@$hostaddr:$ledgerfs/seed ledger_fs/ + echo "Download complete." +} + +function uploadNode() { + hostaddr=${hostaddrs[$1]} + nodeno=$(expr $1 + 1) + containername=$(echo $continfo | jq -r ".hosts.\"$hostaddr\".name") + + username=$(sshskp $sshuser@$hostaddr "sashi list | grep $containername | awk '{ print \$2 }'") + contractfs="/home/$username/$containername/contract_fs" + ledgerfs="/home/$username/$containername/ledger_fs" + + sshskp $sshuser@$hostaddr "rm -r $contractfs/{seed,hmap,log.hpfs} ; rm -r $ledgerfs/{seed,hmap,log.hpfs}" + echo "node$nodeno: Uploading to $contractfs/" + scpskp -r -q contract_fs/seed $sshuser@$hostaddr:$contractfs/ + echo "node$nodeno: Uploading to $ledgerfs/" + scpskp -r -q ledger_fs/seed $sshuser@$hostaddr:$ledgerfs/ + + sshskp $sshuser@$hostaddr "chown -R $username:$username $contractfs/seed ; chown -R $username:$username $ledgerfs/seed" + + echo "node$nodeno: Upload complete." +} + +if [ $mode == "backup" ]; then + + if [ $nodeid = -1 ]; then + echo "Must specify node no." + exit 1 + else + downloadNode $nodeid + fi + exit 0 +fi + +if [ $mode == "restore" ]; then + + if [ $nodeid = -1 ]; then + for i in "${!hostaddrs[@]}"; do + uploadNode $i & + done + wait + else + uploadNode $nodeid + fi + exit 0 +fi + +if [ $mode == "syncwith" ]; then + + if [ $nodeid = -1 ]; then + echo "Must specify node no." + exit 1 + else + downloadNode $nodeid + for i in "${!hostaddrs[@]}"; do + if [ "$i" != $nodeid ]; then + uploadNode $i & + fi + done + wait + rm -r ledger_fs + rm -r contract_fs + fi + exit 0 +fi \ No newline at end of file