ARCHIVE/sashimono
1
0
Fork 0
mirror of https://github.com/EvernodeXRPL/sashimono.git synced 2026-04-29 15:38:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

Improvements for large test cluster management. (#59)

Browse Source 
* VM script improvements.
* Modified docker-pull to create new container
* Replaced vpnkit with slirp4netns for better containerized network performance.
* Added swap allocation config.
Co-authored-by: chalith <desaman.chalith@gmail.com>
This commit is contained in:
Ravin Perera
2021-09-17 12:07:56 +05:30
committed by GitHub
parent f38591eb19
commit 1473d2b059
10 changed files with 384 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
dependencies/user-cgcreate.sh vendored
View File

@@ -23,6 +23,7 @@ fi
# Read config values
max_mem_kbytes=$(jq '.system.max_mem_kbytes' $saconfig)
max_swap_kbytes=$(jq '.system.max_swap_kbytes' $saconfig)
max_cpu_us=$(jq '.system.max_cpu_us' $saconfig)
max_instance_count=$(jq '.system.max_instance_count' $saconfig)
@@ -33,6 +34,11 @@ if [ "$max_mem_kbytes" != "" ] && [ ! ${#max_mem_kbytes} -eq 0 ] && [ "$max_mem_
! instance_mem_kbytes=$(expr $max_mem_kbytes / $max_instance_count) && echo "Max memory limit calculation error." && exit 1
fi
instance_swap_kbytes=0
if [ "$max_swap_kbytes" != "" ] && [ ! ${#max_swap_kbytes} -eq 0 ] && [ "$max_swap_kbytes" -gt 0 ]; then
! instance_swap_kbytes=$(expr $instance_mem_kbytes + $max_swap_kbytes / $max_instance_count) && echo "Max swap memory limit calculation error." && exit 1
fi
instance_cpu_us=0
if [ "$max_cpu_us" != "" ] && [ ! ${#max_cpu_us} -eq 0 ] && [ "$max_cpu_us" -gt 0 ]; then
! instance_cpu_us=$(expr $max_cpu_us / $max_instance_count) && echo "Max cpu limit calculation error." && exit 1
@@ -61,7 +67,7 @@ for user in "${validusers[@]}"; do
if [ $instance_mem_kbytes -gt 0 ] &&
! (cgcreate -g memory:$user$cgroupsuffix &&
echo "${instance_mem_kbytes}K" > /sys/fs/cgroup/memory/$user$cgroupsuffix/memory.limit_in_bytes &&
echo "${instance_mem_kbytes}K" > /sys/fs/cgroup/memory/$user$cgroupsuffix/memory.memsw.limit_in_bytes); then
echo "${instance_swap_kbytes}K" > /sys/fs/cgroup/memory/$user$cgroupsuffix/memory.memsw.limit_in_bytes); then
echo "Memory cgroup creation for $user failed."
has_err=1
fi

15
dependencies/user-install.sh vendored
View File

@@ -5,12 +5,13 @@
# Check for user cpu and memory quotas.
cpu=$1
memory=$2
disk=$3
contract_dir=$4
contract_uid=$5
contract_gid=$6
if [ -z "$cpu" ] || [ -z "$memory" ] || [ -z "$disk" ] || [ -z "$contract_dir" ] || [ -z "$contract_uid" ] || [ -z "$contract_gid" ]; then
echo "Expected: user-install.sh <cpu quota microseconds> <memory quota kbytes> <disk quota kbytes> <contract dir> <contract uid> <contract gid>"
swapmem=$3
disk=$4
contract_dir=$5
contract_uid=$6
contract_gid=$7
if [ -z "$cpu" ] || [ -z "$memory" ] || [ -z "$swapmem" ] || [ -z "$disk" ] || [ -z "$contract_dir" ] || [ -z "$contract_uid" ] || [ -z "$contract_gid" ]; then
echo "Expected: user-install.sh <cpu quota microseconds> <memory quota kbytes> <swap quota kbytes> <disk quota kbytes> <contract dir> <contract uid> <contract gid>"
echo "INVALID_PARAMS,INST_ERR" && exit 1
fi
@@ -69,7 +70,7 @@ dockerd_socket="unix://$user_runtime_dir/docker.sock"
echo "$cpu" >/sys/fs/cgroup/cpu/$user$cgroupsuffix/cpu.cfs_quota_us) && rollback "CGROUP_CPU_CREAT"
! (cgcreate -g memory:$user$cgroupsuffix &&
echo "${memory}K" >/sys/fs/cgroup/memory/$user$cgroupsuffix/memory.limit_in_bytes &&
echo "${memory}K" >/sys/fs/cgroup/memory/$user$cgroupsuffix/memory.memsw.limit_in_bytes) && rollback "CGROUP_MEM_CREAT"
echo "${swapmem}K" >/sys/fs/cgroup/memory/$user$cgroupsuffix/memory.memsw.limit_in_bytes) && rollback "CGROUP_MEM_CREAT"
# Adding disk quota to the group.
setquota -g -F vfsv0 "$user" "$disk" "$disk" 0 0 /

36
examples/client/lib/hp-client-lib.js
View File

@@ -1,10 +1,11 @@
/**
* Hot Pocket javascript client library (for NodeJs and Browser)
* Version 0.5.0
* NodeJs: const HotPocket = require("./hp-client-lib")
* Browser: window.HotPocket
*/
(() => {
(() => {
// Whether we are in Browser or NodeJs.
const isBrowser = !(typeof window === 'undefined');
@@ -21,7 +22,7 @@
TextDecoder = util.TextDecoder;
}
const supportedHpVersion = "0.5.0";
const supportedHpVersion = "0.5.";
const serverChallengeSize = 16;
const outputValidationPassThreshold = 0.8;
const connectionCheckIntervalMs = 1000;
@@ -51,14 +52,16 @@
contractReadResponse: "contract_read_response",
connectionChange: "connection_change",
unlChange: "unl_change",
ledgerEvent: "ledger_event"
ledgerEvent: "ledger_event",
healthEvent: "health_event"
}
Object.freeze(events);
/*--- Included in public interface. ---*/
const notificationChannels = {
unlChange: "unl_change",
ledgerEvent: "ledger_event"
ledgerEvent: "ledger_event",
healthEvent: "health_event"
}
Object.freeze(notificationChannels);
@@ -180,6 +183,7 @@
// Subscribe for unl changes if we have to maintain the trusted server key checks.
subscriptions[notificationChannels.unlChange] = trustedKeysLookup ? true : false;
subscriptions[notificationChannels.ledgerEvent] = false;
subscriptions[notificationChannels.healthEvent] = false;
let status = 0; //0:none, 1:connected, 2:closed
@@ -547,7 +551,7 @@
if (connectionStatus == 0 && m.type == "user_challenge" && m.hp_version && m.contract_id) {
if (m.hp_version != supportedHpVersion) {
if (!m.hp_version.startsWith(supportedHpVersion)) {
liblog(1, `Incompatible Hot Pocket server version. Expected:${supportedHpVersion} Got:${m.hp_version}`);
return false;
}
@@ -690,6 +694,10 @@
ev.inSync = m.in_sync;
emitter.emit(events.ledgerEvent, ev);
}
else if (m.type == "health_event") {
const ev = msgHelper.deserializeHealthEvent(m);
emitter.emit(events.healthEvent, ev);
}
else if (m.type == "ledger_query_result") {
const resolver = ledgerQueryResolvers[m.reply_for];
if (resolver) {
@@ -1141,6 +1149,24 @@
outputHash: this.deserializeValue(l.output_hash)
}
}
this.deserializeHealthEvent = (m) => {
if (m.event === "proposal") {
return {
event: m.event,
commLatency: m.comm_latency,
readLatency: m.read_latency,
batchSize: m.batch_size
}
}
else if (m.event === "connectivity") {
return {
event: m.event,
peerCount: m.peer_count,
weaklyConnected: m.weakly_connected
}
}
}
}
function hexToUint8Array(hexString) {

5
prereq.sh
View File

@@ -30,6 +30,11 @@ else
fi
fi
# Install slirp4netns if not exists (required for high performance rootless networking).
if ! command -v slirp4netns &>/dev/null; then
apt -y install slirp4netns
fi
# Check for pattern <Not starting with a comment><Not whitespace(Device)><Whitespace></><Whitespace><Not whitespace(FS type)><Whitespace><No whitespace(Options)><Whitespace><Number(Dump)><Whitespace><Number(Pass)>
# And whether Options is <Not whitespace>*grpjquota=aquota.group or jqfmt=vfsv0<Not whitespace>*
# If not add groupquota to the options.

7
src/conf.cpp
View File

@@ -63,9 +63,10 @@ namespace conf
cfg.hp.init_user_port = 8081;
cfg.system.max_instance_count = 5;
cfg.system.max_mem_kbytes = 1024000; // Total 1GB RAM
cfg.system.max_mem_kbytes = 1048576; // Total 1GB RAM
cfg.system.max_swap_kbytes = 1572864; // Total 1.5GB RAM
cfg.system.max_cpu_us = 5000000; // CPU cfs period cannot be less than 1ms (i.e. 1000) or larger than 1s (i.e. 1000000) per instance.
cfg.system.max_storage_kbytes = 2048000; // Total 2GB
cfg.system.max_storage_kbytes = 5242880; // Total 5GB
const std::string img_prefix = registry_addr.empty() ? "hotpocketdev" : std::string(registry_addr);
cfg.docker.images["hp.0.5-ubt.20.04"] = img_prefix + "/sashimono:hp.0.5-ubt.20.04";
@@ -242,6 +243,7 @@ namespace conf
const jsoncons::ojson &system = d["system"];
cfg.system.max_mem_kbytes = system["max_mem_kbytes"].as<size_t>();
cfg.system.max_swap_kbytes = system["max_swap_kbytes"].as<size_t>();
cfg.system.max_cpu_us = system["max_cpu_us"].as<size_t>();
cfg.system.max_storage_kbytes = system["max_storage_kbytes"].as<size_t>();
cfg.system.max_instance_count = system["max_instance_count"].as<size_t>();
@@ -324,6 +326,7 @@ namespace conf
jsoncons::ojson system_config;
system_config.insert_or_assign("max_mem_kbytes", cfg.system.max_mem_kbytes);
system_config.insert_or_assign("max_swap_kbytes", cfg.system.max_swap_kbytes);
system_config.insert_or_assign("max_cpu_us", cfg.system.max_cpu_us);
system_config.insert_or_assign("max_storage_kbytes", cfg.system.max_storage_kbytes);
system_config.insert_or_assign("max_instance_count", cfg.system.max_instance_count);

1
src/conf.hpp
View File

@@ -77,6 +77,7 @@ namespace conf
{
size_t max_cpu_us = 0; // Max CPU time the agent process can consume.
size_t max_mem_kbytes = 0; // Max memory the agent process can allocate in KB.
size_t max_swap_kbytes = 0; // Max swap memory the agent process can allocate in KB.
size_t max_storage_kbytes = 0; // Max physical storage the agent process can allocate in KB.
size_t max_instance_count = 0; // Max number of instances that can be created.
};

7
src/hp_manager.cpp
View File

@@ -56,6 +56,7 @@ namespace hp
// Calculate the resources per instance.
instance_resources.cpu_us = conf::cfg.system.max_cpu_us / conf::cfg.system.max_instance_count;
instance_resources.mem_kbytes = conf::cfg.system.max_mem_kbytes / conf::cfg.system.max_instance_count;
instance_resources.swap_kbytes = instance_resources.mem_kbytes + (conf::cfg.system.max_swap_kbytes / conf::cfg.system.max_instance_count);
instance_resources.storage_kbytes = conf::cfg.system.max_storage_kbytes / conf::cfg.system.max_instance_count;
contract_ugid = {CONTRACT_USER_ID, CONTRACT_USER_ID};
@@ -147,7 +148,7 @@ namespace hp
int user_id;
std::string username;
if (install_user(user_id, username, instance_resources.cpu_us, instance_resources.mem_kbytes, instance_resources.storage_kbytes, container_name) == -1)
if (install_user(user_id, username, instance_resources.cpu_us, instance_resources.mem_kbytes, instance_resources.swap_kbytes, instance_resources.storage_kbytes, container_name) == -1)
return -1;
const std::string contract_dir = util::get_user_contract_dir(username, container_name);
@@ -819,13 +820,15 @@ namespace hp
* @param username Username of the created user to be populated.
* @param max_cpu_us CPU quota allowed for this user.
* @param max_mem_kbytes Memory quota allowed for this user.
* @param max_swap_kbytes Swap memory quota allowed for this user.
* @param storage_kbytes Disk quota allowed for this user.
*/
int install_user(int &user_id, std::string &username, const size_t max_cpu_us, const size_t max_mem_kbytes, const size_t storage_kbytes, const std::string container_name)
int install_user(int &user_id, std::string &username, const size_t max_cpu_us, const size_t max_mem_kbytes, const size_t max_swap_kbytes, const size_t storage_kbytes, const std::string container_name)
{
const std::vector<std::string_view> input_params = {
std::to_string(max_cpu_us),
std::to_string(max_mem_kbytes),
std::to_string(max_swap_kbytes),
std::to_string(storage_kbytes),
container_name,
std::to_string(contract_ugid.uid),

3
src/hp_manager.hpp
View File

@@ -50,6 +50,7 @@ namespace hp
{
size_t cpu_us = 0; // CPU time an instance can consume.
size_t mem_kbytes = 0; // Memory an instance can allocate.
size_t swap_kbytes = 0; // Swap memory an instance can allocate.
size_t storage_kbytes = 0; // Physical storage an instance can allocate.
};
@@ -84,7 +85,7 @@ namespace hp
int write_json_values(jsoncons::ojson &d, const msg::config_struct &config);
int install_user(int &user_id, std::string &username, const size_t max_cpu_us, const size_t max_mem_kbytes, const size_t storage_kbytes, const std::string container_name);
int install_user(int &user_id, std::string &username, const size_t max_cpu_us, const size_t max_mem_kbytes, const size_t max_swap_kbytes, const size_t storage_kbytes, const std::string container_name);
int uninstall_user(std::string_view username);

5
test/vm-cluster/.gitignore vendored
View File

@@ -1 +1,4 @@
config.json
config.json
hp.cfg
contract_fs
ledger_fs

337
test/vm-cluster/cluster.sh
View File

@@ -5,6 +5,7 @@
# ./cluster.sh select contract
# ./cluster.sh create 1
# ./cluster.sh create
# ./cluster.sh createall 22861
# ./cluster.sh reconfig
# ./cluster.sh reconfig R
# ./cluster.sh reconfig 1 R
@@ -16,11 +17,22 @@
# reconfig - Re configure the sashimono with given "max_instance_count" in all the hosts (Only update the sa.cfg, Reinstall the sashimono if "R" option is given).
# lcl - Get lcl of the hosts.
# create - Create new sashimono hotpocket instance in each node.
# createall - Create sashimono hotpocket instances in all nodes parallely.
# get-unl - Construct the UNL of all the nodes (Useful when creating cfg for contract upload).
# docker-pull - Pull the latest docker image from docker hub.
# start - Start sashimono hotpocket instance.
# stop - Stop sashimono hotpocket instance.
# destroy - Destroy sashimono hotpocket instance.
# ssh - Login with ssh or execute command on all nodes via ssh.
# sshu - Login with ssh or execute command on all nodes via ssh under instance user.
# attach - Attach to the docker instance output.
# ip - Show ip address of nodes.
# updatecfg - Update the hp config using the local file hp.cfg.
# statefile - Send a local file to instance contract_fs/seed/state/
# umount - Unmount instance contract/ledger fuse mounts. (Used to cleanup orphan mounts)
# backup - Downloads contract and ledger files from the given node.
# restore - Uploads previously downloaded contract and ledger files.
# syncwith - Manually syncs the entire cluster with the given node.
LOCKFILE="/tmp/sashiclusercfg.lock"
trap "rm -f $LOCKFILE" EXIT
@@ -29,12 +41,16 @@ PRINTFORMAT="Node %2s: %s\n"
mode=$1
if [ "$mode" == "select" ] || [ "$mode" == "reconfig" ] || [ "$mode" == "lcl" ] || [ "$mode" == "get-unl" ] || [ "$mode" == "docker-pull" ] || [ "$mode" == "create" ] || [ "$mode" == "start" ] || [ "$mode" == "stop" ] || [ "$mode" == "destroy" ]; then
if [ "$mode" == "select" ] || [ "$mode" == "reconfig" ] || [ "$mode" == "lcl" ] || [ "$mode" == "get-unl" ] || [ "$mode" == "docker-pull" ] ||
[ "$mode" == "create" ] || [ "$mode" == "createall" ] || [ "$mode" == "start" ] || [ "$mode" == "stop" ] || [ "$mode" == "destroy" ] ||
[ "$mode" == "ssh" ] || [ "$mode" == "sshu" ] || [ "$mode" == "attach" ] || [ "$mode" == "ip" ] || [ "$mode" == "updatecfg" ] ||
[ "$mode" == "statefile" ] || [ "$mode" == "umount" ] || [ "$mode" == "backup" ] || [ "$mode" == "restore" ] || [ "$mode" == "syncwith" ]; then
echo "mode: $mode"
else
echo "Invalid command."
echo " Expected: select <contract name> | reconfig [N] [R] | lcl [N] | get-unl | docker-pull [N] | create [N] | start [N] | stop [N] | destroy [N]"
echo " [N]: Optional node no. [R]: 'R' If sashimono needed to reinstall."
echo " Expected: select <contract name> | reconfig [N] [R] | lcl [N] | get-unl | docker-pull [N] | create [N] | createall <peerport> | start [N] | stop [N] |"
echo " destroy [N] | ssh <N>or<command> | sshu <N> | attach <N> | ip [N] | updatecfg [N] | statefile [N] <file> | umount [N] | backup <N> | restore [N] | syncwith <N>"
echo " [N]: Optional node no. <N>: Required node no. [R]: 'R' If sashimono needed to reinstall."
exit 1
fi
@@ -46,7 +62,7 @@ fi
configfile=config.json
if [ ! -f $configfile ]; then
# Create default config file.
echo '{"selected":"contract","contracts":[{"name":"contract","sshuser":"root","sshpass":"<ssh password>","owner_pubkey":"ed.....","contract_id":"<uuid>","docker":{"image":"<docker image key>","id":"","pass":""},"vultr_group":"","hosts":{"host1_ip":{}},"config":{},"sa_config":{"max_instance_count":-1}}],"vultr":{"api_key":"<vultr api key>"}}' | jq . >$configfile
echo '{"selected":"contract","contracts":[{"name":"contract","sshuser":"root","sshpass":"<ssh password>","owner_pubkey":"ed.....","contract_id":"<uuid>","docker":{"repo":"<docker repository>","image":"<docker image key>","id":"","pass":""},"vultr_group":"","hosts":{"host1_ip":{}},"config":{},"sa_config":{"max_instance_count":-1}}],"vultr":{"api_key":"<vultr api key>"}}' | jq . >$configfile
fi
if [ $mode == "select" ]; then
@@ -89,8 +105,10 @@ fi
shopt -s expand_aliases
alias sshskp='ssh -o StrictHostKeychecking=no'
alias scpskp='scp -o StrictHostKeychecking=no'
if [ "$sshpass" != "" ] && [ "$sshpass" != "null" ]; then
alias sshskp="sshpass -p $sshpass ssh -o StrictHostKeychecking=no"
alias scpskp="sshpass -p $sshpass scp -o StrictHostKeychecking=no"
fi
function updateconfig() {
@@ -152,7 +170,7 @@ if [ $mode == "reconfig" ]; then
# If reinstall specified, show warn and take confirmation.
if [ ! -z $reinstall ] && [ $reinstall == "R" ]; then
echo "Warning: you'll lost all the sashimono instances!"
echo "Warning: you'll lose all the sashimono instances!"
echo "Still are you sure you want to reinstall Sashimono?"
read -p "Type 'yes' to confirm reinstall: " confirmation </dev/tty
[ "$confirmation" != "yes" ] && echo "Reinstall cancelled." && exit 0
@@ -244,7 +262,12 @@ fi
if [ $mode == "docker-pull" ]; then
dockerbin=/usr/bin/sashimono-agent/dockerbin/docker
dockerrepo="hotpocketdev/sashimono:"
repo=$(echo $continfo | jq -r '.docker.repo')
if [ "$repo" == "" ] || [ "$repo" == "null" ]; then
echo "repo not specified."
exit 1
fi
# Read the image.
image=$(echo $continfo | jq -r '.docker.image')
if [ "$image" == "" ] || [ "$image" == "null" ]; then
@@ -252,10 +275,12 @@ if [ $mode == "docker-pull" ]; then
exit 1
fi
image="$repo:$image"
# Read docker credentials.
dockerid=$(echo $continfo | jq -r '.docker.id')
dockerpass=$(echo $continfo | jq -r '.docker.pass')
dockerpull="$dockerbin pull $dockerrepo$image"
dockerpull="$dockerbin pull $image"
# If credentials given.
if [ "$dockerid" != "" ] && [ "$dockerid" != "null" ] && [ "$dockerpass" != "" ] && [ "$dockerpass" != "null" ]; then
dockerpull="(echo $dockerpass | $dockerbin login -u $dockerid --password-stdin &>/dev/null) && $dockerpull && $dockerbin logout"
@@ -266,15 +291,23 @@ if [ $mode == "docker-pull" ]; then
hostaddr=${hostaddrs[$1]}
nodeno=$(expr $1 + 1)
containername=$(echo $continfo | jq -r ".hosts.\"$hostaddr\".name")
userport=$(echo $continfo | jq -r ".hosts.\"$hostaddr\".user_port")
peerport=$(echo $continfo | jq -r ".hosts.\"$hostaddr\".peer_port")
if [ "$containername" == "" ] || [ "$containername" == "null" ]; then
printf "$PRINTFORMAT" "$nodeno" "Host info is empty."
exit 1
fi
user="user=\$(find / -type d -path '/home/sashi*/$containername' 2>/dev/null | cut -d/ -f3) || [ ! -z \$user ]"
dpull="sudo -H -u \$user DOCKER_HOST=\"unix:///run/user/\$(id -u \$user)/docker.sock\" bash -c \"$dockerpull\""
command="$user && $dpull"
contractpath="contractpath=\$(find / -type d -path '/home/sashi*/$containername' 2>/dev/null) || [ ! -z \$contractpath ]"
user="user=\$(echo \$contractpath | cut -d/ -f3) || [ ! -z \$user ]"
dockerstop="$dockerbin stop $containername"
dockerrm="$dockerbin rm $containername"
dockercreate="$dockerbin create -t -i --stop-signal=SIGINT --name=$containername -p $userport:$userport -p $peerport:$peerport --restart unless-stopped --mount type=bind,source=\$contractpath,target=/contract $image run /contract"
dpull="sudo -H -u \$user DOCKER_HOST=\"unix:///run/user/\$(id -u \$user)/docker.sock\" bash -c \"$dockerpull && $dockerstop && $dockerrm && $dockercreate\""
command="$contractpath && $user && $dpull"
if ! sshskp $sshuser@$hostaddr $command 1>/dev/null; then
printf "$PRINTFORMAT" "$nodeno" "Error occured pulling $image."
else
@@ -293,7 +326,7 @@ if [ $mode == "docker-pull" ]; then
exit 0
fi
if [ $mode == "create" ]; then
if [ $mode == "create" ] || [ $mode == "createall" ]; then
# Read owner pubkey, contract id and image
ownerpubkey=$(echo $continfo | jq -r '.owner_pubkey')
if [ "$ownerpubkey" = "" ] || [ "$ownerpubkey" = "null" ]; then
@@ -329,13 +362,17 @@ if [ $mode == "create" ]; then
if [ "$1" != 0 ]; then
peers=""
for ((i = 0; i < $1; i++)); do
hostinfo=$(echo $continfo | jq -r ".hosts.\"${hostaddrs[$i]}\"")
peerport=$(echo $hostinfo | jq -r '.peer_port')
if [ -z "$2" ]; then
hostinfo=$(echo $continfo | jq -r ".hosts.\"${hostaddrs[$i]}\"")
peerport=$(echo $hostinfo | jq -r '.peer_port')
if [ "$hostinfo" == "" ] || [ "$hostinfo" == "null" ] ||
[ "$peerport" == "" ] || [ "$peerport" == "null" ]; then
echo "Host info is empty for ${hostaddrs[$i]}"
exit 1
if [ "$hostinfo" == "" ] || [ "$hostinfo" == "null" ] ||
[ "$peerport" == "" ] || [ "$peerport" == "null" ]; then
echo "Host info is empty for ${hostaddrs[$i]}"
exit 1
fi
else
peerport=$2
fi
peers+="\"${hostaddrs[$i]}:$peerport\","
done
@@ -365,13 +402,28 @@ if [ $mode == "create" ]; then
fi
}
if [ $nodeid = -1 ]; then
if [ $mode == "create" ]; then
if [ $nodeid = -1 ]; then
for i in "${!hostaddrs[@]}"; do
createinstance $i
done
else
createinstance $nodeid $peerport
fi
else
# Create all instances parallely with specified peer port.
peerport=$2
[ -z "$peerport" ] && echo "Peer port is required." && exit 1
for i in "${!hostaddrs[@]}"; do
createinstance $i
if [ $i == "0" ]; then
# Create first instance sequentially so others can get its public key for their unl.
echo "Creating first instance..."
createinstance $i $peerport
else
createinstance $i $peerport &
fi
done
wait
else
createinstance $nodeid
fi
exit 0
fi
@@ -531,3 +583,246 @@ if [ $mode == "destroy" ]; then
fi
exit 0
fi
if [ $mode = "ssh" ]; then
if [ $nodeid = -1 ]; then
if [ -n "$2" ]; then
# Interpret second arg as a command to execute on all nodes.
command=${*:2}
echo "Executing '$command' on all nodes..."
for i in "${!hostaddrs[@]}"; do
hostaddr=${hostaddrs[i]}
let n=$i+1
echo "node"$n":" $(sshskp $sshuser@$hostaddr $command) &
done
wait
exit 0
else
echo "Please specify node no. or command to execute on all nodes."
exit 1
fi
else
hostaddr=${hostaddrs[$nodeid]}
sshskp -t $sshuser@$hostaddr
exit 0
fi
fi
if [ $mode == "sshu" ]; then
function sshwithuser() {
hostaddr=${hostaddrs[$1]}
nodeno=$(expr $1 + 1)
containername=$(echo $continfo | jq -r ".hosts.\"$hostaddr\".name")
username=$(sshskp $sshuser@$hostaddr "sashi list | grep $containername | awk '{ print \$2 }'")
ssh_command="cd /home/$username/$containername ; sudo -u $username bash"
sshskp -t $sshuser@$hostaddr $ssh_command
}
if [ $nodeid = -1 ]; then
echo "Must specify node no."
exit 1
else
sshwithuser $nodeid
fi
exit 0
fi
if [ $mode == "attach" ]; then
function attachdocker() {
hostaddr=${hostaddrs[$1]}
nodeno=$(expr $1 + 1)
containername=$(echo $continfo | jq -r ".hosts.\"$hostaddr\".name")
username=$(sshskp $sshuser@$hostaddr "sashi list | grep $containername | awk '{ print \$2 }'")
echo "Press ctrl+P,Q to detach."
ssh_command="sudo -u $username bash -i -c 'docker attach $containername'"
sshskp -t $sshuser@$hostaddr $ssh_command
}
if [ $nodeid = -1 ]; then
echo "Must specify node no."
exit 1
else
attachdocker $nodeid
fi
exit 0
fi
if [ $mode = "ip" ]; then
if [ $nodeid = -1 ]; then
for i in "${!hostaddrs[@]}"; do
let n=$i+1
echo "node"$n": ${hostaddrs[i]}"
done
else
echo "${hostaddrs[$nodeid]}"
fi
exit 0
fi
if [ $mode == "updatecfg" ]; then
function sendcfg() {
hostaddr=${hostaddrs[$1]}
nodeno=$(expr $1 + 1)
containername=$(echo $continfo | jq -r ".hosts.\"$hostaddr\".name")
username=$(sshskp $sshuser@$hostaddr "sashi list | grep $containername | awk '{ print \$2 }'")
originalcfg="/home/$username/$containername/cfg/hp.cfg"
scpskp -q hp.cfg $sshuser@$hostaddr:~/
sshskp $sshuser@$hostaddr "jq -s '.[0] * .[1]' $originalcfg ~/hp.cfg > ~/merged.cfg && mv ~/merged.cfg $originalcfg && chown $username:$username $originalcfg && rm ~/hp.cfg"
echo "node$nodeno: Updated $originalcfg"
}
if [ $nodeid = -1 ]; then
for i in "${!hostaddrs[@]}"; do
sendcfg $i &
done
wait
else
sendcfg $nodeid
fi
exit 0
fi
if [ $mode == "statefile" ]; then
function sendstatefile() {
localfilepath=$2
filename=$(basename $2)
hostaddr=${hostaddrs[$1]}
nodeno=$(expr $1 + 1)
containername=$(echo $continfo | jq -r ".hosts.\"$hostaddr\".name")
username=$(sshskp $sshuser@$hostaddr "sashi list | grep $containername | awk '{ print \$2 }'")
fspath="/home/$username/$containername/contract_fs"
seedpath="$fspath/seed/state"
scpskp -q $localfilepath $sshuser@$hostaddr:$seedpath/
sshskp $sshuser@$hostaddr "chown $username:$username $seedpath/$filename && rm -r $fspath/hmap && rm $fspath/log.hpfs"
echo "node$nodeno: Transferred to $seedpath/$filename"
}
if [ $nodeid = -1 ]; then
for i in "${!hostaddrs[@]}"; do
sendstatefile $i $2 &
done
wait
else
sendstatefile $nodeid $3
fi
exit 0
fi
if [ $mode == "umount" ]; then
function unmountfuse() {
hostaddr=${hostaddrs[$1]}
nodeno=$(expr $1 + 1)
containername=$(echo $continfo | jq -r ".hosts.\"$hostaddr\".name")
username=$(sshskp $sshuser@$hostaddr "sashi list | grep $containername | awk '{ print \$2 }'")
contractmnt="/home/$username/$containername/contract_fs/mnt"
ledgermnt="/home/$username/$containername/ledger_fs/mnt"
sshskp $sshuser@$hostaddr "fusermount -u $contractmnt ; fusermount -u $ledgermnt"
echo "node$nodeno: Unmount complete."
}
if [ $nodeid = -1 ]; then
for i in "${!hostaddrs[@]}"; do
unmountfuse $i &
done
wait
else
unmountfuse $nodeid
fi
exit 0
fi
function downloadNode() {
hostaddr=${hostaddrs[$1]}
nodeno=$(expr $1 + 1)
containername=$(echo $continfo | jq -r ".hosts.\"$hostaddr\".name")
username=$(sshskp $sshuser@$hostaddr "sashi list | grep $containername | awk '{ print \$2 }'")
contractfs="/home/$username/$containername/contract_fs"
ledgerfs="/home/$username/$containername/ledger_fs"
echo "Downloading from node$nodeno"
rm -r contract_fs > /dev/null 2>&1
mkdir contract_fs
scpskp -r -q $sshuser@$hostaddr:$contractfs/seed contract_fs/
rm -r ledger_fs > /dev/null 2>&1
mkdir ledger_fs
scpskp -r -q $sshuser@$hostaddr:$ledgerfs/seed ledger_fs/
echo "Download complete."
}
function uploadNode() {
hostaddr=${hostaddrs[$1]}
nodeno=$(expr $1 + 1)
containername=$(echo $continfo | jq -r ".hosts.\"$hostaddr\".name")
username=$(sshskp $sshuser@$hostaddr "sashi list | grep $containername | awk '{ print \$2 }'")
contractfs="/home/$username/$containername/contract_fs"
ledgerfs="/home/$username/$containername/ledger_fs"
sshskp $sshuser@$hostaddr "rm -r $contractfs/{seed,hmap,log.hpfs} ; rm -r $ledgerfs/{seed,hmap,log.hpfs}"
echo "node$nodeno: Uploading to $contractfs/"
scpskp -r -q contract_fs/seed $sshuser@$hostaddr:$contractfs/
echo "node$nodeno: Uploading to $ledgerfs/"
scpskp -r -q ledger_fs/seed $sshuser@$hostaddr:$ledgerfs/
sshskp $sshuser@$hostaddr "chown -R $username:$username $contractfs/seed ; chown -R $username:$username $ledgerfs/seed"
echo "node$nodeno: Upload complete."
}
if [ $mode == "backup" ]; then
if [ $nodeid = -1 ]; then
echo "Must specify node no."
exit 1
else
downloadNode $nodeid
fi
exit 0
fi
if [ $mode == "restore" ]; then
if [ $nodeid = -1 ]; then
for i in "${!hostaddrs[@]}"; do
uploadNode $i &
done
wait
else
uploadNode $nodeid
fi
exit 0
fi
if [ $mode == "syncwith" ]; then
if [ $nodeid = -1 ]; then
echo "Must specify node no."
exit 1
else
downloadNode $nodeid
for i in "${!hostaddrs[@]}"; do
if [ "$i" != $nodeid ]; then
uploadNode $i &
fi
done
wait
rm -r ledger_fs
rm -r contract_fs
fi
exit 0
fi