From d258f2ebae01e2a48b865e10bee4e04ffa5f937a Mon Sep 17 00:00:00 2001 From: Kavindu Bimsara Fernando <40093425+BimsaraFernando@users.noreply.github.com> Date: Thu, 13 Jun 2024 17:05:59 +0530 Subject: [PATCH] Allow more ports per instance (#381) --- README.md | 2 +- dependencies/user-install.sh | 53 ++++++++++++++++++++----- dependencies/user-uninstall.sh | 38 +++++++++++++++++- evernode-bootstrap-contract | 2 +- evernode-reputation-contract | 1 + installer/sashimono-install.sh | 42 ++++++++++---------- installer/setup.sh | 40 +++++++++++++++++-- src/conf.cpp | 20 +++++++++- src/conf.hpp | 4 +- src/hp_manager.cpp | 72 +++++++++++++++++++++++++++++++--- src/hp_manager.hpp | 9 ++++- src/main.cpp | 20 +++++----- src/msg/json/msg_json.cpp | 16 ++++++++ src/sqlite.cpp | 52 +++++++++++++++--------- 14 files changed, 296 insertions(+), 75 deletions(-) create mode 160000 evernode-reputation-contract diff --git a/README.md b/README.md index adc2afe..b52e48b 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ Run `make installer` ('installer.tar.gz' will be placed in build directory) ## Run Sashimono 1. `./build/sagent new ` (This will create the Sashimono config in build directory. You only have to do this once) - 1. Example: `sudo ./build/sagent new ./build 127.0.0.1 22861 26201 0 3 900000 1048576 3145728 5242880` + 1. Example: `sudo ./build/sagent new ./build 127.0.0.1 22861 26201 36525 39064 0 3 900000 1048576 3145728 5242880` 1. `sudo ./build/sagent run` ## Sashimono Client diff --git a/dependencies/user-install.sh b/dependencies/user-install.sh index 7e09ada..1ae18cb 100755 --- a/dependencies/user-install.sh +++ b/dependencies/user-install.sh @@ -12,13 +12,16 @@ contract_uid=$6 contract_gid=$7 peer_port=$8 user_port=$9 -docker_image=${10} -docker_registry=${11} -outbound_ipv6=${12} -outbound_net_interface=${13} +gp_tcp_port_start=${10} +gp_udp_port_start=${11} +docker_image=${12} +docker_registry=${13} +outbound_ipv6=${14} +outbound_net_interface=${15} + if [ -z "$cpu" ] || [ -z "$memory" ] || [ -z "$swapmem" ] || [ -z "$disk" ] || [ -z "$contract_dir" ] || - [ -z "$contract_uid" ] || [ -z "$contract_gid" ] || [ -z "$peer_port" ] || [ -z "$user_port" ] || + [ -z "$contract_uid" ] || [ -z "$contract_gid" ] || [ -z "$peer_port" ] || [ -z "$user_port" ] || [ -z "$gp_udp_port_start" ] || [ -z "$gp_tcp_port_start" ] || [ -z "$docker_image" ] || [ -z "$docker_registry" ] || [ -z "$outbound_ipv6" ] || [ -z "$outbound_net_interface" ]; then echo "INVALID_PARAMS,INST_ERR" && exit 1 fi @@ -35,6 +38,8 @@ docker_bin=$script_dir/dockerbin docker_service="docker.service" docker_pull_timeout_secs=120 cleanup_script=$user_dir/uninstall_cleanup.sh +gp_udp_port_count=2 +gp_tcp_port_count=2 # Check if users already exists. [ "$(id -u "$user" 2>/dev/null || echo -1)" -ge 0 ] && echo "HAS_USER,INST_ERR" && exit 1 @@ -137,7 +142,7 @@ rule_list=$(sudo ufw status) comment=$prefix-$contract_dir # Add rules for user port. -sed -n -r -e "/${$user_port}\/tcp\s*ALLOW\s*Anywhere/{q100}" <<<"$rule_list" +sed -n -r -e "/${user_port}\/tcp\s*ALLOW\s*Anywhere/{q100}" <<<"$rule_list" res=$? if [ ! $res -eq 100 ]; then user_port_comment=$comment-user @@ -148,7 +153,7 @@ else fi # Add rules for peer port. -sed -n -r -e "/${$peer_port}\s*ALLOW\s*Anywhere/{q100}" <<<"$rule_list" +sed -n -r -e "/${peer_port}\s*ALLOW\s*Anywhere/{q100}" <<<"$rule_list" res=$? if [ ! $res -eq 100 ]; then peer_port_comment=$comment-peer @@ -158,6 +163,34 @@ else echo "Peer port rule already exists. Skipping." fi +# Add rules for general purpose udp ports. +for ((i = 0; i < $gp_udp_port_count; i++)); do + gp_udp_port=$(expr $gp_udp_port_start + $i) + sed -n -r -e "/${gp_udp_port}\s*ALLOW\s*Anywhere/{q100}" <<<"$rule_list" + res=$? + if [ ! $res -eq 100 ]; then + gp_udp_port_comment=$comment-gc-udp-$i + echo "Adding new rule to allow general purpose udp port for new instance from firewall." + sudo ufw allow "$gp_udp_port" comment "$gp_udp_port_comment" + else + echo "General purpose udp port rule already exists. Skipping." + fi +done + +# Add rules for general purpose tcp ports. +for ((i = 0; i < $gp_tcp_port_count; i++)); do + gp_tcp_port=$(expr $gp_tcp_port_start + $i) + sed -n -r -e "/${gp_tcp_port}\s*ALLOW\s*Anywhere/{q100}" <<<"$rule_list" + res=$? + if [ ! $res -eq 100 ]; then + gp_tcp_port_comment=$comment-gc-tcp-$i + echo "Adding new rule to allow general purpose tcp port for new instance from firewall." + sudo ufw allow "$gp_tcp_port" comment "$gp_tcp_port_comment" + else + echo "General purpose tcp rule already exists. Skipping." + fi +done + echo "Installing rootless dockerd for user." sudo -H -u "$user" PATH="$docker_bin":"$PATH" XDG_RUNTIME_DIR="$user_runtime_dir" "$docker_bin"/dockerd-rootless-setuptool.sh install @@ -171,7 +204,7 @@ echo "[Service] # We need to enable ipv6 configurations if outbound ipv6 address is specified. if [ "$outbound_ipv6" != "-" ] && [ "$outbound_net_interface" != "-" ]; then - + # Pass the relevant ipv6 parameters to rootlesskit flags. rootlesskit will in turn pass these to slirp4nets. # Also apply ipv6 route configuration patch in the dockerd process namespace (credits: https://github.com/containers/podman/issues/15850#issuecomment-1320028298) echo " @@ -188,7 +221,7 @@ if [ "$outbound_ipv6" != "-" ] && [ "$outbound_net_interface" != "-" ]; then \"ip6tables\": true, \"mtu\": 65520 }" >$user_dir/.config/docker/daemon.json - + # Add the outbound ipv6 address to the specified network interface. ip addr add $outbound_ipv6 dev $outbound_net_interface @@ -261,4 +294,4 @@ echo "Setting up user cgroup resources." echo "Configured user cgroup resources." echo "$user_id,$user,$dockerd_socket,INST_SUC" -exit 0 +exit 0 \ No newline at end of file diff --git a/dependencies/user-uninstall.sh b/dependencies/user-uninstall.sh index 31c94d3..11bc3c1 100755 --- a/dependencies/user-uninstall.sh +++ b/dependencies/user-uninstall.sh @@ -5,10 +5,14 @@ user=$1 peer_port=$2 user_port=$3 -instance_name=$4 +gp_tcp_port_start=$4 +gp_udp_port_start=$5 +instance_name=$6 prefix="sashi" max_kill_attempts=5 + + # Check whether this is a valid sashimono username. [ ${#user} -lt 24 ] || [ ${#user} -gt 32 ] || [[ ! "$user" =~ ^$prefix[0-9]+$ ]] && echo "ARGS,UNINST_ERR" && exit 1 @@ -28,6 +32,8 @@ user_runtime_dir="/run/user/$user_id" script_dir=$(dirname "$(realpath "$0")") docker_bin=$script_dir/dockerbin cleanup_script=$user_dir/uninstall_cleanup.sh +gp_udp_port_count=2 +gp_tcp_port_count=2 echo "Uninstalling user '$user'." @@ -101,6 +107,34 @@ else echo "Peer port rule not added by Sashimono. Skipping.." fi +# Remove rules for general purpose udp port. +for ((i = 0; i < $gp_udp_port_count; i++)); do + gp_udp_port=$(expr $gp_udp_port_start + $i) + gp_udp_port_comment=$comment-gc-udp-$i + sed -n -r -e "/${gp_udp_port_comment}/{q100}" <<<"$rule_list" + res=$? + if [ $res -eq 100 ]; then + echo "Deleting general purpose udp port rule for instance from firewall." + sudo ufw delete allow "$gp_udp_port" + else + echo "General purpose tcp port rule not added by Sashimono. Skipping.." + fi +done + +# Remove rules for general purpose tcp port. +for ((i = 0; i < $gp_tcp_port_count; i++)); do + gp_tcp_port=$(expr $gp_tcp_port_start + $i) + gp_tcp_port_comment=$comment-gc-tcp-$i + sed -n -r -e "/${gp_tcp_port_comment}/{q100}" <<<"$rule_list" + res=$? + if [ $res -eq 100 ]; then + echo "Deleting general purpose tcp port rule for instance from firewall." + sudo ufw delete allow "$gp_tcp_port" + else + echo "General purpose tcp port rule not added by Sashimono. Skipping.." + fi +done + echo "Deleting contract user '$contract_user'" userdel "$contract_user" @@ -119,4 +153,4 @@ rm -r /home/"${user:?}" [ -d /home/"$user" ] && echo "NOT_CLEAN,UNINST_ERR" && exit 1 echo "UNINST_SUC" -exit 0 +exit 0 \ No newline at end of file diff --git a/evernode-bootstrap-contract b/evernode-bootstrap-contract index 13b6f70..b532568 160000 --- a/evernode-bootstrap-contract +++ b/evernode-bootstrap-contract @@ -1 +1 @@ -Subproject commit 13b6f708bf86af5a8518d27f7573291a5fdeeaab +Subproject commit b5325680455943694ee0d804092386b7b268687b diff --git a/evernode-reputation-contract b/evernode-reputation-contract new file mode 160000 index 0000000..c69fb62 --- /dev/null +++ b/evernode-reputation-contract @@ -0,0 +1 @@ +Subproject commit c69fb62cbe26ab7c76b5e7ceccda74dede85816c diff --git a/installer/sashimono-install.sh b/installer/sashimono-install.sh index 2695c8a..46e588d 100755 --- a/installer/sashimono-install.sh +++ b/installer/sashimono-install.sh @@ -7,25 +7,27 @@ inetaddr=${1} init_peer_port=${2} init_user_port=${3} -country_code=${4} -total_instance_count=${5} -cpu_micro_sec=${6} -ram_kb=${7} -swap_kb=${8} -disk_kb=${9} -lease_amount=${10} -rippled_server=${11} -xrpl_account_address=${12} -xrpl_account_secret_path=${13} -email_address=${14} -tls_key_file=${15} -tls_cert_file=${16} -tls_cabundle_file=${17} -description=${18} -ipv6_subnet=${19} -ipv6_net_interface=${20} -extra_txn_fee=${21} -fallback_rippled_servers=${22} +init_gp_tcp_port=${4} +init_gp_udp_port=${5} +country_code=${6} +total_instance_count=${7} +cpu_micro_sec=${8} +ram_kb=${9} +swap_kb=${10} +disk_kb=${11} +lease_amount=${12} +rippled_server=${13} +xrpl_account_address=${14} +xrpl_account_secret_path=${15} +email_address=${16} +tls_key_file=${17} +tls_cert_file=${18} +tls_cabundle_file=${19} +description=${20} +ipv6_subnet=${21} +ipv6_net_interface=${22} +extra_txn_fee=${23} +fallback_rippled_servers=${24} script_dir=$(dirname "$(realpath "$0")") desired_slirp4netns_version="1.2.1" @@ -597,7 +599,7 @@ if [ -f $SASHIMONO_DATA/sa.cfg ]; then echo "Existing Sashimono data directory found. Updating..." ! $SASHIMONO_BIN/sagent upgrade $SASHIMONO_DATA && abort elif [ ! -f "$SASHIMONO_CONFIG" ]; then - ! $SASHIMONO_BIN/sagent new $SASHIMONO_DATA $inetaddr $init_peer_port $init_user_port $DOCKER_REGISTRY_PORT \ + ! $SASHIMONO_BIN/sagent new $SASHIMONO_DATA $inetaddr $init_peer_port $init_user_port $init_gp_tcp_port $init_gp_udp_port $DOCKER_REGISTRY_PORT \ $total_instance_count $cpu_micro_sec $ram_kb $swap_kb $disk_kb && abort fi diff --git a/installer/setup.sh b/installer/setup.sh index 046ec44..fa38a33 100755 --- a/installer/setup.sh +++ b/installer/setup.sh @@ -26,7 +26,7 @@ log_dir=/tmp/evernode reputationd_script_dir=$(dirname "$(realpath "$0")") root_user="root" - + repo_owner="EvernodeXRPL" repo_name="evernode-resources" desired_branch="main" @@ -587,6 +587,35 @@ done } + function set_init_gp_ports() { + + # Take default ports in interactive mode or if 'default' is specified. + # Picked default ports according to https://www.iana.org/assignments/service-names-port-numbers/service-names-port-numbers.xhtml + # (36525-36601) and (39064-39680) ranges are unassigned. + + # Default starting ports. + init_gp_tcp_port=36525 + init_gp_udp_port=39064 + gp_tcp_port_count=2 + gp_udp_port_count=2 + + if [ -n "$init_gp_tcp_port" ] && [ -n "$init_gp_udp_port" ] && confirm "Selected default general purpose port ranges (TCP: $init_gp_tcp_port-$((init_gp_tcp_port + gp_tcp_port_count * alloc_instcount)), UDP: $init_gp_udp_port-$((init_gp_udp_port + gp_udp_port_count * alloc_instcount))). + This needs to be publicly reachable over internet. \n\nAre these the ports you want to use?"; then + return 0 + fi + + init_gp_tcp_port="" + init_gp_udp_port="" + while [ -z "$init_gp_tcp_port" ]; do + read -ep "Please specify the starting port of the public 'General purpose TCP port range' your server is reachable at: " init_gp_tcp_port /etc/systemd/system/$EVERNODE_AUTO_UPDATE_SERVICE.timer # Filter logs with STAGE prefix and ommit the prefix when echoing. # If STAGE log contains -p arg, move the cursor to previous log line and overwrite the log. - ! UPGRADE=$upgrade EVERNODE_REGISTRY_ADDRESS=$registry_address ./sashimono-install.sh $inetaddr $init_peer_port $init_user_port $countrycode $alloc_instcount \ + ! UPGRADE=$upgrade EVERNODE_REGISTRY_ADDRESS=$registry_address ./sashimono-install.sh $inetaddr $init_peer_port $init_user_port $init_gp_tcp_port $init_gp_udp_port $countrycode $alloc_instcount \ $alloc_cpu $alloc_ramKB $alloc_swapKB $alloc_diskKB $lease_amount $rippled_server $xrpl_address $key_file_path $email_address \ $tls_key_file $tls_cert_file $tls_cabundle_file $description $ipv6_subnet $ipv6_net_interface $extra_txn_fee $fallback_rippled_servers 2>&1 | tee -a >(stdbuf --output=L grep -v "\[INFO\]" | awk '{ cmd="date -u +\"%Y-%m-%d %H:%M:%S\""; cmd | getline utc_time; close(cmd); print utc_time, $0 }' >>$logfile) | stdbuf --output=L grep -E '\[STAGE\]|\[INFO\]' | @@ -2301,6 +2332,9 @@ WantedBy=timers.target" >/etc/systemd/system/$EVERNODE_AUTO_UPDATE_SERVICE.timer [ ! -f "$SASHIMONO_CONFIG" ] && set_init_ports echo -e "Using peer port range $init_peer_port-$((init_peer_port + alloc_instcount)) and user port range $init_user_port-$((init_user_port + alloc_instcount))).\n" + + [ ! -f "$SASHIMONO_CONFIG" ] && set_init_gp_ports + echo -e "Using General purpose TCP port range $init_gp_tcp_port-$((init_gp_tcp_port + gp_tcp_port_count * alloc_instcount)) and general purpose UDP port range $init_gp_udp_port-$((init_gp_udp_port + gp_udp_port_count * alloc_instcount))).\n" [ ! -f "$MB_XRPL_CONFIG" ] && set_lease_amount echo -e "Lease amount set as $lease_amount EVRs per Moment.\n" diff --git a/src/conf.cpp b/src/conf.cpp index faa9852..0955f4a 100644 --- a/src/conf.cpp +++ b/src/conf.cpp @@ -33,7 +33,7 @@ namespace conf * Create config here. * @return 0 for success. -1 for failure. */ - int create(std::string_view host_addr, const uint16_t init_peer_port, const uint16_t init_user_port, const uint16_t docker_registry_port, + int create(std::string_view host_addr, const uint16_t init_peer_port, const uint16_t init_user_port,const uint16_t init_gp_tcp_port, const uint16_t init_gp_udp_port, const uint16_t docker_registry_port, const size_t inst_count, const size_t cpu_us, const size_t ram_kbytes, const size_t swap_kbytes, const size_t disk_kbytes) { if (util::is_file_exists(ctx.config_file)) @@ -62,6 +62,8 @@ namespace conf cfg.hp.host_address = host_addr.empty() ? "127.0.0.1" : std::string(host_addr); cfg.hp.init_peer_port = !init_peer_port ? 22861 : init_peer_port; cfg.hp.init_user_port = !init_user_port ? 26201 : init_user_port; + cfg.hp.init_gp_tcp_port = !init_gp_tcp_port ? 36525 : init_gp_tcp_port; + cfg.hp.init_gp_udp_port = !init_gp_udp_port ? 39064 : init_gp_udp_port; cfg.system.max_instance_count = !inst_count ? 3 : inst_count; cfg.system.max_mem_kbytes = !ram_kbytes ? 1048576 : ram_kbytes; @@ -229,6 +231,20 @@ namespace conf std::cerr << "Configured init user port invalid. Should be greater than 1024\n"; return -1; } + + cfg.hp.init_gp_tcp_port = hp["init_gp_tcp_port"].as(); + if (cfg.hp.init_gp_tcp_port <= 1024) + { + std::cerr << "Configured init general purpose tcp port invalid. Should be greater than 1024\n"; + return -1; + } + + cfg.hp.init_gp_udp_port = hp["init_gp_udp_port"].as(); + if (cfg.hp.init_gp_udp_port <= 1024) + { + std::cerr << "Configured init general purpose udp port invalid. Should be greater than 1024\n"; + return -1; + } } catch (const std::exception &e) { @@ -324,6 +340,8 @@ namespace conf hp_config.insert_or_assign("host_address", cfg.hp.host_address); hp_config.insert_or_assign("init_peer_port", cfg.hp.init_peer_port); hp_config.insert_or_assign("init_user_port", cfg.hp.init_user_port); + hp_config.insert_or_assign("init_gp_tcp_port", cfg.hp.init_gp_tcp_port); + hp_config.insert_or_assign("init_gp_udp_port", cfg.hp.init_gp_udp_port); d.insert_or_assign("hp", hp_config); } diff --git a/src/conf.hpp b/src/conf.hpp index 04bface..4ac0edf 100644 --- a/src/conf.hpp +++ b/src/conf.hpp @@ -71,6 +71,8 @@ namespace conf std::string host_address; uint16_t init_peer_port = 0; uint16_t init_user_port = 0; + uint16_t init_gp_tcp_port = 0; + uint16_t init_gp_udp_port = 0; }; struct system_config @@ -125,7 +127,7 @@ namespace conf int init(); - int create(std::string_view host_addr, const uint16_t init_peer_port, const uint16_t init_user_port, const uint16_t docker_registry_port, + int create(std::string_view host_addr, const uint16_t init_peer_port, const uint16_t init_user_port,const uint16_t init_gp_tcp_port, const uint16_t init_gp_udp_port, const uint16_t docker_registry_port, const size_t inst_count, const size_t cpu_us, const size_t ram_kbytes, const size_t swap_kbytes, const size_t disk_kbytes); void set_dir_paths(std::string exepath, std::string datadir); diff --git a/src/hp_manager.cpp b/src/hp_manager.cpp index 2efee50..d8f3cf0 100644 --- a/src/hp_manager.cpp +++ b/src/hp_manager.cpp @@ -32,7 +32,7 @@ namespace hp // We keep docker logs at size limit of 10mb, We only need these logs for docker instance failure debugging since all other logs are kept in files. // For the local log driver compression, minimum max-file should be 2. So we keep two logs each max-size is 5mb constexpr const char *DOCKER_CREATE = "DOCKER_HOST=unix:///run/user/$(id -u %s)/docker.sock timeout --foreground -v -s SIGINT %ss %s/dockerbin/docker create -t -i --stop-signal=SIGINT --log-driver local \ - --log-opt max-size=5m --log-opt max-file=2 --name=%s -p %s:%s -p %s:%s -p %s:%s/udp --restart unless-stopped --mount type=bind,source=%s,target=/contract %s run /contract"; + --log-opt max-size=5m --log-opt max-file=2 --name=%s -p %s:%s -p %s:%s -p %s:%s/udp -p %s:%s -p %s:%s -p %s:%s/udp -p %s:%s/udp --restart unless-stopped --mount type=bind,source=%s,target=/contract %s run /contract"; constexpr const char *DOCKER_START = "DOCKER_HOST=unix:///run/user/$(id -u %s)/docker.sock %s/dockerbin/docker start %s"; constexpr const char *DOCKER_STOP = "DOCKER_HOST=unix:///run/user/$(id -u %s)/docker.sock %s/dockerbin/docker stop %s"; constexpr const char *DOCKER_REMOVE = "DOCKER_HOST=unix:///run/user/$(id -u %s)/docker.sock %s/dockerbin/docker rm -f %s"; @@ -88,8 +88,7 @@ namespace hp } // Populate the vacant ports vector with vacant ports of destroyed containers. - sqlite::get_vacant_ports(db, vacant_ports); - + get_vacant_ports_list(vacant_ports); // Calculate the resources per instance. instance_resources.cpu_us = conf::cfg.system.max_cpu_us / conf::cfg.system.max_instance_count; instance_resources.mem_kbytes = conf::cfg.system.max_mem_kbytes / conf::cfg.system.max_instance_count; @@ -182,7 +181,7 @@ namespace hp sqlite::get_max_ports(db, last_assigned_ports); last_port_assign_from_vacant = false; } - instance_ports = {(uint16_t)(last_assigned_ports.peer_port + 1), (uint16_t)(last_assigned_ports.user_port + 1)}; + instance_ports = {(uint16_t)(last_assigned_ports.peer_port + 1), (uint16_t)(last_assigned_ports.user_port + 1), (uint16_t)(last_assigned_ports.gp_tcp_port_start + 2), (uint16_t)(last_assigned_ports.gp_udp_port_start + 2) }; } int user_id; @@ -313,11 +312,23 @@ namespace hp { const std::string user_port = std::to_string(assigned_ports.user_port); const std::string peer_port = std::to_string(assigned_ports.peer_port); + const std::string gp_tcp_port_1 = std::to_string(assigned_ports.gp_tcp_port_start); + const std::string gp_tcp_port_2 = std::to_string(assigned_ports.gp_tcp_port_start + 1 ); + const std::string gp_udp_port_1 = std::to_string(assigned_ports.gp_udp_port_start); + const std::string gp_udp_port_2 = std::to_string(assigned_ports.gp_udp_port_start + 1 ); const std::string timeout = std::to_string(DOCKER_CREATE_TIMEOUT_SECS); - const int len = 376 + username.length() + timeout.length() + conf::ctx.exe_dir.length() + container_name.length() + (user_port.length() * 2) + (peer_port.length() * 4) + contract_dir.length() + image_name.length(); + const int len = 376 + username.length() + timeout.length() + conf::ctx.exe_dir.length() + container_name.length() + (user_port.length() * 2) + (peer_port.length() * 4) + (gp_tcp_port_1.length() * 2) + (gp_tcp_port_2.length() * 2) + (gp_udp_port_1.length() * 2) + (gp_udp_port_2.length() * 2) + contract_dir.length() + image_name.length(); char command[len]; sprintf(command, DOCKER_CREATE, username.data(), timeout.data(), conf::ctx.exe_dir.data(), container_name.data(), - user_port.data(), user_port.data(), peer_port.data(), peer_port.data(), peer_port.data(), peer_port.data(), contract_dir.data(), image_name.data()); + user_port.data(), user_port.data(), + peer_port.data(), peer_port.data(), + peer_port.data(), peer_port.data(), + gp_tcp_port_1.data(), gp_tcp_port_1.data(), + gp_tcp_port_2.data(), gp_tcp_port_2.data(), + gp_udp_port_1.data(), gp_udp_port_1.data(), + gp_udp_port_2.data(), gp_udp_port_2.data(), + contract_dir.data(), image_name.data()); + LOG_INFO << "Creating the docker container. name: " << container_name; if (system(command) != 0) { @@ -906,6 +917,9 @@ namespace hp std::to_string(contract_ugid.gid), std::to_string(instance_ports.peer_port), std::to_string(instance_ports.user_port), + std::to_string(instance_ports.gp_tcp_port_start), + std::to_string(instance_ports.gp_udp_port_start), + docker_image, conf::cfg.docker.registry_address, outbound_ipv6, @@ -1019,7 +1033,53 @@ namespace hp return 0; } + /** + * Populate the given vector with vacant ports which are not already assigned. + * @param vacant_ports Ports vector to hold port pairs from database. + */ + void get_vacant_ports_list(std::vector &vacant_ports) + { + const int gp_tcp_port_count=2; + const int gp_udp_port_count=2; + //get all instances + std::vector instances; + get_instance_list(instances); + + //no instances + if (instances.empty()) { + return; + } + + //Get the max instance + const std::vector::iterator element_max_peer_port = std::max_element(instances.begin(), instances.end(), + [](const hp::instance_info& a, const hp::instance_info& b) { + return (uint16_t)(a.assigned_ports.user_port) < (uint16_t)(b.assigned_ports.user_port); + }); + + + ports init_ports = {(uint16_t)(conf::cfg.hp.init_peer_port), (uint16_t)(conf::cfg.hp.init_user_port), (uint16_t)(conf::cfg.hp.init_gp_tcp_port), (uint16_t)(conf::cfg.hp.init_gp_udp_port)}; + + //Keep increasing init port (peer port) until it reaches max port + //If init port values did not match with an item in the instances list, add init port values to vacant ports list. + while (init_ports.peer_port < element_max_peer_port->assigned_ports.peer_port) + { + + bool is_item_available = std::find_if(instances.begin(),instances.end(),[init_ports](const instance_info& instance){ + return instance.assigned_ports.peer_port == init_ports.peer_port; + }) != instances.end(); + + if(!is_item_available){ + vacant_ports.push_back(init_ports); + } + + init_ports.peer_port++; + init_ports.user_port++; + init_ports.gp_tcp_port_start+=gp_tcp_port_count; + init_ports.gp_udp_port_start+=gp_udp_port_count; + } + + } /** * Check whether there's a pending reboot and cgrules service is running and configured. * @return true if active and configured otherwise false. diff --git a/src/hp_manager.hpp b/src/hp_manager.hpp index ac073d5..1e0d60a 100644 --- a/src/hp_manager.hpp +++ b/src/hp_manager.hpp @@ -20,15 +20,18 @@ namespace hp EXITED }; - // Stores port pair assigned to a container. + // Stores ports assigned to a container. struct ports { uint16_t peer_port = 0; uint16_t user_port = 0; + uint16_t gp_tcp_port_start = 0; + uint16_t gp_udp_port_start = 0; + bool operator==(const ports &other) const { - return peer_port == other.peer_port && user_port == other.user_port; + return peer_port == other.peer_port && user_port == other.user_port && gp_tcp_port_start == other.gp_tcp_port_start && gp_udp_port_start == other.gp_udp_port_start; } }; @@ -109,5 +112,7 @@ namespace hp bool system_ready(); + void get_vacant_ports_list(std::vector &vacant_ports); + } // namespace hp #endif \ No newline at end of file diff --git a/src/main.cpp b/src/main.cpp index 94df75e..c7a49f0 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -38,7 +38,7 @@ int parse_cmd(int argc, char **argv) { conf::ctx.command = argv[1]; - if ((conf::ctx.command == "new" && argc >= 2 && argc <= 12) || + if ((conf::ctx.command == "new" && argc >= 2 && argc <= 14) || (conf::ctx.command == "run" && argc >= 2 && argc <= 3) || (conf::ctx.command == "upgrade" && argc >= 2 && argc <= 3) || (conf::ctx.command == "version" && argc == 2) || @@ -134,18 +134,20 @@ int main(int argc, char **argv) // This will create a new config. const std::string host_addr = (argc >= 4) ? argv[3] : ""; - uint16_t init_peer_port = 0, init_user_port = 0, docker_registry_port = 0; + uint16_t init_peer_port = 0, init_user_port = 0, init_gp_tcp_port = 0, init_gp_udp_port = 0, docker_registry_port = 0; size_t inst_count = 0, cpu_us = 0, ram_kbytes = 0, swap_kbytes = 0, disk_kbytes = 0; if (((argc >= 5) && util::stoul(argv[4], init_peer_port) != 0) || ((argc >= 6) && util::stoul(argv[5], init_user_port) != 0) || - ((argc >= 7) && util::stoul(argv[6], docker_registry_port) != 0) || - ((argc >= 8) && (util::stoull(argv[7], inst_count) != 0 || inst_count == 0)) || - ((argc >= 9) && (util::stoull(argv[8], cpu_us) != 0 || cpu_us == 0)) || - ((argc >= 10) && (util::stoull(argv[9], ram_kbytes) != 0 || ram_kbytes == 0)) || - ((argc >= 11) && (util::stoull(argv[10], swap_kbytes) != 0 || swap_kbytes == 0)) || - ((argc >= 12) && (util::stoull(argv[11], disk_kbytes) != 0 || disk_kbytes == 0)) || - conf::create(host_addr, init_peer_port, init_user_port, docker_registry_port, inst_count, cpu_us, ram_kbytes, swap_kbytes, disk_kbytes) != 0) + ((argc >= 7) && util::stoul(argv[6], init_gp_tcp_port) != 0) || + ((argc >= 8) && util::stoul(argv[7], init_gp_udp_port) != 0) || + ((argc >= 9) && util::stoul(argv[8], docker_registry_port) != 0) || + ((argc >= 10) && (util::stoull(argv[9], inst_count) != 0 || inst_count == 0)) || + ((argc >= 11) && (util::stoull(argv[10], cpu_us) != 0 || cpu_us == 0)) || + ((argc >= 12) && (util::stoull(argv[11], ram_kbytes) != 0 || ram_kbytes == 0)) || + ((argc >= 13) && (util::stoull(argv[12], swap_kbytes) != 0 || swap_kbytes == 0)) || + ((argc >= 14) && (util::stoull(argv[13], disk_kbytes) != 0 || disk_kbytes == 0)) || + conf::create(host_addr, init_peer_port, init_user_port, init_gp_tcp_port, init_gp_udp_port, docker_registry_port, inst_count, cpu_us, ram_kbytes, swap_kbytes, disk_kbytes) != 0) { std::cerr << "Invalid Sashimono Agent config creation args.\n"; std::cerr << docker_registry_port << ", " << inst_count << ", " << cpu_us << ", " << ram_kbytes << ", " diff --git a/src/msg/json/msg_json.cpp b/src/msg/json/msg_json.cpp index 4b74e9f..c9b15a2 100644 --- a/src/msg/json/msg_json.cpp +++ b/src/msg/json/msg_json.cpp @@ -709,6 +709,14 @@ namespace msg::json msg += "user_port"; msg += SEP_COLON; msg += std::to_string(info.assigned_ports.user_port); + msg += SEP_COMMA; + msg += "gp_tcp_port"; + msg += SEP_COLON; + msg += std::to_string(info.assigned_ports.gp_tcp_port_start); + msg += SEP_COMMA; + msg += "gp_udp_port"; + msg += SEP_COLON; + msg += std::to_string(info.assigned_ports.gp_udp_port_start); msg += "\"}"; } @@ -773,6 +781,14 @@ namespace msg::json msg += "user_port"; msg += SEP_COLON_NOQUOTE; msg += std::to_string(instance.assigned_ports.user_port); + msg += SEP_COMMA_NOQUOTE; + msg += "gp_tcp_port"; + msg += SEP_COLON_NOQUOTE; + msg += std::to_string(instance.assigned_ports.gp_tcp_port_start); + msg += SEP_COMMA_NOQUOTE; + msg += "gp_udp_port"; + msg += SEP_COLON_NOQUOTE; + msg += std::to_string(instance.assigned_ports.gp_udp_port_start); // Include matching lease information. const auto lease = std::find_if(leases.begin(), leases.end(), [&](const hp::lease_info &l) diff --git a/src/sqlite.cpp b/src/sqlite.cpp index fd784d5..932dbdf 100644 --- a/src/sqlite.cpp +++ b/src/sqlite.cpp @@ -22,26 +22,26 @@ namespace sqlite constexpr const char *INSERT_INTO_HP_INSTANCE = "INSERT INTO instances(" "owner_pubkey, time, username, status, name, ip," - "peer_port, user_port, pubkey, contract_id, image_name" - ") VALUES(?,?,?,?,?,?,?,?,?,?,?)"; + "peer_port, user_port, init_gp_tcp_port, init_gp_udp_port, pubkey, contract_id, image_name" + ") VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?)"; - constexpr const char *GET_VACANT_PORTS_FROM_HP = "SELECT DISTINCT peer_port, user_port FROM " + constexpr const char *GET_VACANT_PORTS_FROM_HP = "SELECT DISTINCT peer_port, user_port, init_gp_tcp_port, init_gp_udp_port FROM " "instances WHERE status == ? AND user_port NOT IN" "(SELECT user_port FROM instances WHERE status != ?)"; - constexpr const char *GET_MAX_PORTS_FROM_HP = "SELECT max(peer_port), max(user_port) FROM instances WHERE status != ?"; + constexpr const char *GET_MAX_PORTS_FROM_HP = "SELECT max(peer_port), max(user_port), max(init_gp_tcp_port), max(init_gp_udp_port) FROM instances WHERE status != ?"; constexpr const char *UPDATE_STATUS_IN_HP = "UPDATE instances SET status = ? WHERE name = ?"; - constexpr const char *IS_CONTAINER_EXISTS = "SELECT username, status, peer_port, user_port FROM instances WHERE name = ?"; + constexpr const char *IS_CONTAINER_EXISTS = "SELECT username, status, peer_port, user_port, init_gp_tcp_port, init_gp_udp_port FROM instances WHERE name = ?"; constexpr const char *GET_ALOCATED_INSTANCE_COUNT = "SELECT COUNT(name) FROM instances WHERE status != ?"; constexpr const char *GET_RUNNING_INSTANCE_NAMES = "SELECT name FROM instances WHERE status = ?"; - constexpr const char *GET_INSTANCE_LIST = "SELECT name, username, user_port, peer_port, status, image_name, contract_id FROM instances WHERE status != ?"; + constexpr const char *GET_INSTANCE_LIST = "SELECT name, username, user_port, peer_port, init_gp_tcp_port, init_gp_udp_port, status, image_name, contract_id FROM instances WHERE status != ?"; - constexpr const char *GET_INSTANCE = "SELECT name, username, user_port, peer_port, status, image_name FROM instances WHERE name == ? AND status != ?"; + constexpr const char *GET_INSTANCE = "SELECT name, username, user_port, peer_port, init_gp_tcp_port, init_gp_udp_port, status, image_name FROM instances WHERE name == ? AND status != ?"; constexpr const char *IS_TABLE_EXISTS = "SELECT * FROM sqlite_master WHERE type='table' AND name = ?"; @@ -303,6 +303,8 @@ namespace sqlite table_column_info("ip", COLUMN_DATA_TYPE::TEXT), table_column_info("peer_port", COLUMN_DATA_TYPE::INT), table_column_info("user_port", COLUMN_DATA_TYPE::INT), + table_column_info("init_gp_tcp_port", COLUMN_DATA_TYPE::INT), + table_column_info("init_gp_udp_port", COLUMN_DATA_TYPE::INT), table_column_info("pubkey", COLUMN_DATA_TYPE::TEXT), table_column_info("contract_id", COLUMN_DATA_TYPE::TEXT), table_column_info("image_name", COLUMN_DATA_TYPE::TEXT)}; @@ -333,9 +335,11 @@ namespace sqlite sqlite3_bind_text(stmt, 6, info.ip.data(), info.ip.length(), SQLITE_STATIC) == SQLITE_OK && sqlite3_bind_int64(stmt, 7, info.assigned_ports.peer_port) == SQLITE_OK && sqlite3_bind_int64(stmt, 8, info.assigned_ports.user_port) == SQLITE_OK && - sqlite3_bind_text(stmt, 9, info.pubkey.data(), info.pubkey.length(), SQLITE_STATIC) == SQLITE_OK && - sqlite3_bind_text(stmt, 10, info.contract_id.data(), info.contract_id.length(), SQLITE_STATIC) == SQLITE_OK && - sqlite3_bind_text(stmt, 11, info.image_name.data(), info.image_name.length(), SQLITE_STATIC) == SQLITE_OK && + sqlite3_bind_int64(stmt, 9, info.assigned_ports.gp_tcp_port_start) == SQLITE_OK && + sqlite3_bind_int64(stmt, 10, info.assigned_ports.gp_udp_port_start) == SQLITE_OK && + sqlite3_bind_text(stmt, 11, info.pubkey.data(), info.pubkey.length(), SQLITE_STATIC) == SQLITE_OK && + sqlite3_bind_text(stmt, 12, info.contract_id.data(), info.contract_id.length(), SQLITE_STATIC) == SQLITE_OK && + sqlite3_bind_text(stmt, 13, info.image_name.data(), info.image_name.length(), SQLITE_STATIC) == SQLITE_OK && sqlite3_step(stmt) == SQLITE_DONE) { sqlite3_finalize(stmt); @@ -366,6 +370,8 @@ namespace sqlite info.status = std::string(reinterpret_cast(sqlite3_column_text(stmt, 1))); info.assigned_ports.peer_port = sqlite3_column_int64(stmt, 2); info.assigned_ports.user_port = sqlite3_column_int64(stmt, 3); + info.assigned_ports.gp_tcp_port_start = sqlite3_column_int64(stmt, 4); + info.assigned_ports.gp_udp_port_start = sqlite3_column_int64(stmt, 5); // Finalize and distroys the statement. sqlite3_finalize(stmt); @@ -414,13 +420,15 @@ namespace sqlite { const uint16_t peer_port = sqlite3_column_int64(stmt, 0); const uint16_t user_port = sqlite3_column_int64(stmt, 1); + const uint16_t gp_tcp_port_start = sqlite3_column_int64(stmt, 2); + const uint16_t gp_udp_port_start = sqlite3_column_int64(stmt, 3); - max_ports = {peer_port, user_port}; + max_ports = {peer_port, user_port, gp_tcp_port_start, gp_udp_port_start}; } // Initialize with default config values if either of the ports are zero. - if (max_ports.peer_port == 0 || max_ports.user_port == 0) + if (max_ports.peer_port == 0 || max_ports.user_port == 0 || max_ports.gp_tcp_port_start == 0 || max_ports.gp_udp_port_start == 0) { - max_ports = {(uint16_t)(conf::cfg.hp.init_peer_port - 1), (uint16_t)(conf::cfg.hp.init_user_port - 1)}; + max_ports = {(uint16_t)(conf::cfg.hp.init_peer_port - 1), (uint16_t)(conf::cfg.hp.init_user_port - 1), (uint16_t)(conf::cfg.hp.init_gp_tcp_port - 2), (uint16_t)(conf::cfg.hp.init_gp_udp_port - 2)}; } // Finalize and distroys the statement. @@ -446,7 +454,9 @@ namespace sqlite { const uint16_t peer_port = sqlite3_column_int64(stmt, 0); const uint16_t user_port = sqlite3_column_int64(stmt, 1); - vacant_ports.push_back({peer_port, user_port}); + const uint16_t gp_tcp_port = sqlite3_column_int64(stmt, 2); + const uint16_t gp_udp_port = sqlite3_column_int64(stmt, 3); + vacant_ports.push_back({peer_port, user_port, gp_tcp_port, gp_udp_port}); } } @@ -500,9 +510,11 @@ namespace sqlite info.username = reinterpret_cast(sqlite3_column_text(stmt, 1)); info.assigned_ports.user_port = sqlite3_column_int64(stmt, 2); info.assigned_ports.peer_port = sqlite3_column_int64(stmt, 3); - info.status = reinterpret_cast(sqlite3_column_text(stmt, 4)); - info.image_name = reinterpret_cast(sqlite3_column_text(stmt, 5)); - info.contract_id = reinterpret_cast(sqlite3_column_text(stmt, 6)); + info.assigned_ports.gp_tcp_port_start = sqlite3_column_int64(stmt, 4); + info.assigned_ports.gp_udp_port_start = sqlite3_column_int64(stmt, 5); + info.status = reinterpret_cast(sqlite3_column_text(stmt, 6)); + info.image_name = reinterpret_cast(sqlite3_column_text(stmt, 7)); + info.contract_id = reinterpret_cast(sqlite3_column_text(stmt, 8)); instances.push_back(info); } } @@ -558,8 +570,10 @@ namespace sqlite instance.username = reinterpret_cast(sqlite3_column_text(stmt, 1)); instance.assigned_ports.user_port = sqlite3_column_int64(stmt, 2); instance.assigned_ports.peer_port = sqlite3_column_int64(stmt, 3); - instance.status = reinterpret_cast(sqlite3_column_text(stmt, 4)); - instance.image_name = reinterpret_cast(sqlite3_column_text(stmt, 5)); + instance.assigned_ports.gp_tcp_port_start = sqlite3_column_int64(stmt, 4); + instance.assigned_ports.gp_udp_port_start = sqlite3_column_int64(stmt, 5); + instance.status = reinterpret_cast(sqlite3_column_text(stmt, 6)); + instance.image_name = reinterpret_cast(sqlite3_column_text(stmt, 7)); // Finalize and distroys the statement. sqlite3_finalize(stmt);