diff --git a/CMakeLists.txt b/CMakeLists.txt index c19e6e8..f3fd717 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -66,7 +66,7 @@ add_custom_target(installer COMMAND mkdir -p ./build/sashimono-installer COMMAND bash -c "cp -r ./build/{sagent,hpfs,hpws,user-install.sh,user-uninstall.sh,contract_template} ./build/sashimono-installer/" COMMAND bash -c "cp -r ./installer/{docker-install.sh,sashimono-install.sh,sashimono-uninstall.sh} ./build/sashimono-installer/" - COMMAND bash -c "cp -r ./dependencies/libblake3.so ./build/sashimono-installer/" + COMMAND bash -c "cp -r ./dependencies/{user-cgcreate.sh,libblake3.so} ./build/sashimono-installer/" COMMAND tar cfz ./build/sashimono-installer.tar.gz --directory=./build/ sashimono-installer COMMAND rm -r ./build/sashimono-installer ) diff --git a/dependencies/user-cgcreate.sh b/dependencies/user-cgcreate.sh new file mode 100755 index 0000000..8d798f8 --- /dev/null +++ b/dependencies/user-cgcreate.sh @@ -0,0 +1,71 @@ +#!/bin/bash + +datadir=$1 +if [ -z "$datadir" ]; then + echo "Invalid arguments." + echo "Expected: user-cgcreate.sh " + exit 1 +fi + +saconfig="$1/sa.cfg" +if [ ! -f "$saconfig" ]; then + echo "Config file does not exist." + echo "Run \"sagent new $datadir\" command." + exit 1 +fi + +# Calculate resourses +# jq command is used for json manipulation. +if ! command -v jq &>/dev/null; then + echo "jq utility not found. Installing.." + apt-get install -y jq >/dev/null 2>&1 +fi + +# Read config values +max_mem_kbytes=$(jq '.system.max_mem_kbytes' $saconfig) +max_cpu_us=$(jq '.system.max_cpu_us' $saconfig) +max_instance_count=$(jq '.system.max_instance_count' $saconfig) + +([ "$max_instance_count" == "" ] || [ ${#max_instance_count} -eq 0 ] || [ "$max_instance_count" -le 0 ]) && echo "max_instance_count cannot be empty." && exit 1 + +instance_mem_kbytes=0 +if [ "$max_mem_kbytes" != "" ] && [ ! ${#max_mem_kbytes} -eq 0 ] && [ "$max_mem_kbytes" -gt 0 ]; then + ! instance_mem_kbytes=$(expr $max_mem_kbytes / $max_instance_count) && echo "Max memory limit calculation error." && exit 1 +fi + +instance_cpu_us=0 +if [ "$max_cpu_us" != "" ] && [ ! ${#max_cpu_us} -eq 0 ] && [ "$max_cpu_us" -gt 0 ]; then + ! instance_cpu_us=$(expr $max_cpu_us / $max_instance_count) && echo "Max cpu limit calculation error." && exit 1 +fi + +prefix="sashi" +cgroupsuffix="-cg" +users=$(cut -d: -f1 /etc/passwd | grep "^$prefix" | sort) +readarray -t userarr <<<"$users" +validusers=() +for user in "${userarr[@]}"; do + [ ${#user} -lt 24 ] || [ ${#user} -gt 32 ] || [[ ! "$user" =~ ^$prefix[0-9]+$ ]] && continue + validusers+=("$user") +done + +has_err=0 +for user in "${validusers[@]}"; do + # Setup user cgroup. + if [ $instance_cpu_us -gt 0 ] && + ! (cgcreate -g cpu:$user$cgroupsuffix && + echo "$instance_cpu_us" > /sys/fs/cgroup/cpu/$user$cgroupsuffix/cpu.cfs_quota_us); then + echo "CPU cgroup creation for $user failed." + has_err=1 + fi + + if [ $instance_mem_kbytes -gt 0 ] && + ! (cgcreate -g memory:$user$cgroupsuffix && + echo "${instance_mem_kbytes}K" > /sys/fs/cgroup/memory/$user$cgroupsuffix/memory.limit_in_bytes && + echo "${instance_mem_kbytes}K" > /sys/fs/cgroup/memory/$user$cgroupsuffix/memory.memsw.limit_in_bytes); then + echo "Memory cgroup creation for $user failed." + has_err=1 + fi +done + +[ $has_err -eq 1 ] && exit 1 +exit 0 diff --git a/dependencies/user-install.sh b/dependencies/user-install.sh index 34c906d..5ef281d 100755 --- a/dependencies/user-install.sh +++ b/dependencies/user-install.sh @@ -7,8 +7,10 @@ cpu=$1 memory=$2 disk=$3 contract_dir=$4 -if [ -z "$cpu" ] || [ -z "$memory" ] || [ -z "$disk" ]; then - echo "Expected: user-install.sh " +contract_uid=$5 +contract_gid=$6 +if [ -z "$cpu" ] || [ -z "$memory" ] || [ -z "$disk" ] || [ -z "$contract_dir" ] || [ -z "$contract_uid" ] || [ -z "$contract_gid" ]; then + echo "Expected: user-install.sh " echo "INVALID_PARAMS,INST_ERR" && exit 1 fi @@ -47,11 +49,11 @@ user_runtime_dir="/run/user/$user_id" dockerd_socket="unix://$user_runtime_dir/docker.sock" # Setup user cgroup. -! (cgcreate -g cpu:"$user"$cgroupsuffix && - echo "$cpu" >/sys/fs/cgroup/cpu/"$user"$cgroupsuffix/cpu.cfs_quota_us) && echo rollback "CGROUP_CPU_CREAT" -! (cgcreate -g memory:"$user"$cgroupsuffix && - echo "${memory}K" >/sys/fs/cgroup/memory/"$user"$cgroupsuffix/memory.limit_in_bytes && - echo "${memory}K" >/sys/fs/cgroup/memory/"$user"$cgroupsuffix/memory.memsw.limit_in_bytes) && echo rollback "CGROUP_MEM_CREAT" +! (cgcreate -g cpu:$user$cgroupsuffix && + echo "$cpu" >/sys/fs/cgroup/cpu/$user$cgroupsuffix/cpu.cfs_quota_us) && rollback "CGROUP_CPU_CREAT" +! (cgcreate -g memory:$user$cgroupsuffix && + echo "${memory}K" >/sys/fs/cgroup/memory/$user$cgroupsuffix/memory.limit_in_bytes && + echo "${memory}K" >/sys/fs/cgroup/memory/$user$cgroupsuffix/memory.memsw.limit_in_bytes) && rollback "CGROUP_MEM_CREAT" # Adding disk quota to the new user. setquota -u -F vfsv0 "$user" "$disk" "$disk" 0 0 / @@ -83,6 +85,16 @@ svcstat=$(sudo -u "$user" XDG_RUNTIME_DIR="$user_runtime_dir" systemctl --user i echo "Installed rootless dockerd." echo "Adding hpfs services for the instance." + +# Taking the uid and gid offsets +uoffset=$(grep "^$user:[0-9]\+:[0-9]\+$" /etc/subuid | cut -d: -f2) +[ -z $uoffset ] && rollback "SUBUID_ERR" +goffset=$(grep "^$user:[0-9]\+:[0-9]\+$" /etc/subgid | cut -d: -f2) +[ -z $goffset ] && rollback "SUBGID_ERR" +hpfs_uid=$(expr $uoffset + $contract_uid) +hpfs_gid=$(expr $goffset + $contract_gid) + +# UGID will be passed to hpfs in next PBI, with resolving cgroup issue. echo "[Unit] Description=Running and monitoring contract fs. StartLimitIntervalSec=0 diff --git a/dependencies/user-uninstall.sh b/dependencies/user-uninstall.sh index 0fcdfca..51a424d 100755 --- a/dependencies/user-uninstall.sh +++ b/dependencies/user-uninstall.sh @@ -67,8 +67,8 @@ fi echo "Removing cgroups" # Delete config values. -cgdelete -g cpu:"$user"$cgroupsuffix -cgdelete -g memory:"$user"$cgroupsuffix +cgdelete -g cpu:$user$cgroupsuffix +cgdelete -g memory:$user$cgroupsuffix echo "Deleting user '$user'" userdel "$user" diff --git a/installer/sashimono-install.sh b/installer/sashimono-install.sh index 9b62ce9..a833a42 100755 --- a/installer/sashimono-install.sh +++ b/installer/sashimono-install.sh @@ -6,6 +6,7 @@ sashimono_bin=/usr/bin/sashimono-agent docker_bin=/usr/bin/sashimono-agent/dockerbin sashimono_data=/etc/sashimono sashimono_service="sashimono-agent" +cgcreate_service="sashimono-cgcreate" group="sashimonousers" cgroupsuffix="-cg" script_dir=$(dirname "$(realpath "$0")") @@ -55,7 +56,7 @@ function rollback() { } # Install Sashimono agent binaries into sashimono bin dir. -cp "$script_dir"/{sagent,hpfs,hpws,user-install.sh,user-uninstall.sh} $sashimono_bin +cp "$script_dir"/{sagent,hpfs,hpws,user-cgcreate.sh,user-install.sh,user-uninstall.sh} $sashimono_bin chmod -R +x $sashimono_bin # Download and install rootless dockerd. @@ -72,6 +73,19 @@ chmod -R +x $sashimono_bin cp -r "$script_dir"/contract_template $sashimono_data $sashimono_bin/sagent new $sashimono_data +# Install Sashimono Agent cgcreate service. +# This is a onshot service which runs only once. +echo "[Unit] +Description=Sashimono cgroup creation service. +After=network.target +[Service] +User=root +Group=root +Type=oneshot +ExecStart=$sashimono_bin/user-cgcreate.sh $sashimono_data +[Install] +WantedBy=multi-user.target" >/etc/systemd/system/$cgcreate_service.service + # Install Sashimono Agent systemd service. # StartLimitIntervalSec=0 to make unlimited retries. RestartSec=5 is to keep 5 second gap between restarts. echo "[Unit] @@ -90,8 +104,11 @@ RestartSec=5 WantedBy=multi-user.target" >/etc/systemd/system/$sashimono_service.service systemctl daemon-reload +systemctl enable $cgcreate_service +systemctl start $cgcreate_service systemctl enable $sashimono_service systemctl start $sashimono_service +# Both of these services needed to be restarted if sa.cfg max instance resources are manually changed. echo "Sashimono installed successfully." echo "Please restart your cgroup rule generator service or reboot your server for changes to apply." diff --git a/installer/sashimono-uninstall.sh b/installer/sashimono-uninstall.sh index 1b417e4..d18361f 100755 --- a/installer/sashimono-uninstall.sh +++ b/installer/sashimono-uninstall.sh @@ -6,6 +6,7 @@ sashimono_bin=/usr/bin/sashimono-agent docker_bin=/usr/bin/sashimono-agent/dockerbin sashimono_data=/etc/sashimono sashimono_service="sashimono-agent" +cgcreate_service="sashimono-cgcreate" group="sashimonousers" cgroupsuffix="-cg" quiet=$1 @@ -55,6 +56,11 @@ if [ $ucount -gt 0 ]; then fi fi +echo "Removing Sashimono cgroup creation service..." +systemctl stop $cgcreate_service +systemctl disable $cgcreate_service +rm /etc/systemd/system/$cgcreate_service.service + echo "Removing Sashimono service..." systemctl stop $sashimono_service systemctl disable $sashimono_service @@ -71,5 +77,4 @@ groupdel $group sed -i -r "/^@$group\s+cpu,memory\s+%u$cgroupsuffix/d" /etc/cgrules.conf echo "Sashimono uninstalled successfully." -echo "Please restart your cgroup rule generator service or reboot your server for changes to apply." exit 0 diff --git a/src/conf.hpp b/src/conf.hpp index 9bfc63e..ad52a8c 100644 --- a/src/conf.hpp +++ b/src/conf.hpp @@ -2,6 +2,7 @@ #define _SA_CONF_ #include "pchheader.hpp" +#include "util/util.hpp" namespace conf { @@ -40,6 +41,45 @@ namespace conf } }; + struct ugid + { + uid_t uid = 0; + gid_t gid = 0; + + bool empty() const + { + return uid <= 0 && gid <= 0; + } + + int from_string(std::string_view str) + { + if (str.empty()) + return 0; + + std::vector ids; + util::split_string(ids, str, ":"); + if (ids.size() == 2) + { + const int _uid = atoi(ids[0].c_str()); + const int _gid = atoi(ids[1].c_str()); + + if (_uid > 0 && _gid > 0) + { + uid = _uid; + gid = _gid; + return 0; + } + } + + return -1; + } + + const std::string to_string() const + { + return (uid == 0 && gid == 0) ? "" : (std::to_string(uid) + ":" + std::to_string(gid)); + } + }; + struct log_config { std::string log_level; // Log severity level (dbg, inf, wrn, wrr) diff --git a/src/hp_manager.cpp b/src/hp_manager.cpp index 34008e7..9ef6683 100644 --- a/src/hp_manager.cpp +++ b/src/hp_manager.cpp @@ -25,6 +25,9 @@ namespace hp std::thread hp_monitor_thread; bool is_shutting_down = false; + conf::ugid contract_ugid; + constexpr int CONTRACT_USER_ID = 10000; + // We instruct the demon to restart the container automatically once the container exits except manually stopping. constexpr const char *DOCKER_CREATE = "DOCKER_HOST=unix:///run/user/$(id -u %s)/docker.sock %s/dockerbin/docker create -t -i --stop-signal=SIGINT --name=%s -p %s:%s -p %s:%s \ --restart unless-stopped --mount type=bind,source=%s,target=/contract %s run /contract"; @@ -59,6 +62,7 @@ namespace hp instance_resources.cpu_us = conf::cfg.system.max_cpu_us / conf::cfg.system.max_instance_count; instance_resources.mem_kbytes = conf::cfg.system.max_mem_kbytes / conf::cfg.system.max_instance_count; instance_resources.storage_kbytes = conf::cfg.system.max_storage_kbytes / conf::cfg.system.max_instance_count; + contract_ugid = {CONTRACT_USER_ID, CONTRACT_USER_ID}; return 0; } @@ -558,6 +562,10 @@ namespace hp d["node"]["public_key"] = pubkey_hex; d["node"]["private_key"] = util::to_hex(seckey); d["contract"]["id"] = contract_id; + + // Contract UGID will be passed to hpcore in next PBI, with resolving cgroup issue. + // d["contract"]["run_as"] = contract_ugid.to_string(); + jsoncons::ojson unl(jsoncons::json_array_arg); unl.push_back(util::to_hex(pubkey)); d["contract"]["unl"] = unl; @@ -765,7 +773,13 @@ namespace hp */ int install_user(int &user_id, std::string &username, const size_t max_cpu_us, const size_t max_mem_kbytes, const size_t storage_kbytes, const std::string container_name) { - const std::vector input_params = {std::to_string(max_cpu_us), std::to_string(max_mem_kbytes), std::to_string(storage_kbytes), container_name}; + const std::vector input_params = { + std::to_string(max_cpu_us), + std::to_string(max_mem_kbytes), + std::to_string(storage_kbytes), + container_name, + std::to_string(contract_ugid.uid), + std::to_string(contract_ugid.gid)}; std::vector output_params; if (util::execute_bash_file(conf::ctx.user_install_sh, output_params, input_params) == -1) return -1;