Added cgcreate service to create cgroups at the system boot (#32)

CMakeLists.txt

@@ -66,7 +66,7 @@ add_custom_target(installer
   COMMAND mkdir -p ./build/sashimono-installer
   COMMAND bash -c "cp -r ./build/{sagent,hpfs,hpws,user-install.sh,user-uninstall.sh,contract_template} ./build/sashimono-installer/"
   COMMAND bash -c "cp -r ./installer/{docker-install.sh,sashimono-install.sh,sashimono-uninstall.sh} ./build/sashimono-installer/"
-  COMMAND bash -c "cp -r ./dependencies/libblake3.so ./build/sashimono-installer/"
+  COMMAND bash -c "cp -r ./dependencies/{user-cgcreate.sh,libblake3.so} ./build/sashimono-installer/"
   COMMAND tar cfz  ./build/sashimono-installer.tar.gz --directory=./build/ sashimono-installer
   COMMAND rm -r ./build/sashimono-installer
 )

dependencies/user-cgcreate.sh

@@ -0,0 +1,71 @@
+#!/bin/bash
+
+datadir=$1
+if [ -z "$datadir" ]; then
+    echo "Invalid arguments."
+    echo "Expected: user-cgcreate.sh <sashimino data dir>"
+    exit 1
+fi
+
+saconfig="$1/sa.cfg"
+if [ ! -f "$saconfig" ]; then
+    echo "Config file does not exist."
+    echo "Run \"sagent new $datadir\" command."
+    exit 1
+fi
+
+# Calculate resourses
+# jq command is used for json manipulation.
+if ! command -v jq &>/dev/null; then
+    echo "jq utility not found. Installing.."
+    apt-get install -y jq >/dev/null 2>&1
+fi
+
+# Read config values
+max_mem_kbytes=$(jq '.system.max_mem_kbytes' $saconfig)
+max_cpu_us=$(jq '.system.max_cpu_us' $saconfig)
+max_instance_count=$(jq '.system.max_instance_count' $saconfig)
+
+([ "$max_instance_count" == "" ] || [ ${#max_instance_count} -eq 0 ] || [ "$max_instance_count" -le 0 ]) && echo "max_instance_count cannot be empty." && exit 1
+
+instance_mem_kbytes=0
+if [ "$max_mem_kbytes" != "" ] && [ ! ${#max_mem_kbytes} -eq 0 ] && [ "$max_mem_kbytes" -gt 0 ]; then
+    ! instance_mem_kbytes=$(expr $max_mem_kbytes / $max_instance_count) && echo "Max memory limit calculation error." && exit 1
+fi
+
+instance_cpu_us=0
+if [ "$max_cpu_us" != "" ] && [ ! ${#max_cpu_us} -eq 0 ] && [ "$max_cpu_us" -gt 0 ]; then
+    ! instance_cpu_us=$(expr $max_cpu_us / $max_instance_count) && echo "Max cpu limit calculation error." && exit 1
+fi
+
+prefix="sashi"
+cgroupsuffix="-cg"
+users=$(cut -d: -f1 /etc/passwd | grep "^$prefix" | sort)
+readarray -t userarr <<<"$users"
+validusers=()
+for user in "${userarr[@]}"; do
+    [ ${#user} -lt 24 ] || [ ${#user} -gt 32 ] || [[ ! "$user" =~ ^$prefix[0-9]+$ ]] && continue
+    validusers+=("$user")
+done
+
+has_err=0
+for user in "${validusers[@]}"; do
+    # Setup user cgroup.
+    if  [ $instance_cpu_us -gt 0 ] &&
+        ! (cgcreate -g cpu:$user$cgroupsuffix &&
+        echo "$instance_cpu_us" > /sys/fs/cgroup/cpu/$user$cgroupsuffix/cpu.cfs_quota_us); then
+        echo "CPU cgroup creation for $user failed."
+        has_err=1
+    fi
+
+    if [ $instance_mem_kbytes -gt 0 ] &&
+        ! (cgcreate -g memory:$user$cgroupsuffix &&
+        echo "${instance_mem_kbytes}K" > /sys/fs/cgroup/memory/$user$cgroupsuffix/memory.limit_in_bytes &&
+        echo "${instance_mem_kbytes}K" > /sys/fs/cgroup/memory/$user$cgroupsuffix/memory.memsw.limit_in_bytes); then
+        echo "Memory cgroup creation for $user failed."
+        has_err=1
+    fi
+done
+
+[ $has_err -eq 1 ] && exit 1
+exit 0

dependencies/user-install.sh

@@ -7,8 +7,10 @@ cpu=$1
 memory=$2
 disk=$3
 contract_dir=$4
-if [ -z "$cpu" ] || [ -z "$memory" ] || [ -z "$disk" ]; then
-    echo "Expected: user-install.sh <cpu quota microseconds> <memory quota kbytes> <disk quota kbytes>"
+contract_uid=$5
+contract_gid=$6
+if [ -z "$cpu" ] || [ -z "$memory" ] || [ -z "$disk" ] || [ -z "$contract_dir" ] || [ -z "$contract_uid" ] || [ -z "$contract_gid" ]; then
+    echo "Expected: user-install.sh <cpu quota microseconds> <memory quota kbytes> <disk quota kbytes> <contract dir> <contract uid> <contract gid>"
     echo "INVALID_PARAMS,INST_ERR" && exit 1
 fi
 
@@ -47,11 +49,11 @@ user_runtime_dir="/run/user/$user_id"
 dockerd_socket="unix://$user_runtime_dir/docker.sock"
 
 # Setup user cgroup.
-! (cgcreate -g cpu:"$user"$cgroupsuffix &&
-    echo "$cpu" >/sys/fs/cgroup/cpu/"$user"$cgroupsuffix/cpu.cfs_quota_us) && echo rollback "CGROUP_CPU_CREAT"
-! (cgcreate -g memory:"$user"$cgroupsuffix &&
-    echo "${memory}K" >/sys/fs/cgroup/memory/"$user"$cgroupsuffix/memory.limit_in_bytes &&
-    echo "${memory}K" >/sys/fs/cgroup/memory/"$user"$cgroupsuffix/memory.memsw.limit_in_bytes) && echo rollback "CGROUP_MEM_CREAT"
+! (cgcreate -g cpu:$user$cgroupsuffix &&
+    echo "$cpu" >/sys/fs/cgroup/cpu/$user$cgroupsuffix/cpu.cfs_quota_us) && rollback "CGROUP_CPU_CREAT"
+! (cgcreate -g memory:$user$cgroupsuffix &&
+    echo "${memory}K" >/sys/fs/cgroup/memory/$user$cgroupsuffix/memory.limit_in_bytes &&
+    echo "${memory}K" >/sys/fs/cgroup/memory/$user$cgroupsuffix/memory.memsw.limit_in_bytes) && rollback "CGROUP_MEM_CREAT"
 
 # Adding disk quota to the new user.
 setquota -u -F vfsv0 "$user" "$disk" "$disk" 0 0 /
@@ -83,6 +85,16 @@ svcstat=$(sudo -u "$user" XDG_RUNTIME_DIR="$user_runtime_dir" systemctl --user i
 echo "Installed rootless dockerd."
 
 echo "Adding hpfs services for the instance."
+
+# Taking the uid and gid offsets
+uoffset=$(grep "^$user:[0-9]\+:[0-9]\+$" /etc/subuid | cut -d: -f2)
+[ -z $uoffset ] && rollback "SUBUID_ERR"
+goffset=$(grep "^$user:[0-9]\+:[0-9]\+$" /etc/subgid | cut -d: -f2)
+[ -z $goffset ] && rollback "SUBGID_ERR"
+hpfs_uid=$(expr $uoffset + $contract_uid)
+hpfs_gid=$(expr $goffset + $contract_gid)
+
+# UGID will be passed to hpfs in next PBI, with resolving cgroup issue.
 echo "[Unit]
 Description=Running and monitoring contract fs.
 StartLimitIntervalSec=0

dependencies/user-uninstall.sh

@@ -67,8 +67,8 @@ fi
 
 echo "Removing cgroups"
 # Delete config values.
-cgdelete -g cpu:"$user"$cgroupsuffix
-cgdelete -g memory:"$user"$cgroupsuffix
+cgdelete -g cpu:$user$cgroupsuffix
+cgdelete -g memory:$user$cgroupsuffix
 
 echo "Deleting user '$user'"
 userdel "$user"

installer/sashimono-install.sh

@@ -6,6 +6,7 @@ sashimono_bin=/usr/bin/sashimono-agent
 docker_bin=/usr/bin/sashimono-agent/dockerbin
 sashimono_data=/etc/sashimono
 sashimono_service="sashimono-agent"
+cgcreate_service="sashimono-cgcreate"
 group="sashimonousers"
 cgroupsuffix="-cg"
 script_dir=$(dirname "$(realpath "$0")")
@@ -55,7 +56,7 @@ function rollback() {
 }
 
 # Install Sashimono agent binaries into sashimono bin dir.
-cp "$script_dir"/{sagent,hpfs,hpws,user-install.sh,user-uninstall.sh} $sashimono_bin
+cp "$script_dir"/{sagent,hpfs,hpws,user-cgcreate.sh,user-install.sh,user-uninstall.sh} $sashimono_bin
 chmod -R +x $sashimono_bin
 
 # Download and install rootless dockerd.
@@ -72,6 +73,19 @@ chmod -R +x $sashimono_bin
 cp -r "$script_dir"/contract_template $sashimono_data
 $sashimono_bin/sagent new $sashimono_data
 
+# Install Sashimono Agent cgcreate service.
+# This is a onshot service which runs only once.
+echo "[Unit]
+Description=Sashimono cgroup creation service.
+After=network.target
+[Service]
+User=root
+Group=root
+Type=oneshot
+ExecStart=$sashimono_bin/user-cgcreate.sh $sashimono_data
+[Install]
+WantedBy=multi-user.target" >/etc/systemd/system/$cgcreate_service.service
+
 # Install Sashimono Agent systemd service.
 # StartLimitIntervalSec=0 to make unlimited retries. RestartSec=5 is to keep 5 second gap between restarts.
 echo "[Unit]
@@ -90,8 +104,11 @@ RestartSec=5
 WantedBy=multi-user.target" >/etc/systemd/system/$sashimono_service.service
 
 systemctl daemon-reload
+systemctl enable $cgcreate_service
+systemctl start $cgcreate_service
 systemctl enable $sashimono_service
 systemctl start $sashimono_service
+# Both of these services needed to be restarted if sa.cfg max instance resources are manually changed.
 
 echo "Sashimono installed successfully."
 echo "Please restart your cgroup rule generator service or reboot your server for changes to apply."

installer/sashimono-uninstall.sh

@@ -6,6 +6,7 @@ sashimono_bin=/usr/bin/sashimono-agent
 docker_bin=/usr/bin/sashimono-agent/dockerbin
 sashimono_data=/etc/sashimono
 sashimono_service="sashimono-agent"
+cgcreate_service="sashimono-cgcreate"
 group="sashimonousers"
 cgroupsuffix="-cg"
 quiet=$1
@@ -55,6 +56,11 @@ if [ $ucount -gt 0 ]; then
     fi
 fi
 
+echo "Removing Sashimono cgroup creation service..."
+systemctl stop $cgcreate_service
+systemctl disable $cgcreate_service
+rm /etc/systemd/system/$cgcreate_service.service
+
 echo "Removing Sashimono service..."
 systemctl stop $sashimono_service
 systemctl disable $sashimono_service
@@ -71,5 +77,4 @@ groupdel $group
 sed -i -r "/^@$group\s+cpu,memory\s+%u$cgroupsuffix/d" /etc/cgrules.conf
 
 echo "Sashimono uninstalled successfully."
-echo "Please restart your cgroup rule generator service or reboot your server for changes to apply."
 exit 0

src/conf.hpp

@@ -2,6 +2,7 @@
 #define _SA_CONF_
 
 #include "pchheader.hpp"
+#include "util/util.hpp"
 
 namespace conf
 {
@@ -40,6 +41,45 @@ namespace conf
         }
     };
 
+    struct ugid
+    {
+        uid_t uid = 0;
+        gid_t gid = 0;
+
+        bool empty() const
+        {
+            return uid <= 0 && gid <= 0;
+        }
+
+        int from_string(std::string_view str)
+        {
+            if (str.empty())
+                return 0;
+
+            std::vector<std::string> ids;
+            util::split_string(ids, str, ":");
+            if (ids.size() == 2)
+            {
+                const int _uid = atoi(ids[0].c_str());
+                const int _gid = atoi(ids[1].c_str());
+
+                if (_uid > 0 && _gid > 0)
+                {
+                    uid = _uid;
+                    gid = _gid;
+                    return 0;
+                }
+            }
+
+            return -1;
+        }
+
+        const std::string to_string() const
+        {
+            return (uid == 0 && gid == 0) ? "" : (std::to_string(uid) + ":" + std::to_string(gid));
+        }
+    };
+
     struct log_config
     {
         std::string log_level;                            // Log severity level (dbg, inf, wrn, wrr)

src/hp_manager.cpp

@@ -25,6 +25,9 @@ namespace hp
     std::thread hp_monitor_thread;
     bool is_shutting_down = false;
 
+    conf::ugid contract_ugid;
+    constexpr int CONTRACT_USER_ID = 10000;
+
     // We instruct the demon to restart the container automatically once the container exits except manually stopping.
     constexpr const char *DOCKER_CREATE = "DOCKER_HOST=unix:///run/user/$(id -u %s)/docker.sock %s/dockerbin/docker create -t -i --stop-signal=SIGINT --name=%s -p %s:%s -p %s:%s \
                                             --restart unless-stopped --mount type=bind,source=%s,target=/contract %s run /contract";
@@ -59,6 +62,7 @@ namespace hp
         instance_resources.cpu_us = conf::cfg.system.max_cpu_us / conf::cfg.system.max_instance_count;
         instance_resources.mem_kbytes = conf::cfg.system.max_mem_kbytes / conf::cfg.system.max_instance_count;
         instance_resources.storage_kbytes = conf::cfg.system.max_storage_kbytes / conf::cfg.system.max_instance_count;
+        contract_ugid = {CONTRACT_USER_ID, CONTRACT_USER_ID};
 
         return 0;
     }
@@ -558,6 +562,10 @@ namespace hp
         d["node"]["public_key"] = pubkey_hex;
         d["node"]["private_key"] = util::to_hex(seckey);
         d["contract"]["id"] = contract_id;
+        
+        // Contract UGID will be passed to hpcore in next PBI, with resolving cgroup issue.
+        // d["contract"]["run_as"] = contract_ugid.to_string();
+        
         jsoncons::ojson unl(jsoncons::json_array_arg);
         unl.push_back(util::to_hex(pubkey));
         d["contract"]["unl"] = unl;
@@ -765,7 +773,13 @@ namespace hp
     */
     int install_user(int &user_id, std::string &username, const size_t max_cpu_us, const size_t max_mem_kbytes, const size_t storage_kbytes, const std::string container_name)
     {
-        const std::vector<std::string_view> input_params = {std::to_string(max_cpu_us), std::to_string(max_mem_kbytes), std::to_string(storage_kbytes), container_name};
+        const std::vector<std::string_view> input_params = {
+            std::to_string(max_cpu_us),
+            std::to_string(max_mem_kbytes),
+            std::to_string(storage_kbytes),
+            container_name,
+            std::to_string(contract_ugid.uid),
+            std::to_string(contract_ugid.gid)};
         std::vector<std::string> output_params;
         if (util::execute_bash_file(conf::ctx.user_install_sh, output_params, input_params) == -1)
             return -1;