Added Vultr api scripts. (#315)

This commit is contained in:
Ravin Perera
2021-05-29 13:01:33 +05:30
committed by GitHub
parent 7cd56a1475
commit 2f0a6673d5
3 changed files with 299 additions and 32 deletions

View File

@@ -25,6 +25,9 @@
# lcl - Displays the lcls of all nodes.
# pubkey - Displays the pubkey on specified node or entire cluster.
shopt -s expand_aliases
alias sshskp='ssh -o StrictHostKeychecking=no'
mode=$1
hpcore=$(realpath ../..)
@@ -51,7 +54,7 @@ fi
configfile=config.json
if [ ! -f $configfile ]; then
# Create default config file.
echo '{"selected":"contract","contracts":[{"name":"contract","sshuser":"root","sshpass":"","hosts":[],"config":{}}]}' | jq . > $configfile
echo '{"selected":"contract","contracts":[{"name":"contract","sshuser":"root","hosts":[],"config":{}}]}' | jq . > $configfile
fi
if [ $mode = "select" ]; then
@@ -86,7 +89,6 @@ fi
# Read ssh user and password and set contract directory based on username.
sshuser=$(echo $continfo | jq -r '.sshuser')
sshpass=$(echo $continfo | jq -r '.sshpass')
if [ "$sshuser" = "" ]; then
echo "sshuser not specified."
exit 1
@@ -98,8 +100,31 @@ fi
contdir=$basedir/$selectedcont
hpfiles="hpfiles/"$selectedcont
# Call Vultr rest api GET. (params: endpoint, apikey)
function vultrget() {
local _result=$(curl --silent "https://api.vultr.com/v2/$1" -X GET -H "Authorization: Bearer $2" -H "Content-Type: application/json" -w "\n%{http_code}")
local _parts
readarray -t _parts < <(printf '%s' "$_result") # break parts by new line.
if [[ ${_parts[1]} == 2* ]]; then # Check for 2xx status code.
[ ! -z "${_parts[0]}" ] && echo ${_parts[0]} # Return api output if there is any.
else
>&2 echo "Error on vultrget code:${_parts[1]} body:${_parts[0]}" && exit 1
fi
}
# Read the hosts list.
readarray -t hostaddrs <<< $(echo $continfo | jq -r '.hosts[]')
readarray -t hostaddrs < <(printf '%s' "$(echo $continfo | jq -r '.hosts[]')")
# Check whether the first host is "vultr:<group_name>". If so read ips from vultr.
readarray -d ":" -t _host1parts < <(printf '%s' "${hostaddrs[0]}")
if [[ ${_host1parts[0]} == "vultr" ]]; then
_vultrapikey=$(jq -r ".vultr.api_key" $configfile)
[ -z $_vultrapikey ] && >&2 echo "Vultr api key not found." && exit 1
_vultrvms=$(vultrget "instances?tag=${_host1parts[1]}" "$_vultrapikey")
[ -z "$_vultrvms" ] && exit 1
_vultrips=$(echo $(echo $_vultrvms | jq -r ".instances | sort_by(.label) | .[] | .main_ip"))
readarray -d " " -t hostaddrs < <(printf '%s' "$_vultrips") # Populate hostaddrs with ips retrieved from vultr.
echo "Retrieved ${#hostaddrs[@]} host addresses from vultr."
fi
hostcount=${#hostaddrs[@]}
# Read the contract config which should be applied to hp.cfg.
@@ -137,12 +162,12 @@ if [ $mode = "start" ]; then
for (( i=0; i<$hostcount; i++ ))
do
hostaddr=${hostaddrs[i]}
sshpass -p $sshpass ssh $sshuser@$hostaddr $command &
sshskp $sshuser@$hostaddr $command &
done
wait
else
hostaddr=${hostaddrs[$nodeid]}
sshpass -p $sshpass ssh $sshuser@$hostaddr $command
sshskp $sshuser@$hostaddr $command
fi
exit 0
fi
@@ -153,12 +178,12 @@ if [ $mode = "stop" ]; then
for (( i=0; i<$hostcount; i++ ))
do
hostaddr=${hostaddrs[i]}
sshpass -p $sshpass ssh $sshuser@$hostaddr $command &
sshskp $sshuser@$hostaddr $command &
done
wait
else
hostaddr=${hostaddrs[$nodeid]}
sshpass -p $sshpass ssh $sshuser@$hostaddr $command
sshskp $sshuser@$hostaddr $command
fi
exit 0
fi
@@ -170,12 +195,12 @@ if [ $mode = "check" ]; then
do
hostaddr=${hostaddrs[i]}
let n=$i+1
echo "node"$n":" $(sshpass -p $sshpass ssh $sshuser@$hostaddr $command) &
echo "node"$n":" $(sshskp $sshuser@$hostaddr $command) &
done
wait
else
hostaddr=${hostaddrs[$nodeid]}
sshpass -p $sshpass ssh $sshuser@$hostaddr $command
sshskp $sshuser@$hostaddr $command
fi
exit 0
fi
@@ -186,7 +211,7 @@ if [ $mode = "log" ]; then
exit 1
fi
hostaddr=${hostaddrs[$nodeid]}
sshpass -p $sshpass ssh -t $sshuser@$hostaddr screen -r -S hp_$(basename $contdir)
sshskp -t $sshuser@$hostaddr screen -r -S hp_$(basename $contdir)
exit 0
fi
@@ -197,12 +222,12 @@ if [ $mode = "kill" ]; then
do
hostaddr=${hostaddrs[i]}
let n=$i+1
echo "node"$n":" $(sshpass -p $sshpass ssh $sshuser@$hostaddr $command) &
echo "node"$n":" $(sshskp $sshuser@$hostaddr $command) &
done
wait
else
hostaddr=${hostaddrs[$nodeid]}
sshpass -p $sshpass ssh $sshuser@$hostaddr $command
sshskp $sshuser@$hostaddr $command
fi
exit 0
fi
@@ -213,7 +238,7 @@ if [ $mode = "reboot" ]; then
exit 1
fi
hostaddr=${hostaddrs[$nodeid]}
sshpass -p $sshpass ssh $sshuser@$hostaddr 'sudo reboot'
sshskp $sshuser@$hostaddr 'sudo reboot'
exit 0
fi
@@ -227,7 +252,7 @@ if [ $mode = "ssh" ]; then
do
hostaddr=${hostaddrs[i]}
let n=$i+1
echo "node"$n":" $(sshpass -p $sshpass ssh $sshuser@$hostaddr $command) &
echo "node"$n":" $(sshskp $sshuser@$hostaddr $command) &
done
wait
exit 0
@@ -237,7 +262,7 @@ if [ $mode = "ssh" ]; then
fi
else
hostaddr=${hostaddrs[$nodeid]}
sshpass -p $sshpass ssh -t $sshuser@$hostaddr "cd $contdir ; bash"
sshskp -t $sshuser@$hostaddr "cd $contdir ; bash"
exit 0
fi
fi
@@ -251,7 +276,7 @@ if [ $mode = "ssl" ]; then
do
hostaddr=${hostaddrs[i]}
let n=$i+1
echo "node"$n":" $(sshpass -p $sshpass ssh $sshuser@$hostaddr $command) &
echo "node"$n":" $(sshskp $sshuser@$hostaddr $command) &
done
wait
else
@@ -263,7 +288,7 @@ if [ $mode = "ssl" ]; then
if [ -n "$3" ]; then
command="$contdir/ssl.sh $3"
hostaddr=${hostaddrs[$nodeid]}
sshpass -p $sshpass ssh $sshuser@$hostaddr $command
sshskp $sshuser@$hostaddr $command
else
echo "Please specify ssl account notification email."
exit 1
@@ -278,7 +303,7 @@ if [ $mode = "lcl" ]; then
do
hostaddr=${hostaddrs[i]}
let n=$i+1
echo "node"$n":" $(sshpass -p $sshpass ssh $sshuser@$hostaddr $command) &
echo "node"$n":" $(sshskp $sshuser@$hostaddr $command) &
done
wait
exit 0
@@ -291,12 +316,12 @@ if [ $mode = "pubkey" ]; then
do
hostaddr=${hostaddrs[i]}
let n=$i+1
echo "node"$n":" $(sshpass -p $sshpass ssh $sshuser@$hostaddr $command) &
echo "node"$n":" $(sshskp $sshuser@$hostaddr $command) &
done
wait
else
hostaddr=${hostaddrs[$nodeid]}
sshpass -p $sshpass ssh $sshuser@$hostaddr $command
sshskp $sshuser@$hostaddr $command
fi
exit 0
fi
@@ -332,14 +357,14 @@ if [ $nodeid = -1 ]; then
hostaddr=${hostaddrs[i]}
let n=$i+1
# Setup node. (This will download hp.cfg in 'new', 'reconfig', 'updateconfig' modes)
/bin/bash ./setup-node.sh $mode $n $sshuser $sshpass $hostaddr $basedir $contdir $hpfiles &
/bin/bash ./setup-node.sh $mode $n $sshuser $hostaddr $basedir $contdir $hpfiles &
done
wait
else
hostaddr=${hostaddrs[$nodeid]}
let n=$nodeid+1
# Setup node. (This will download hp.cfg in 'new', 'reconfig', 'updateconfig' modes)
/bin/bash ./setup-node.sh $mode $n $sshuser $sshpass $hostaddr $basedir $contdir $hpfiles
/bin/bash ./setup-node.sh $mode $n $sshuser $hostaddr $basedir $contdir $hpfiles
fi
rm -r hpfiles > /dev/null 2>&1
@@ -451,12 +476,12 @@ if [ $nodeid = -1 ]; then
hostaddr=${hostaddrs[i]}
let n=$i+1
sshpass -p $sshpass scp ./cfg/node$n-merged.cfg $sshuser@$hostaddr:$contdir/cfg/hp.cfg &
scp ./cfg/node$n-merged.cfg $sshuser@$hostaddr:$contdir/cfg/hp.cfg &
done
wait
else
let n=$nodeid+1
sshpass -p $sshpass scp ./cfg/node$n-merged.cfg $sshuser@$hostaddr:$contdir/cfg/hp.cfg
scp ./cfg/node$n-merged.cfg $sshuser@$hostaddr:$contdir/cfg/hp.cfg
fi
rm -r ./cfg

View File

@@ -3,26 +3,25 @@
mode=$1
nodeid=$2
sshuser=$3
sshpass=$4
hostaddr=$5
basedir=$6
contdir=$7 # Contract directory
hpfiles=$8 # HP files dir
hostaddr=$4
basedir=$5
contdir=$6 # Contract directory
hpfiles=$7 # HP files dir
echo $nodeid. $hostaddr
if [ $mode = "new" ] || [ $mode = "updatebin" ]; then
echo "Uploading hp files to $basedir..."
sshpass -p $sshpass scp -rp hpfiles $sshuser@$hostaddr:$basedir/
scp -o StrictHostKeyChecking=no -rp hpfiles $sshuser@$hostaddr:$basedir/
echo "Upload finished."
fi
# Run hp setup script on the node and download the generated hp.cfg
if [ $mode = "new" ] || [ $mode = "reconfig" ]; then
echo "Configuring HP..."
sshpass -p $sshpass ssh $sshuser@$hostaddr $basedir/$hpfiles/setup-hp.sh $mode $basedir $contdir $hpfiles $hostaddr
ssh -o StrictHostKeyChecking=no $sshuser@$hostaddr $basedir/$hpfiles/setup-hp.sh $mode $basedir $contdir $hpfiles $hostaddr
fi
if [ $mode = "new" ] || [ $mode = "reconfig" ] || [ $mode = "updateconfig" ]; then
sshpass -p $sshpass scp $sshuser@$hostaddr:$contdir/cfg/hp.cfg ./cfg/node$nodeid.cfg
scp -o StrictHostKeyChecking=no $sshuser@$hostaddr:$contdir/cfg/hp.cfg ./cfg/node$nodeid.cfg
fi

243
test/vm-cluster/vultr.sh Executable file
View File

@@ -0,0 +1,243 @@
#!/bin/bash
# Vultr API script
# Usage examples:
# ./vultr.sh create mycluster 3
# ./vultr.sh info mycluster
# ./vultr.sh delete mycluster
# ./vultr.sh expand mycluster 2
# ./vultr.sh shrink mycluster 2
planid="vc2-1c-1gb" # $5/month
osid=387 # Ubuntu 20.04
# Order of Vultr regions to distribute servers across the globe.
regions=("nrt" "syd" "fra" "yto" "icn" "cdg" "atl" "sgp" "lhr" "ord" "ams" "nrt" "dfw" "syd" "fra" "lax" "icn" "syd" "cdg" "mia" "sgp" "syd" "lhr" "ewr" "nrt" "syd" "fra" "sea" "icn" "syd" "cdg" "sjc")
# jq command is used for json manipulation.
if ! command -v jq &> /dev/null
then
sudo apt-get install -y jq
fi
if ! command -v curl &> /dev/null
then
sudo apt-get install -y curl
fi
configfile=config.json
[ ! -f $configfile ] && >&2 echo "config.json not found." && exit 1
apikey=$(jq -r ".vultr.api_key" $configfile)
[ -z $apikey ] && >&2 echo "Vultr api key not found." && exit 1
startscriptid=$(jq -c -r ".vultr.startup_script_id" $configfile)
sshkeyids=$(jq -c -r ".vultr.ssh_key_ids" $configfile)
# Common api calling function. (params: httpmethod, endpoint, bodyparams)
function apicall() {
local url="https://api.vultr.com/v2/$2"
local _result=""
if [ -z "$3" ]; then
_result=$(curl --silent "$url" -X $1 -H "Authorization: Bearer $apikey" -H "Content-Type: application/json" -w "\n%{http_code}")
else
_result=$(curl --silent "$url" -X $1 -H "Authorization: Bearer $apikey" -H "Content-Type: application/json" -w "\n%{http_code}" --data "$3")
fi
local _parts
readarray -t _parts < <(printf '%s' "$_result") # break parts by new line.
if [[ ${_parts[1]} == 2* ]]; then # Check for 2xx status code.
[ ! -z "${_parts[0]}" ] && echo ${_parts[0]} # Return api output if there is any.
else
>&2 echo "Error on $1 $url code:${_parts[1]} body:${_parts[0]}"
exit 1
fi
}
function apiget() {
if [ -z "$2" ]; then
apicall GET $1
else
apicall GET "$1/$2"
fi
}
function apigetquery() {
apicall GET $1?$2
}
function apipost() {
apicall POST $1 "$2"
}
function apidelete() {
apicall DELETE "$1/$2"
}
# Vultr specific api calls.
function getplans() {
apiget "plans"
}
function getregions() {
apiget "regions"
}
function getoses() {
apiget "os"
}
function getsshkeys() {
apiget "ssh-keys"
}
function getstartscripts() {
apiget "startup-scripts"
}
# Generates vm name using the standard pattern. (parmas: groupname, nodenumber)
function vmname() {
echo $1$(printf "%03d" $2) # Pad node number with 3 zeros.
}
# Creates a vm. (params: groupname, vmname, regionid)
function createvm() {
echo "Creating vm '$2' in $3..."
local _vminfo=$(apipost "instances" '{"tag":"'$1'", "label":"'$2'", "region":"'$3'", "os_id":'$osid', "plan":"'$planid'", "hostname":"'$2'", "script_id":"'$startscriptid'", "sshkey_id":'$sshkeyids', "backups":"disabled"}')
[ -z "$_vminfo" ] && exit 1
local _vmid=$(echo $_vminfo | jq -r ".instance.id")
local _vmip=$(echo $_vminfo | jq -r ".instance.main_ip")
for (( i=0; i<20; i++ ))
do
if [ "$_vmip" == "0.0.0.0" ]; then
sleep 1
_vminfo=$(apiget "instances" $_vmid)
_vmip=$(echo $_vminfo | jq -r ".instance.main_ip")
else
break
fi
done
echo $2": "$_vmip
}
# Deletes a vm. (params: id)
function deletevm() {
echo "Deleting vm "$1"..."
apidelete "instances" $1
}
# Returns space-delimited list of vm field values sorted by vm label in specified group. (params: groupname, fieldname)
function getgroupvmfield() {
[ -z "$1" ] && >&2 echo "getgroupvmfield: Group name not specified." && exit 1
[ -z "$2" ] && >&2 echo "getgroupvmfield: Field name not specified." && exit 1
local _list=$(apigetquery "instances" "tag=$1")
[ -z "$_list" ] && exit 1
# Get field values sorted by the vm label.
local _vals=$(echo $_list | jq -r ".instances | sort_by(.label) | .[] | .$2")
echo $_vals
}
# Deletes a group of vms. (params: groupname)
function deletevmgroup() {
echo "Deleting all vms in group '"$1"'..."
local _ids=$(getgroupvmfield "$1" "id")
[ -z "$_ids" ] && exit 1
local _arr
readarray -d " " -t _arr < <(printf '%s' "$_ids") # break parts by space character.
echo ${#_arr[@]}" vms found to delete..."
for id in "${_arr[@]}"
do
deletevm $id &
done
wait
echo "Done."
}
# Creates a group of vms. (params: groupname, count, startnumber)
function createvmgroup() {
echo "Creating "$2" vms in group '"$1"'..."
local -i start=$3
[ $start == 0 ] && start=1
local -i end=$start+$2
local -i rcount=${#regions[@]} # region count.
for (( i=$start; i<$end; i++ ))
do
local -i r=$((($i - 1) % $rcount))
createvm "$1" $(vmname $1 $i) "${regions[$r]}" &
done
wait
echo "Done."
}
# Grows a vm group. (params: groupname, expandbycount)
function expandvmgroup() {
# Get current no. of vms in the group. (this returns ids sorted by vm label)
local _ids=$(getgroupvmfield "$1" "id")
[ -z "$_ids" ] && exit 1
local _arr
readarray -d " " -t _arr < <(printf '%s' "$_ids") # break parts by space character.
local -i _oldcount=${#_arr[@]}
local -i _newcount=$_oldcount+$2
[ $_oldcount -ge $_newcount ] && exit 1
echo "Expanding '"$1"' group from "$_oldcount" to "$_newcount" nodes..."
let -i startnum=$_oldcount+1
createvmgroup $1 $2 $startnum
}
# Shrinks a vm group. (params: groupname, shrinkbycount)
function shrinkvmgroup() {
# Get current set of vms in the group. (this returns ids sorted by vm label)
local _ids=$(getgroupvmfield "$1" "id")
[ -z "$_ids" ] && exit 1
local _arr
readarray -d " " -t _arr < <(printf '%s' "$_ids") # break parts by space character.
local -i _oldcount=${#_arr[@]}
local -i _newcount=$_oldcount-$2
echo "Shrinking '"$1"' group from "$_oldcount" to "$_newcount" nodes..."
for (( i=$_newcount; i<$_oldcount; i++ ))
do
deletevm ${_arr[$i]} &
done
wait
echo "Done."
}
# DIsplays information about existing cluster. (params: groupname)
function displayinfo() {
local _ips=$(getgroupvmfield "$1" "main_ip")
local _arr
readarray -d " " -t _arr < <(printf '%s' "$_ips") # break parts by space character.
echo "Total "${#_arr[@]}" vms found in '"$1"'"
for (( i=0; i<${#_arr[@]}; i++ ))
do
let n=$i+1
echo $n. ${_arr[$i]}
done
}
mode=$1
name=$2
num=$3
if [ "$mode" = "create" ] || [ "$mode" = "info" ] || [ "$mode" = "delete" ] || [ "$mode" = "expand" ] || [ "$mode" = "shrink" ]; then
echo "mode: $mode"
else
echo "Invalid command."
echo " Expected: create <name> <count> | info <name> | delete <name> | expand <name> <by> | shrink <name> <by>"
exit 1
fi
if [ $mode = "create" ]; then
createvmgroup $name $num
exit 0
fi
if [ $mode = "info" ]; then
displayinfo $name
exit 0
fi
if [ $mode = "delete" ]; then
deletevmgroup $name
exit 0
fi
if [ $mode = "expand" ]; then
expandvmgroup $name $num
exit 0
fi
if [ $mode = "shrink" ]; then
shrinkvmgroup $name $num
exit 0
fi