From 4e75b497ec8ed68515e7960c23c4c57bbca1ee53 Mon Sep 17 00:00:00 2001
From: Ravin Perera <33562092+ravinsp@users.noreply.github.com>
Date: Tue, 11 May 2021 09:01:09 +0530
Subject: [PATCH] Improvements to vm cluster scripts. (#304)

* Improvements to vm cluster scripts.
* Fixed sync priority change issue.
---
 src/hpfs/hpfs_sync.cpp                       |   2 +-
 test/vm-cluster/.gitignore                   |   2 +-
 test/vm-cluster/cluster.sh                   | 444 +++++++++++--------
 test/vm-cluster/consensus-test-continuous.sh |  60 ---
 test/vm-cluster/setup-hp.sh                  |  20 +-
 test/vm-cluster/setup-node.sh                |  27 ++
 test/vm-cluster/setup-vm.sh                  |  24 -
 7 files changed, 300 insertions(+), 279 deletions(-)
 delete mode 100644 test/vm-cluster/consensus-test-continuous.sh
 create mode 100755 test/vm-cluster/setup-node.sh
 delete mode 100755 test/vm-cluster/setup-vm.sh

diff --git a/src/hpfs/hpfs_sync.cpp b/src/hpfs/hpfs_sync.cpp
index c238342f..a13092c2 100644
--- a/src/hpfs/hpfs_sync.cpp
+++ b/src/hpfs/hpfs_sync.cpp
@@ -169,7 +169,7 @@ namespace hpfs
                 if (is_shutting_down)
                     break;
 
-                if (result != SYNC_HASH_CHANGED)
+                if (result != SYNC_HASH_CHANGED && result != SYNC_PRIORITY_CHANGED)
                 {
                     // The finished target can be removed from the list.
                     {
diff --git a/test/vm-cluster/.gitignore b/test/vm-cluster/.gitignore
index e480b42b..670d8eb3 100644
--- a/test/vm-cluster/.gitignore
+++ b/test/vm-cluster/.gitignore
@@ -1,3 +1,3 @@
 cfg
-vmconfig*
+*.json
 hpfiles
\ No newline at end of file
diff --git a/test/vm-cluster/cluster.sh b/test/vm-cluster/cluster.sh
index 2b1e0ee6..c0b25c49 100755
--- a/test/vm-cluster/cluster.sh
+++ b/test/vm-cluster/cluster.sh
@@ -1,53 +1,113 @@
 #!/bin/bash
-# Hot Pocket VM cluster management script.
+# Hot Pocket cluster management script.
 
 # Usage examples:
 # ./cluster.sh new
-# ./cluster.sh update
+# ./cluster.sh updatebin
 # ./cluster.sh start 1
 # ./cluster.sh start
 
+# Command modes:
+# info - Displays information about current cluster configuration status.
+# select - Sets the currently active contract from the list of contracts defined in cluster config file.
+# new - Install hot pocket dependencies and hot pocket with example contracts to each node.
+# updatebin - Deploy updated hot pocket and example binaries into specified node or entire cluster.
+# updateconfig - Updates the config file of specified node or entire cluster.
+# reconfig - Cleans and reconfigures the entire cluster using already uploaded HP binaries.
+# start - Run hot pocket on specified node or entire cluster.
+# stop - Gracefully stop hot pocket (if running) on specified node or entire cluster.
+# check - Get hot pocket running process ids on specified node or entire cluster.
+# log - Stream hot pocket console output log (if running) on specified node.
+# kill - Force kill hot pocket (if running) on specified node or entire cluster.
+# reboot - Reboot specified node.
+# ssh - Open up an ssh terminal for the specified node.
+# ssl - Creates LetsEncrypt ssl certs matching with the domain name.
+# lcl - Displays the lcls of all nodes.
+# pubkey - Displays the pubkey on specified node or entire cluster.
+
+mode=$1
+hpcore=$(realpath ../..)
+
+if [ "$mode" = "info" ] || [ "$mode" = "select" ] ||
+   [ "$mode" = "new" ] || [ "$mode" = "updatebin" ] || [ "$mode" = "updateconfig" ] || [ "$mode" = "reconfig" ] || \
+   [ "$mode" = "start" ] || [ "$mode" = "stop" ] || [ "$mode" = "check" ] || [ "$mode" = "log" ] || [ "$mode" = "kill" ] || \
+   [ "$mode" = "ssh" ] || [ "$mode" = "reboot" ] || [ "$mode" = "ssl" ] || [ "$mode" = "lcl" ] || [ "$mode" = "pubkey" ]; then
+    echo "mode: $mode"
+else
+    echo "Invalid command."
+    echo " Expected: info | select | new | updatebin <N> | updateconfig [N] | reconfig" \
+        " | start [N] | stop [N] | check [N] | log <N> | kill [N] | reboot <N> | ssh <N>or<command>" \
+        " | ssl <email>or<N> <email> | lcl | pubkey [N]"
+    echo " <N>: Required node no.   [N]: Optional node no."
+    exit 1
+fi
+
 # jq command is used for json manipulation.
 if ! command -v jq &> /dev/null
 then
-    echo "jq command not found. Install with 'sudo apt-get install -y jq'"
-    exit 1
+    sudo apt-get install -y jq
 fi
 
-conf=vmconfig.json
-if [ ! -f $conf ]; then
+configfile=config.json
+if [ ! -f $configfile ]; then
     # Create default config file.
-    echo '{"vmuser":"root","vmpass":"","vms":[],"contracts":[{"name":"contract","config":{}}]}' | jq . > $conf
+    echo '{"selected":"contract","contracts":[{"name":"contract","sshuser":"root","sshpass":"","hosts":[],"config":{}}]}' | jq . > $configfile
 fi
 
-vmuser=$(jq -r '.vmuser' $conf)
+if [ $mode = "select" ]; then
+    selectedcont=$2
+    if [ "$selectedcont" = "" ]; then
+        echo "Please specify contract name to select."
+        exit 1
+    fi
+    continfo=$(jq -r ".contracts[] | select(.name == \"$selectedcont\")" $configfile)
+    if [ "$continfo" = "" ]; then
+        echo "No configuration found for selected contract '"$selectedcont"'"
+        exit 1
+    fi
 
-if [ "$vmuser" = "" ]; then
-    echo "vmuser not specified."
+    # Set the 'selected' field value on cluster config file.
+    jq ".selected = \"$selectedcont\"" $configfile > $configfile.tmp && mv $configfile.tmp $configfile
+    echo "Selected '"$selectedcont"'"
+    exit 0
+fi
+
+selectedcont=$(jq -r '.selected' $configfile)
+if [ "$selectedcont" = "" ]; then
+    echo "No contract selected."
     exit 1
-elif [ "$CONTRACT" = "" ]; then
-    CONTRACT=contract # Default contract name (can be set with 'export CONTRACT=<name>'').
 fi
 
-if [ "$vmuser" = "root" ]; then
-    basedir=/$vmuser
+continfo=$(jq -r ".contracts[] | select(.name == \"$selectedcont\")" $configfile)
+if [ "$continfo" = "" ]; then
+    echo "No configuration found for selected contract '"$selectedcont"'"
+    exit 1
+fi
+
+# Read ssh user and password and set contract directory based on username.
+sshuser=$(echo $continfo | jq -r '.sshuser')
+sshpass=$(echo $continfo | jq -r '.sshpass')
+if [ "$sshuser" = "" ]; then
+    echo "sshuser not specified."
+    exit 1
+elif [ "$sshuser" = "root" ]; then
+    basedir=/$sshuser
 else
-    basedir=/home/$vmuser
+    basedir=/home/$sshuser
 fi
+contdir=$basedir/$selectedcont
 
-contconfig=$(jq -r ".contracts[] | select(.name == \"${CONTRACT}\") | .config" $conf)
+# Read the hosts list.
+readarray -t hostaddrs <<< $(echo $continfo | jq -r '.hosts[]')
+hostcount=${#hostaddrs[@]}
+
+# Read the contract config which should be applied to hp.cfg.
+contconfig=$(echo $continfo | jq -r '.config')
 if [ "$contconfig" = "" ] || [ "$contconfig" = "{}" ]; then
     # Apply default config.
     contconfig="{\"user\": {\"port\": 8080}, \"mesh\":{ \"port\": 22860}, \"contract\": {\"roundtime\": 2000 }, \"log\":{\"loglevel\": \"inf\", \"loggers\":[\"console\",\"file\"]}}"
 fi
 
-vmpass=$(jq -r '.vmpass' $conf)
-readarray -t vmaddrs <<< $(jq -r '.vms[]' $conf)
-contdir=$basedir/$CONTRACT
-vmcount=${#vmaddrs[@]}
-mode=$1
-hpcore=$(realpath ../..)
-
 # Check if second arg (nodeid) is a number or not.
 # If it's a number then reduce 1 from it to get zero-based node index.
 if ! [[ $2 =~ ^[0-9]+$ ]] ; then
@@ -56,35 +116,15 @@ else
     let nodeid=$2-1
 fi
 
-if [ "$mode" = "info" ] || [ "$mode" = "new" ] || [ "$mode" = "update" ] || [ "$mode" = "reconfig" ] || \
-   [ "$mode" = "start" ] || [ "$mode" = "stop" ] || [ "$mode" = "check" ] || [ "$mode" = "log" ] || [ "$mode" = "kill" ] || \
-   [ "$mode" = "ssh" ] || [ "$mode" = "reboot" ] || [ "$mode" = "ssl" ] || [ "$mode" = "lcl" ] || [ "$mode" = "pubkey" ]; then
-    echo "mode: $mode ($contdir)"
-else
-    echo "Invalid command. [ info | new | update | reconfig" \
-        " | start [N] | stop [N] | check [N] | log <N> | kill [N] | reboot <N> | ssh <N>or<command>" \
-        " | ssl <email>or<N> <email> | lcl | pubkey <N> ] expected."
-    exit 1
-fi
-
-# Command modes:
-# info - Displays information about current cluster configuration status.
-# new - Install hot pocket dependencies and hot pocket with example contracts to each vm.
-# update - Deploy updated hot pocket and example binaries into each vm.
-# reconfig - Reconfigures the entire cluster using already uploaded HP binaries.
-# start - Run hot pocket on specified vm node or entire cluster.
-# stop - Gracefully stop hot pocket (if running) on specified vm node or entire cluster.
-# check - Get hot pocket running process ids on specified vm node or entire cluster.
-# log - Stream hot pocket console output log (if running) on specified vm node.
-# kill - Force kill hot pocket (if running) on specified vm node or entire cluster.
-# reboot - Reboot specified vm node.
-# ssh - Open up an ssh terminal for the specified vm node.
-# ssl - Creates LetsEncrypt ssl certs matching with the vm domain name.
-# lcl - Displays the lcls of all nodes.
-# pubkey - Displays the pubkey on specified vm node or entire cluster.
+echo " dir: "$contdir
 
 if [ $mode = "info" ]; then
-    echo "${vmaddrs[*]}" | tr ' ' '\n'
+    for (( i=0; i<$hostcount; i++ ))
+    do
+        let n=$i+1
+        hostaddr=${hostaddrs[i]}
+        echo "node"$n": "$hostaddr
+    done
     echo $contconfig
     exit 0
 fi
@@ -93,16 +133,15 @@ if [ $mode = "start" ]; then
     # Use the screen command so that the execution does not stop when ssh session ends.
     command="mkdir -p $contdir/screen && screen -c $contdir/hp.screenrc -m -d bash $contdir/start.sh"
     if [ $nodeid = -1 ]; then
-        for (( i=0; i<$vmcount; i++ ))
+        for (( i=0; i<$hostcount; i++ ))
         do
-            vmaddr=${vmaddrs[i]}
-            let nodeid=$i+1
-            sshpass -p $vmpass ssh $vmuser@$vmaddr $command &
+            hostaddr=${hostaddrs[i]}
+            sshpass -p $sshpass ssh $sshuser@$hostaddr $command &
         done
         wait
     else
-        vmaddr=${vmaddrs[$nodeid]}
-        sshpass -p $vmpass ssh $vmuser@$vmaddr $command
+        hostaddr=${hostaddrs[$nodeid]}
+        sshpass -p $sshpass ssh $sshuser@$hostaddr $command
     fi
     exit 0
 fi
@@ -110,16 +149,15 @@ fi
 if [ $mode = "stop" ]; then
     command="$contdir/stop.sh"
     if [ $nodeid = -1 ]; then
-        for (( i=0; i<$vmcount; i++ ))
+        for (( i=0; i<$hostcount; i++ ))
         do
-            vmaddr=${vmaddrs[i]}
-            let nodeid=$i+1
-            sshpass -p $vmpass ssh $vmuser@$vmaddr $command &
+            hostaddr=${hostaddrs[i]}
+            sshpass -p $sshpass ssh $sshuser@$hostaddr $command &
         done
         wait
     else
-        vmaddr=${vmaddrs[$nodeid]}
-        sshpass -p $vmpass ssh $vmuser@$vmaddr $command
+        hostaddr=${hostaddrs[$nodeid]}
+        sshpass -p $sshpass ssh $sshuser@$hostaddr $command
     fi
     exit 0
 fi
@@ -127,16 +165,16 @@ fi
 if [ $mode = "check" ]; then
     command="$contdir/check.sh"
     if [ $nodeid = -1 ]; then
-        for (( i=0; i<$vmcount; i++ ))
+        for (( i=0; i<$hostcount; i++ ))
         do
-            vmaddr=${vmaddrs[i]}
-            let nodeid=$i+1
-            echo "node"$nodeid":" $(sshpass -p $vmpass ssh $vmuser@$vmaddr $command) &
+            hostaddr=${hostaddrs[i]}
+            let n=$i+1
+            echo "node"$n":" $(sshpass -p $sshpass ssh $sshuser@$hostaddr $command) &
         done
         wait
     else
-        vmaddr=${vmaddrs[$nodeid]}
-        sshpass -p $vmpass ssh $vmuser@$vmaddr $command
+        hostaddr=${hostaddrs[$nodeid]}
+        sshpass -p $sshpass ssh $sshuser@$hostaddr $command
     fi
     exit 0
 fi
@@ -146,24 +184,24 @@ if [ $mode = "log" ]; then
         echo "Please specify node no.."
         exit 1
     fi
-    vmaddr=${vmaddrs[$nodeid]}
-    sshpass -p $vmpass ssh -t $vmuser@$vmaddr screen -r -S hp_$(basename $contdir)
+    hostaddr=${hostaddrs[$nodeid]}
+    sshpass -p $sshpass ssh -t $sshuser@$hostaddr screen -r -S hp_$(basename $contdir)
     exit 0
 fi
 
 if [ $mode = "kill" ]; then
     command="$contdir/kill.sh"
     if [ $nodeid = -1 ]; then
-        for (( i=0; i<$vmcount; i++ ))
+        for (( i=0; i<$hostcount; i++ ))
         do
-            vmaddr=${vmaddrs[i]}
-            let nodeid=$i+1
-            echo "node"$nodeid":" $(sshpass -p $vmpass ssh $vmuser@$vmaddr $command) &
+            hostaddr=${hostaddrs[i]}
+            let n=$i+1
+            echo "node"$n":" $(sshpass -p $sshpass ssh $sshuser@$hostaddr $command) &
         done
         wait
     else
-        vmaddr=${vmaddrs[$nodeid]}
-        sshpass -p $vmpass ssh $vmuser@$vmaddr $command
+        hostaddr=${hostaddrs[$nodeid]}
+        sshpass -p $sshpass ssh $sshuser@$hostaddr $command
     fi
     exit 0
 fi
@@ -173,8 +211,8 @@ if [ $mode = "reboot" ]; then
         echo "Please specify node no."
         exit 1
     fi
-    vmaddr=${vmaddrs[$nodeid]}
-    sshpass -p $vmpass ssh $vmuser@$vmaddr 'sudo reboot'
+    hostaddr=${hostaddrs[$nodeid]}
+    sshpass -p $sshpass ssh $sshuser@$hostaddr 'sudo reboot'
     exit 0
 fi
 
@@ -184,11 +222,11 @@ if [ $mode = "ssh" ]; then
             # Interpret second arg as a command to execute on all nodes.
             command=${*:2}
             echo "Executing '$command' on all nodes..."
-            for (( i=0; i<$vmcount; i++ ))
+            for (( i=0; i<$hostcount; i++ ))
             do
-                vmaddr=${vmaddrs[i]}
-                let nodeid=$i+1
-                echo "node"$nodeid":" $(sshpass -p $vmpass ssh $vmuser@$vmaddr $command) &
+                hostaddr=${hostaddrs[i]}
+                let n=$i+1
+                echo "node"$n":" $(sshpass -p $sshpass ssh $sshuser@$hostaddr $command) &
             done
             wait
             exit 0
@@ -197,8 +235,8 @@ if [ $mode = "ssh" ]; then
             exit 1
         fi
     else
-        vmaddr=${vmaddrs[$nodeid]}
-        sshpass -p $vmpass ssh -t $vmuser@$vmaddr "cd $contdir ; bash"
+        hostaddr=${hostaddrs[$nodeid]}
+        sshpass -p $sshpass ssh -t $sshuser@$hostaddr "cd $contdir ; bash"
         exit 0
     fi
 fi
@@ -208,11 +246,11 @@ if [ $mode = "ssl" ]; then
         if [ -n "$2" ]; then
             # If nodeid is not specified, interpret second arg as the ssl account notification email.
             command="$contdir/ssl.sh $2"
-            for (( i=0; i<$vmcount; i++ ))
+            for (( i=0; i<$hostcount; i++ ))
             do
-                vmaddr=${vmaddrs[i]}
-                let nodeid=$i+1
-                echo "node"$nodeid":" $(sshpass -p $vmpass ssh $vmuser@$vmaddr $command) &
+                hostaddr=${hostaddrs[i]}
+                let n=$i+1
+                echo "node"$n":" $(sshpass -p $sshpass ssh $sshuser@$hostaddr $command) &
             done
             wait
         else
@@ -223,8 +261,8 @@ if [ $mode = "ssl" ]; then
         # if nodeid is specified, interpret third arg as the ssl account notification email.
         if [ -n "$3" ]; then
             command="$contdir/ssl.sh $3"
-            vmaddr=${vmaddrs[$nodeid]}
-            sshpass -p $vmpass ssh $vmuser@$vmaddr $command
+            hostaddr=${hostaddrs[$nodeid]}
+            sshpass -p $sshpass ssh $sshuser@$hostaddr $command
         else
             echo "Please specify ssl account notification email."
             exit 1
@@ -235,11 +273,11 @@ fi
 
 if [ $mode = "lcl" ]; then
     command="$contdir/lcl.sh"
-    for (( i=0; i<$vmcount; i++ ))
+    for (( i=0; i<$hostcount; i++ ))
     do
-        vmaddr=${vmaddrs[i]}
-        let nodeid=$i+1
-        echo "node"$nodeid":" $(sshpass -p $vmpass ssh $vmuser@$vmaddr $command) &
+        hostaddr=${hostaddrs[i]}
+        let n=$i+1
+        echo "node"$n":" $(sshpass -p $sshpass ssh $sshuser@$hostaddr $command) &
     done
     wait
     exit 0
@@ -248,30 +286,30 @@ fi
 if [ $mode = "pubkey" ]; then
     command="cat $contdir/cfg/hp.cfg | grep public_key | cut -d '\"' -f4"
     if [ $nodeid = -1 ]; then
-        for (( i=0; i<$vmcount; i++ ))
+        for (( i=0; i<$hostcount; i++ ))
         do
-            vmaddr=${vmaddrs[i]}
-            let nodeid=$i+1
-            echo "node"$nodeid":" $(sshpass -p $vmpass ssh $vmuser@$vmaddr $command) &
+            hostaddr=${hostaddrs[i]}
+            let n=$i+1
+            echo "node"$n":" $(sshpass -p $sshpass ssh $sshuser@$hostaddr $command) &
         done
         wait
     else
-        vmaddr=${vmaddrs[$nodeid]}
-        sshpass -p $vmpass ssh $vmuser@$vmaddr $command
+        hostaddr=${hostaddrs[$nodeid]}
+        sshpass -p $sshpass ssh $sshuser@$hostaddr $command
     fi
     exit 0
 fi
 
-# All code below this will only execute in 'new', 'update' or 'reconfig' mode.
+# All code below this will only execute in 'new', 'updatebin' or 'reconfig' mode.
 # Run setup/configuration of entire cluster.
 
 # Copy required files to remote node hpfiles dir.
 
-if [ $mode = "new" ] || [ $mode = "update" ]; then
+if [ $mode = "new" ] || [ $mode = "updatebin" ]; then
     mkdir -p hpfiles/{bin,ssl,nodejs_contract}
     strip $hpcore/build/hpcore
     strip $hpcore/build/appbill
-    cp $hpcore/build/hpcore hpfiles/bin/
+    cp $hpcore/build/{hpcore,hpfs,hpws} hpfiles/bin/
     cp $hpcore/examples/nodejs_contract/{package.json,echo_contract.js,hp-contract-lib.js} \
         hpfiles/nodejs_contract/
 fi
@@ -281,103 +319,143 @@ if [ $mode = "new" ]; then
     cp ./setup-hp.sh hpfiles/
 fi
 
-if [ $mode = "new" ] || [ $mode = "reconfig" ]; then
+if [ $mode = "new" ] || [ $mode = "reconfig" ] || [ $mode = "updateconfig" ]; then
     mkdir ./cfg > /dev/null 2>&1
 fi
 
-# Run vm setup for all nodes.
-for (( i=0; i<$vmcount; i++ ))
-do
-    vmaddr=${vmaddrs[i]}
-    let n=$i+1
+# Running node setup script on specified node or entire cluster.
+if [ $nodeid = -1 ]; then
+    for (( i=0; i<$hostcount; i++ ))
+    do
+        hostaddr=${hostaddrs[i]}
+        let n=$i+1
+        # Setup node. (This will download hp.cfg in 'new', 'reconfig', 'updateconfig' modes)
+        /bin/bash ./setup-node.sh $mode $n $sshuser $sshpass $hostaddr $basedir $contdir &
+    done
+    wait
+else
+    hostaddr=${hostaddrs[$nodeid]}
+    let n=$nodeid+1
+    # Setup node. (This will download hp.cfg in 'new', 'reconfig', 'updateconfig' modes)
+    /bin/bash ./setup-node.sh $mode $n $sshuser $sshpass $hostaddr $basedir $contdir
+fi
 
-    # Setup vm. (This will download hp.cfg in 'new' or 'reconfig' modes)
-    /bin/bash ./setup-vm.sh $mode $n $vmuser $vmpass $vmaddr $basedir $contdir &
-done
-
-wait
 rm -r hpfiles > /dev/null 2>&1
 
-if [ $mode = "update" ]; then
+if [ $mode = "updatebin" ]; then
     exit 0
 fi
 
-# All code below this will only execute in 'new' or 'reconfig' mode.
-# Update downloaded hp.cfg files from all nodes to be part of the same UNL cluster.
+# All code below this will only execute in 'new', 'reconfig' or 'updateconfig' mode.
 
-# Locally update values of download hp.cfg files.
-peerport=$(echo $contconfig | jq -r ".mesh.port")
-for (( i=0; i<$vmcount; i++ ))
-do
-    vmaddr=${vmaddrs[i]}
-    let n=$i+1
+# In 'new' and 'reconfig' modes, update downloaded hp.cfg files from all nodes to be part of the same UNL cluster.
+# In 'updateconfig' mode, simply update the config of specified node or entire cluster.
 
-    # Collect each node's pub key and peer address.
-    pubkeys[i]=$(jq -r ".node.public_key" ./cfg/node$n.cfg)
-    peers[i]="$vmaddr:$peerport"
-done
+if [ $mode = "new" ] || [ $mode = "reconfig" ]; then
 
-# Function to generate JSON array string while skiping a given index.
-function joinarr {
-    arrname=$1[@]
-    arr=("${!arrname}")
-    skip=$2
+    # Locally update values of download hp.cfg files.
+    peerport=$(echo $contconfig | jq -r ".mesh.port")
+    for (( i=0; i<$hostcount; i++ ))
+    do
+        hostaddr=${hostaddrs[i]}
+        let n=$i+1
 
-    let prevlast=$vmcount-2
-    # Resetting prevlast if nothing is given to skip.
-    if [ $skip -lt 0 ]
-    then
-        let prevlast=prevlast+1
+        # Collect each node's pub key and peer address.
+        pubkeys[i]=$(jq -r ".node.public_key" ./cfg/node$n.cfg)
+        peers[i]="$hostaddr:$peerport"
+    done
+
+    # Function to generate JSON array string while skiping a given index.
+    function joinarr {
+        arrname=$1[@]
+        arr=("${!arrname}")
+        skip=$2
+
+        let prevlast=$hostcount-2
+        # Resetting prevlast if nothing is given to skip.
+        if [ $skip -lt 0 ]
+        then
+            let prevlast=prevlast+1
+        fi
+
+        j=0
+        str="["
+        for (( i=0; i<$hostcount; i++ ))
+        do
+            if [ "$i" != "$skip" ]
+            then
+                str="$str\"${arr[i]}\""
+                
+                if [ $j -lt $prevlast ]
+                then
+                    str="$str,"
+                fi
+                let j=j+1
+            fi
+        done
+        str="$str]"
+
+        echo $str # This returns the result.
+    }
+
+    # Loop through all nodes hp.cfg.
+    echo "Generating merged hp.cfg files..."
+    for (( j=0; j<$hostcount; j++ ))
+    do
+        let n=$j+1
+
+        # Prepare peer and unl lists (skip self node for peers).
+        mypeers=$(joinarr peers $j)
+        # Skip param is passed as -1 to stop skipping self pubkey.
+        myunl=$(joinarr pubkeys -1)
+
+        # Merge json contents to produce final config.
+        echo "$(cat ./cfg/node$n.cfg)" \
+            '{"contract": {"id": "3c349abe-4d70-4f50-9fa6-018f1f2530ab", "bin_path": "/usr/bin/node", "bin_args": "'$basedir'/hpfiles/nodejs_contract/echo_contract.js", "unl": '${myunl}'}}'\
+            '{"mesh": {"known_peers": '${mypeers}'}}'\
+            $contconfig \
+            | jq --slurp 'reduce .[] as $item ({}; . * $item)' > ./cfg/node$n-merged.cfg
+    done
+    
+elif [ $mode = "updateconfig" ]; then
+
+    echo "Generating merged hp.cfg files..."
+    if [ $nodeid = -1 ]; then
+        for (( i=0; i<$hostcount; i++ ))
+        do
+            hostaddr=${hostaddrs[i]}
+            let n=$i+1
+
+            # Merge json contents to produce final config.
+            echo "$(cat ./cfg/node$n.cfg)" \
+                $contconfig \
+                | jq --slurp 'reduce .[] as $item ({}; . * $item)' > ./cfg/node$n-merged.cfg
+        done
+    else
+        # Merge json contents to produce final config.
+        let n=$nodeid+1
+        echo "$(cat ./cfg/node$n.cfg)" \
+            $contconfig \
+            | jq --slurp 'reduce .[] as $item ({}; . * $item)' > ./cfg/node$n-merged.cfg
     fi
 
-    j=0
-    str="["
-    for (( i=0; i<$vmcount; i++ ))
+fi
+
+# Upload local hp.cfg files back to specified node or entire cluster.
+echo "Uploading configured hp.cfg..."
+if [ $nodeid = -1 ]; then
+    for (( i=0; i<$hostcount; i++ ))
     do
-        if [ "$i" != "$skip" ]
-        then
-            str="$str\"${arr[i]}\""
-            
-            if [ $j -lt $prevlast ]
-            then
-                str="$str,"
-            fi
-            let j=j+1
-        fi
+        hostaddr=${hostaddrs[i]}
+        let n=$i+1
+
+        sshpass -p $sshpass scp ./cfg/node$n-merged.cfg $sshuser@$hostaddr:$contdir/cfg/hp.cfg &
     done
-    str="$str]"
-
-    echo $str # This returns the result.
-}
-
-# Loop through all nodes hp.cfg.
-for (( j=0; j<$vmcount; j++ ))
-do
-    let n=$j+1
-
-    # Prepare peer and unl lists (skip self node for peers).
-    mypeers=$(joinarr peers $j)
-    # Skip param is passed as -1 to stop skipping self pubkey.
-    myunl=$(joinarr pubkeys -1)
-
-    # Merge json contents to produce final config.
-    echo "$(cat ./cfg/node$n.cfg)" \
-        '{"contract": {"id": "3c349abe-4d70-4f50-9fa6-018f1f2530ab", "bin_path": "/usr/bin/node", "bin_args": "'$basedir'/hpfiles/nodejs_contract/echo_contract.js", "unl": '${myunl}'}}'\
-        '{"mesh": {"known_peers": '${mypeers}'}}'\
-        $contconfig \
-        | jq --slurp 'reduce .[] as $item ({}; . * $item)' > ./cfg/node$n-merged.cfg
-done
-
-for (( j=0; j<$vmcount; j++ ))
-do
-    # Upload local hp.cfg file back to remote vm.
-    let n=$j+1
-    vmaddr=${vmaddrs[j]}
-
-    echo "Uploading configured hp.cfg..."
-    sshpass -p $vmpass scp ./cfg/node$n-merged.cfg $vmuser@$vmaddr:$contdir/cfg/hp.cfg &
-done
-wait
+    wait
+else
+    let n=$nodeid+1
+    sshpass -p $sshpass scp ./cfg/node$n-merged.cfg $sshuser@$hostaddr:$contdir/cfg/hp.cfg
+fi
 
 rm -r ./cfg
 echo "Done."
diff --git a/test/vm-cluster/consensus-test-continuous.sh b/test/vm-cluster/consensus-test-continuous.sh
deleted file mode 100644
index 70c3c4a8..00000000
--- a/test/vm-cluster/consensus-test-continuous.sh
+++ /dev/null
@@ -1,60 +0,0 @@
-#!/bin/bash
-#Script from Richard Holland
-
-WINDOWSIZE=60 # size of window in seconds to examine for successful consensus rounds
-PIPE=concon.pipe
-clusterloc=$(pwd)/hpcluster
-n=1
-let pubport=8080+$n
-while true; do
-    
-    stat $PIPE 2>/dev/null >/dev/null
-    PIPEEXISTS=$?
-    if [ ! "$PIPEEXISTS" -eq "0" ]; then
-        mkfifo $PIPE
-    else
-        dd if=$PIPE iflag=nonblock of=/dev/null 2> /dev/null > /dev/null
-    fi
-
-    exec 9<>$PIPE
-
-    echo 'starting ...'
-    STARTTIME=`date +%s`
-    nohup sudo ~/hpcore run ~/contract > $PIPE 2>> $PIPE 3>MARKER &
-    PID=$!
-    sleep 1
-
-    LASTROUND=0
-    SUCCESSFULCLOSES=0
-    while true
-    do
-        if read -t 1 line <&9; then
-            TSRAW="`echo $line | cut -d" " -f1,2`"
-            TS=`date --date="$TSRAW" +"%s" 2> /dev/null`
-            if [ "$?" -gt " 0" ]; then
-                ISFUSE=`echo $line | grep hpstatefs | wc -l`
-                if [ "$ISFUSE" -gt "0" ]; then
-                    continue
-                fi
-                echo "Irregular line: $line"
-                continue
-            fi
-            if [ "$LASTROUND" -eq "0" ]; then
-                LASTROUND=$TS
-            fi
-            
-            SUCCESS="`echo $line | grep '****Ledger created****' | wc -l`"
-            SUCCESSFULCLOSES="`echo $SUCCESSFULCLOSES + $SUCCESS | bc`"
-
-            SHOULDPRINT=`echo "$TS - $LASTROUND > $WINDOWSIZE" | bc`
-
-            if [ "$SHOULDPRINT" -gt "0" ];
-            then
-                echo "Window ending $TSRAW contained $SUCCESSFULCLOSES successful consensus rounds"
-                SUCCESSFULCLOSES=0
-                LASTROUND=$TS
-            fi
-        fi
-
-    done
-done
\ No newline at end of file
diff --git a/test/vm-cluster/setup-hp.sh b/test/vm-cluster/setup-hp.sh
index e58eaf81..1e7076cb 100755
--- a/test/vm-cluster/setup-hp.sh
+++ b/test/vm-cluster/setup-hp.sh
@@ -3,7 +3,7 @@
 mode=$1
 basedir=$2
 contdir=$3 # Contract directory
-vmaddr=$4
+hostaddr=$4
 
 if [[ ! -f /swapfile ]]
 then
@@ -64,7 +64,7 @@ if [ $mode = "new" ] || [ $mode = "reconfig" ]; then
    popd > /dev/null 2>&1
 
    # Create getpid script (gets process ids belonging to this contract dir)
-   echo "ps -fp \$(pidof \$*) | grep $contdir | awk '{print \$2}' | tr '\n' ' '" > $contdir/getpid.sh
+   echo "pids=\$(pidof \$*) && [ ! -z \"\$pids\" ] && ps -fp \$pids | grep -w $contdir | awk '{print \$2}' | tr '\n' ' '" > $contdir/getpid.sh
    sudo chmod +x $contdir/getpid.sh
 
    # Create start.sh script
@@ -72,28 +72,28 @@ if [ $mode = "new" ] || [ $mode = "reconfig" ]; then
    sudo chmod +x $contdir/start.sh
    
    # Create stop.sh script (sending SIGINT to hpcore)
-   echo "kill -2 \$($contdir/getpid.sh hpcore)" > $contdir/stop.sh
+   echo "pids=\$($contdir/getpid.sh hpcore) && [ ! -z \$pids ] && kill -2 \$pids" > $contdir/stop.sh
    sudo chmod +x $contdir/stop.sh
 
    # Create check.sh script (print pids belonging to this contract dir)
-   echo "echo hpcore pid:\$($contdir/getpid.sh hpcore)  hpfs pid:\$($contdir/getpid.sh hpfs)  hpws pid:\$($contdir/getpid.sh hpws)" > $contdir/check.sh
+   echo "echo hpcore: \$($contdir/getpid.sh hpcore) , hpfs: \$($contdir/getpid.sh hpfs) , hpws: \$($contdir/getpid.sh hpws)" > $contdir/check.sh
    sudo chmod +x $contdir/check.sh
 
    # Create kill.sh script
-   echo "sudo kill \$($contdir/getpid.sh hpcore hpfs hpws)" > $contdir/kill.sh
+   echo "pids=\$($contdir/getpid.sh hpcore hpfs hpws) && [ ! -z \$pids ] && sudo kill \$pids" > $contdir/kill.sh
    sudo chmod +x $contdir/kill.sh
 
    # Create lcl.sh script
    echo "max_shard_no=\$(ls -v $contdir/ledger_fs/seed/primary/ | tail -2 | head -1)" > $contdir/lcl.sh
-   echo "echo \"select seq_no || '-' || lower(hex(ledger_hash)) from ledger order by seq_no DESC limit 1;\" | sqlite3 file:$contdir/ledger_fs/seed/primary/\$max_shard_no/ledger.sqlite?mode=ro" >> $contdir/lcl.sh
+   echo "[ ! -z \$max_shard_no ] && echo \"select seq_no || '-' || lower(hex(ledger_hash)) from ledger order by seq_no DESC limit 1;\" | sqlite3 file:$contdir/ledger_fs/seed/primary/\$max_shard_no/ledger.sqlite?mode=ro" >> $contdir/lcl.sh
    sudo chmod +x $contdir/lcl.sh
 
    # Create ssl.sh script
-   # This installs LetsEncrypt certbot and generates the SSL certs matching with vm domain name.
+   # This installs LetsEncrypt certbot and generates the SSL certs matching with the host's domain name.
    echo "snap install --classic certbot && ln -s /snap/bin/certbot /usr/bin/certbot > /dev/null 2>&1" > $contdir/ssl.sh
-   echo "certbot certonly --standalone -n -m \$1 --agree-tos -d $vmaddr" >> $contdir/ssl.sh
-   echo "cp /etc/letsencrypt/live/$vmaddr/fullchain.pem $basedir/hpfiles/ssl/tlscert.pem" >> $contdir/ssl.sh
-   echo "cp /etc/letsencrypt/live/$vmaddr/privkey.pem $basedir/hpfiles/ssl/tlskey.pem" >> $contdir/ssl.sh
+   echo "certbot certonly --standalone -n -m \$1 --agree-tos -d $hostaddr" >> $contdir/ssl.sh
+   echo "cp /etc/letsencrypt/live/$hostaddr/fullchain.pem $basedir/hpfiles/ssl/tlscert.pem" >> $contdir/ssl.sh
+   echo "cp /etc/letsencrypt/live/$hostaddr/privkey.pem $basedir/hpfiles/ssl/tlskey.pem" >> $contdir/ssl.sh
    echo "cp -rf $basedir/hpfiles/ssl/* $contdir/cfg/" >> $contdir/ssl.sh
    sudo chmod +x $contdir/ssl.sh
 
diff --git a/test/vm-cluster/setup-node.sh b/test/vm-cluster/setup-node.sh
new file mode 100755
index 00000000..42897517
--- /dev/null
+++ b/test/vm-cluster/setup-node.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+
+mode=$1
+nodeid=$2
+sshuser=$3
+sshpass=$4
+hostaddr=$5
+basedir=$6
+contdir=$7 # Contract directory
+
+echo $nodeid. $hostaddr
+
+if [ $mode = "new" ] || [ $mode = "updatebin" ]; then
+    echo "Uploading hp files to $basedir..."
+    sshpass -p $sshpass scp -rp hpfiles $sshuser@$hostaddr:$basedir/
+    echo "Upload finished."
+fi
+
+# Run hp setup script on the node and download the generated hp.cfg
+if [ $mode = "new" ] || [ $mode = "reconfig" ]; then
+    echo "Configuring HP..."
+    sshpass -p $sshpass ssh $sshuser@$hostaddr $basedir/hpfiles/setup-hp.sh $mode $basedir $contdir $hostaddr
+fi
+
+if [ $mode = "new" ] || [ $mode = "reconfig" ] || [ $mode = "updateconfig" ]; then
+    sshpass -p $sshpass scp $sshuser@$hostaddr:$contdir/cfg/hp.cfg ./cfg/node$nodeid.cfg
+fi
\ No newline at end of file
diff --git a/test/vm-cluster/setup-vm.sh b/test/vm-cluster/setup-vm.sh
deleted file mode 100755
index a867cc00..00000000
--- a/test/vm-cluster/setup-vm.sh
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/bin/bash
-
-mode=$1
-nodeid=$2
-vmuser=$3
-vmpass=$4
-vmaddr=$5
-basedir=$6
-contdir=$7 # Contract directory
-
-echo $nodeid. $vmaddr
-
-if [ $mode = "new" ] || [ $mode = "update" ]; then
-    echo "Uploading hp files to $basedir..."
-    sshpass -p $vmpass scp -rp hpfiles $vmuser@$vmaddr:$basedir/
-    echo "Upload finished."
-fi
-
-if [ $mode = "new" ] || [ $mode = "reconfig" ]; then
-    # Run hp setup script on the VM and download the generated hp.cfg
-    echo "Configuring HP..."
-    sshpass -p $vmpass ssh $vmuser@$vmaddr $basedir/hpfiles/setup-hp.sh $mode $basedir $contdir $vmaddr
-    sshpass -p $vmpass scp $vmuser@$vmaddr:$contdir/cfg/hp.cfg ./cfg/node$nodeid.cfg
-fi
\ No newline at end of file