From f54cabf6d0aaa07712f9819d5b725e15ad26d47a Mon Sep 17 00:00:00 2001 From: ravinsp <33562092+ravinsp@users.noreply.github.com> Date: Mon, 16 Dec 2019 12:26:27 +0530 Subject: [PATCH] Added consensus test scripts. --- consensus-test-continuous.sh | 100 +++++++++++++++++++ consensus-test-loop.sh | 53 ++++++++++ test/vm-cluster/consensus-test-continuous.sh | 60 +++++++++++ test/vm-cluster/setup-vm.sh | 3 +- 4 files changed, 215 insertions(+), 1 deletion(-) create mode 100644 consensus-test-continuous.sh create mode 100644 consensus-test-loop.sh create mode 100644 test/vm-cluster/consensus-test-continuous.sh diff --git a/consensus-test-continuous.sh b/consensus-test-continuous.sh new file mode 100644 index 00000000..76b69531 --- /dev/null +++ b/consensus-test-continuous.sh @@ -0,0 +1,100 @@ +#!/bin/bash +#Script from Richard Holland + +WINDOWSIZE=60 # size of window in seconds to examine for successful consensus rounds +PIPE=concon.pipe +clusterloc=$(pwd)/hpcluster +n=1 +let pubport=8080+$n +while true; do + + stat $PIPE 2>/dev/null >/dev/null + PIPEEXISTS=$? + if [ ! "$PIPEEXISTS" -eq "0" ]; then + mkfifo $PIPE + else + dd if=$PIPE iflag=nonblock of=/dev/null 2> /dev/null > /dev/null + fi + + exec 9<>$PIPE + + echo 'starting ...' + STARTTIME=`date +%s` + nohup docker run --rm -t --network=hpnet --name=node${n} \ + -p ${pubport}:${pubport} -a stderr -a stdout \ + --device /dev/fuse --cap-add SYS_ADMIN --security-opt apparmor:unconfined \ + --mount type=bind,source=${clusterloc}/node${n},target=/contract \ + hpcore:latest run /contract > $PIPE 2>> $PIPE 3>MARKER & + PID=$! + sleep 1 + + EXISTINGCONTAINER="" + kill -s 0 $PID 2> /dev/null 1> /dev/null + if [ "$?" -gt "0" ]; + then + echo 'exists, killing existing ...' + + while true; do + read -t 0 line <&9 + if [ ! "$?" -eq "0" ]; then + break; + fi + read -t 1 line <&9 + if [ "`echo $line | grep "already in use by container" | wc -l`" -gt "0" ]; then + EXISTINGCONTAINER=`echo $line | grep -Eo 'container "[a-f0-9]+"' | cut -d '"' -f2` + break + fi + done + + if [ ! -z "$EXISTINGCONTAINER" ]; then + KPID=`ps aux | grep "$EXISTINGCONTAINER" | grep -v grep | awk '{print $2}'` + if [ ! -z "$KPID" ]; then + echo "killing $KPID" + kill -9 $KPID + fi + fi + continue + else + disown $PID + echo "running" + fi + + LASTROUND=0 + SUCCESSFULCLOSES=0 + while true + do + if read -t 1 line <&9; then + TSRAW="`echo $line | cut -d" " -f1,2`" + TS=`date --date="$TSRAW" +"%s" 2> /dev/null` + if [ "$?" -gt " 0" ]; then + ISFUSE=`echo $line | grep hpstatefs | wc -l` + if [ "$ISFUSE" -gt "0" ]; then + continue + fi + echo "Irregular line: $line" + continue + fi + if [ "$LASTROUND" -eq "0" ]; then + LASTROUND=$TS + fi + + SUCCESS="`echo $line | grep '****Stage 3 consensus reached****' | wc -l`" + SUCCESSFULCLOSES="`echo $SUCCESSFULCLOSES + $SUCCESS | bc`" + + SHOULDPRINT=`echo "$TS - $LASTROUND > $WINDOWSIZE" | bc` + + if [ "$SHOULDPRINT" -gt "0" ]; + then + echo "Window ending $TSRAW contained $SUCCESSFULCLOSES successful consensus rounds" + SUCCESSFULCLOSES=0 + LASTROUND=$TS + fi + fi + + kill -s 0 $PID > /dev/null 2> /dev/null + if [ "$?" -gt "0" ]; then + echo docker process went away, quitting + exit + fi + done +done \ No newline at end of file diff --git a/consensus-test-loop.sh b/consensus-test-loop.sh new file mode 100644 index 00000000..716da8db --- /dev/null +++ b/consensus-test-loop.sh @@ -0,0 +1,53 @@ +#!/bin/bash +#Script from Richard Holland + +clusterloc=$(pwd)/hpcluster +n=1 +let pubport=8080+$n +while true; do + CONSENSUS="0" + STARTTIME=`date +%s` + nohup docker run --rm -t --network=hpnet --name=node${n} \ + -p ${pubport}:${pubport} -a stderr -a stdout \ + --device /dev/fuse --cap-add SYS_ADMIN --security-opt apparmor:unconfined \ + --mount type=bind,source=${clusterloc}/node${n},target=/contract \ + hpcore:latest run /contract > contest.out 2>> contest.out & + PID=$! + sleep 1 + kill -s 0 $PID 2> /dev/null 1> /dev/null + if [ "$?" -gt "0" ]; + then + CONTAINERID=`cat contest.out | grep -Eo 'container "[a-f0-9]+"' | cut -d '"' -f2` + if [ ! -z "$CONTAINERID" ]; then + KPID=`ps aux | grep "$CONTAINERID " | grep -v grep | awk '{print $2}'` + if [ ! -z "$KPID" ]; then + kill -9 $KPID + continue + fi + fi + else + disown $PID + fi + echo "Started waiting for consensus on $PID at $STARTTIME" + kill -s 0 $PID 2> /dev/null 1> /dev/null + while [ "$?" -eq "0" ]; + do + if [ "`cat contest.out | grep '****Stage 3 consensus reached****' | wc -l`" -gt "0" ]; + then + break + fi + sleep 1 + UPTOTIME=`date +%s` + TIMEELAPSED=`echo $UPTOTIME - $STARTTIME | bc ` + SHOULDPRINT=`echo $TIMEELAPSED % 10 | bc` + if [ "$SHOULDPRINT" -eq "0" ]; then + echo "Still waiting after $TIMEELAPSED" + fi + kill -s 0 $PID 2> /dev/null 1> /dev/null + done + ENDTIME=`date +%s` + TIMEELAPSED=`echo $ENDTIME - $STARTTIME | bc ` + echo "Reached consensus after $TIMEELAPSED seconds [PID=$PID], killing and restarting..." + nohup kill -9 $PID 2> /dev/null 1> /dev/null + sleep 1 +done \ No newline at end of file diff --git a/test/vm-cluster/consensus-test-continuous.sh b/test/vm-cluster/consensus-test-continuous.sh new file mode 100644 index 00000000..e0c04084 --- /dev/null +++ b/test/vm-cluster/consensus-test-continuous.sh @@ -0,0 +1,60 @@ +#!/bin/bash +#Script from Richard Holland + +WINDOWSIZE=60 # size of window in seconds to examine for successful consensus rounds +PIPE=concon.pipe +clusterloc=$(pwd)/hpcluster +n=1 +let pubport=8080+$n +while true; do + + stat $PIPE 2>/dev/null >/dev/null + PIPEEXISTS=$? + if [ ! "$PIPEEXISTS" -eq "0" ]; then + mkfifo $PIPE + else + dd if=$PIPE iflag=nonblock of=/dev/null 2> /dev/null > /dev/null + fi + + exec 9<>$PIPE + + echo 'starting ...' + STARTTIME=`date +%s` + nohup ~/hpcore run ~/contract > $PIPE 2>> $PIPE 3>MARKER & + PID=$! + sleep 1 + + LASTROUND=0 + SUCCESSFULCLOSES=0 + while true + do + if read -t 1 line <&9; then + TSRAW="`echo $line | cut -d" " -f1,2`" + TS=`date --date="$TSRAW" +"%s" 2> /dev/null` + if [ "$?" -gt " 0" ]; then + ISFUSE=`echo $line | grep hpstatefs | wc -l` + if [ "$ISFUSE" -gt "0" ]; then + continue + fi + echo "Irregular line: $line" + continue + fi + if [ "$LASTROUND" -eq "0" ]; then + LASTROUND=$TS + fi + + SUCCESS="`echo $line | grep '****Stage 3 consensus reached****' | wc -l`" + SUCCESSFULCLOSES="`echo $SUCCESSFULCLOSES + $SUCCESS | bc`" + + SHOULDPRINT=`echo "$TS - $LASTROUND > $WINDOWSIZE" | bc` + + if [ "$SHOULDPRINT" -gt "0" ]; + then + echo "Window ending $TSRAW contained $SUCCESSFULCLOSES successful consensus rounds" + SUCCESSFULCLOSES=0 + LASTROUND=$TS + fi + fi + + done +done \ No newline at end of file diff --git a/test/vm-cluster/setup-vm.sh b/test/vm-cluster/setup-vm.sh index a65b5e24..1f8a6663 100755 --- a/test/vm-cluster/setup-vm.sh +++ b/test/vm-cluster/setup-vm.sh @@ -15,10 +15,11 @@ if [ $mode = "new" ]; then $hpcore/examples/echocontract/contract.js \ /usr/local/lib/x86_64-linux-gnu/libfuse3.so.3 \ /usr/local/bin/fusermount3 \ + ./consensus-test-continuous.sh \ ./setup-hp.sh \ geveo@$vmip:~/ - sshpass -p $vmpass ssh geveo@$vmip 'chmod 700 ~/setup-hp.sh && ~/setup-hp.sh' + sshpass -p $vmpass ssh geveo@$vmip 'chmod 700 ~/consensus-test-continuous.sh && chmod 700 ~/setup-hp.sh && ~/setup-hp.sh' sshpass -p $vmpass scp geveo@$vmip:~/contract/cfg/hp.cfg ./cfg/node$nodeid.json else sshpass -p $vmpass scp $hpcore/build/hpcore \