From d82ab8f626002d3933b2bd57049341f8e40900e6 Mon Sep 17 00:00:00 2001
From: Ravin Perera <33562092+ravinsp@users.noreply.github.com>
Date: Fri, 20 Dec 2019 11:29:52 +0530
Subject: [PATCH] Reorganized local and remote cluster setup scripts. (#75)

---
 .dockerignore                                 |   5 --
 .gitignore                                    |   3 +-
 CMakeLists.txt                                |   5 +-
 .../.gitignore                                |   0
 examples/echo_contract/Dockerfile             |  17 ++++
 .../contract.js                               |   0
 examples/echocontract/Dockerfile              |  17 ----
 examples/hpclient/.gitignore                  |   2 +-
 examples/random_contract/rnd_contrac.c        |  82 ++++++++++++++++++
 .../rnd_contract                              | Bin
 fusermount3 => fusebin/fusermount3            | Bin
 libfuse3.so.3 => fusebin/libfuse3.so.3        | Bin
 fusebin/readme.txt                            |   2 +
 reset.sh                                      |  17 ----
 test/.gitignore                               |   1 -
 test/comm-tests/.gitignore                    |   1 +
 test/{ => comm-tests}/global.js               |   0
 test/{ => comm-tests}/package-lock.json       |   0
 test/{ => comm-tests}/package.json            |   0
 test/{ => comm-tests}/test.js                 |   0
 test/{ => comm-tests}/test_challenge.js       |   0
 test/local-cluster/.dockerignore              |   1 +
 test/local-cluster/.gitignore                 |   2 +
 Dockerfile => test/local-cluster/Dockerfile   |   8 +-
 .../local-cluster/cluster-create.sh           |  10 +--
 .../local-cluster/cluster-start.sh            |   0
 .../consensus-test-continuous.sh              |   0
 .../local-cluster/consensus-test-loop.sh      |   0
 test/vm-cluster/cluster.sh                    |  18 +++-
 test/vm-cluster/setup-vm.sh                   |  12 +--
 30 files changed, 143 insertions(+), 60 deletions(-)
 delete mode 100644 .dockerignore
 rename examples/{echocontract => echo_contract}/.gitignore (100%)
 create mode 100644 examples/echo_contract/Dockerfile
 rename examples/{echocontract => echo_contract}/contract.js (100%)
 delete mode 100644 examples/echocontract/Dockerfile
 create mode 100644 examples/random_contract/rnd_contrac.c
 rename examples/{rndcontract => random_contract}/rnd_contract (100%)
 rename fusermount3 => fusebin/fusermount3 (100%)
 rename libfuse3.so.3 => fusebin/libfuse3.so.3 (100%)
 create mode 100644 fusebin/readme.txt
 delete mode 100755 reset.sh
 delete mode 100644 test/.gitignore
 create mode 100644 test/comm-tests/.gitignore
 rename test/{ => comm-tests}/global.js (100%)
 rename test/{ => comm-tests}/package-lock.json (100%)
 rename test/{ => comm-tests}/package.json (100%)
 rename test/{ => comm-tests}/test.js (100%)
 rename test/{ => comm-tests}/test_challenge.js (100%)
 create mode 100644 test/local-cluster/.dockerignore
 create mode 100644 test/local-cluster/.gitignore
 rename Dockerfile => test/local-cluster/Dockerfile (71%)
 rename cluster-create.sh => test/local-cluster/cluster-create.sh (91%)
 rename cluster-start.sh => test/local-cluster/cluster-start.sh (100%)
 rename consensus-test-continuous.sh => test/local-cluster/consensus-test-continuous.sh (100%)
 rename consensus-test-loop.sh => test/local-cluster/consensus-test-loop.sh (100%)

diff --git a/.dockerignore b/.dockerignore
deleted file mode 100644
index d4eb2f11..00000000
--- a/.dockerignore
+++ /dev/null
@@ -1,5 +0,0 @@
-**/**
-!build/hpcore
-!build/hpstatemon
-!libfuse3.so.3
-!fusermount3
diff --git a/.gitignore b/.gitignore
index 35a93b5d..11c3520b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -9,5 +9,4 @@ hpsupport.dir
 **/Makefile
 **/CMakeCache.txt
 **/cmake_install.cmake
-**/CMakeFiles/**
-hpcluster
\ No newline at end of file
+**/CMakeFiles
\ No newline at end of file
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 1818023b..e0014f64 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -150,7 +150,10 @@ target_precompile_headers(hpstatemon REUSE_FROM hpsupport)
 # Create docker image from hpcore build output with 'make docker'
 # Requires docker to be runnable without 'sudo'
 add_custom_target(docker
-  COMMAND docker build -t hpcore:latest .
+  COMMAND mkdir -p ./test/local-cluster/bin
+  COMMAND cp ./build/hpcore ./build/hpstatemon ./build/appbill ./test/local-cluster/bin/
+  COMMAND cp ./fusebin/fusermount3 ./fusebin/libfuse3.so.3 ./test/local-cluster/bin/
+  COMMAND docker build -t hpcore:latest ./test/local-cluster
 )
 set_target_properties(docker PROPERTIES EXCLUDE_FROM_ALL TRUE)
 add_dependencies(docker hpcore)
diff --git a/examples/echocontract/.gitignore b/examples/echo_contract/.gitignore
similarity index 100%
rename from examples/echocontract/.gitignore
rename to examples/echo_contract/.gitignore
diff --git a/examples/echo_contract/Dockerfile b/examples/echo_contract/Dockerfile
new file mode 100644
index 00000000..5b33eb36
--- /dev/null
+++ b/examples/echo_contract/Dockerfile
@@ -0,0 +1,17 @@
+FROM hpcore:latest as hpcore-nodejs
+
+RUN apt-get update
+RUN apt-get install -y build-essential
+RUN apt-get install -y curl
+RUN curl -sL https://deb.nodesource.com/setup_10.x | bash -
+RUN apt-get install -y nodejs
+
+FROM hpcore-nodejs
+
+RUN /hp/hpcore new /echo_contract
+COPY ./examples/echo_contract/contract.js /echo_contract/bin/contract.js
+RUN mv /echo_contract/cfg/hp.cfg /echo_contract/cfg/hp.json
+RUN node -p "JSON.stringify({...require('/echo_contract/cfg/hp.json'), binary:'/usr/bin/node', binargs:'./bin/contract.js' }, null, 2)" > /echo_contract/cfg/hp.cfg
+RUN rm /echo_contract/cfg/hp.json
+
+ENTRYPOINT ["/hp/hpcore", "run", "/echo_contract"]
\ No newline at end of file
diff --git a/examples/echocontract/contract.js b/examples/echo_contract/contract.js
similarity index 100%
rename from examples/echocontract/contract.js
rename to examples/echo_contract/contract.js
diff --git a/examples/echocontract/Dockerfile b/examples/echocontract/Dockerfile
deleted file mode 100644
index d0b602a6..00000000
--- a/examples/echocontract/Dockerfile
+++ /dev/null
@@ -1,17 +0,0 @@
-FROM hpcore:latest as hpcore-nodejs
-
-RUN apt-get update
-RUN apt-get install -y build-essential
-RUN apt-get install -y curl
-RUN curl -sL https://deb.nodesource.com/setup_10.x | bash -
-RUN apt-get install -y nodejs
-
-FROM hpcore-nodejs
-
-RUN /hp/hpcore new /echocontract
-COPY ./examples/echocontract/contract.js /echocontract/bin/contract.js
-RUN mv /echocontract/cfg/hp.cfg /echocontract/cfg/hp.json
-RUN node -p "JSON.stringify({...require('/echocontract/cfg/hp.json'), binary:'/usr/bin/node', binargs:'./bin/contract.js' }, null, 2)" > /echocontract/cfg/hp.cfg
-RUN rm /echocontract/cfg/hp.json
-
-ENTRYPOINT ["/hp/hpcore", "run", "/echocontract"]
\ No newline at end of file
diff --git a/examples/hpclient/.gitignore b/examples/hpclient/.gitignore
index 394522f4..b512c09d 100644
--- a/examples/hpclient/.gitignore
+++ b/examples/hpclient/.gitignore
@@ -1 +1 @@
-node_modules/**
\ No newline at end of file
+node_modules
\ No newline at end of file
diff --git a/examples/random_contract/rnd_contrac.c b/examples/random_contract/rnd_contrac.c
new file mode 100644
index 00000000..c4d65875
--- /dev/null
+++ b/examples/random_contract/rnd_contrac.c
@@ -0,0 +1,82 @@
+/*
+ * This is a simple program that makes a series of random modifications to an arbitrarily large specified file
+ * designed as part of a test for file-ptrace and merkle tree generation and updating for hotpocket
+ * Originally from: https://github.com/codetsunami/file-ptracer/blob/master/rnd_contract.c
+ */
+
+// Compile with: gcc rnd_contract.c -o rnd_contract -g
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+int main(int argc, char **argv)
+{
+
+    if (argc != 3)
+        return fprintf(stderr, "usage %s <file path to randomly edit> <max no. of block modifications>\n", argv[0]);
+
+    // Read from input ts
+    char buf[128];
+    read(STDIN_FILENO, buf, 128);
+
+    char tsbuf[14];
+    memcpy(tsbuf, &buf[100], 13);
+    tsbuf[13] = '\0';
+    int ts = atoi(tsbuf);
+    //printf("args input: %.13s\n", tsbuf);
+
+    srand(ts);
+
+    FILE *f = fopen(argv[1], "rb+");
+    if (!f)
+        return fprintf(stderr, "could not open file %s\n", argv[1]);
+
+    int max_mod = strtol(argv[2], NULL, 10);
+
+    int fd = fileno(f);
+
+    fseek(f, 0L, SEEK_END);
+    size_t len = ftell(f);
+
+    // we don't need to rewind because now we'll use pwrite
+
+    // pick a random number of modications to make between 1 and 50
+    int mods = rand() % max_mod + 1;
+
+    pid_t pid = getpid();
+
+    for (int i = 0; i < mods; ++i)
+    {
+        // pick a random file offset
+        size_t offset = rand() % len;
+
+        // pick a random number of bytes to write, up to 100
+        size_t bytestowrite = rand() % 100;
+
+        char buf[100];
+
+        // write the bytes to a buffer
+        for (int n = 0; n < bytestowrite; ++n)
+            buf[n] = rand() % 0xff;
+
+        // write the buffer to the random file location
+        int n = pwrite(fd, buf, bytestowrite, offset);
+        if (!n)
+            continue;
+        int start_block = offset / (4 * 1024 * 1024);
+        int end_block = (offset + bytestowrite) / (4 * 1024 * 1024);
+        for (int i = start_block; i <= end_block; ++i)
+        {
+            //printf("@@@ pid %d wrote to block %d ... %d bytes\n", pid, i, n);
+            fflush(stdout);
+        }
+    }
+
+    // done!
+
+    fclose(f);
+}
diff --git a/examples/rndcontract/rnd_contract b/examples/random_contract/rnd_contract
similarity index 100%
rename from examples/rndcontract/rnd_contract
rename to examples/random_contract/rnd_contract
diff --git a/fusermount3 b/fusebin/fusermount3
similarity index 100%
rename from fusermount3
rename to fusebin/fusermount3
diff --git a/libfuse3.so.3 b/fusebin/libfuse3.so.3
similarity index 100%
rename from libfuse3.so.3
rename to fusebin/libfuse3.so.3
diff --git a/fusebin/readme.txt b/fusebin/readme.txt
new file mode 100644
index 00000000..6733ff3d
--- /dev/null
+++ b/fusebin/readme.txt
@@ -0,0 +1,2 @@
+Files in this directory contains binaries required for fuse integration for Ubuntu/Debian environment.
+This is used for test cluster setup scripts.
\ No newline at end of file
diff --git a/reset.sh b/reset.sh
deleted file mode 100755
index 023edd36..00000000
--- a/reset.sh
+++ /dev/null
@@ -1,17 +0,0 @@
-#!/bin/bash
-
-nodes=3
-sudo ./cluster-create.sh $nodes
-WD=`pwd`
-# Setup initial state data for all nodes but one.
-for (( i=1; i<$nodes; i++ ))
-do
-    
-    sudo mkdir -p ~/hpcore/hpcluster/node$i/statehist/0/data/
-    pushd ~/hpcore/hpcluster/node$i/statehist/0/data/
-    >appbill.table
-    $WD/build/appbill --credit 705bf26354ee4c63c0e5d5d883c07cefc3196d049bd3825f827eb3bc23ead035 10000
-    popd
-    #sudo cp -r ~/Downloads/big.mkv ~/hpcore/hpcluster/node$i/statehist/0/data/
-
-done
diff --git a/test/.gitignore b/test/.gitignore
deleted file mode 100644
index 394522f4..00000000
--- a/test/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-node_modules/**
\ No newline at end of file
diff --git a/test/comm-tests/.gitignore b/test/comm-tests/.gitignore
new file mode 100644
index 00000000..b512c09d
--- /dev/null
+++ b/test/comm-tests/.gitignore
@@ -0,0 +1 @@
+node_modules
\ No newline at end of file
diff --git a/test/global.js b/test/comm-tests/global.js
similarity index 100%
rename from test/global.js
rename to test/comm-tests/global.js
diff --git a/test/package-lock.json b/test/comm-tests/package-lock.json
similarity index 100%
rename from test/package-lock.json
rename to test/comm-tests/package-lock.json
diff --git a/test/package.json b/test/comm-tests/package.json
similarity index 100%
rename from test/package.json
rename to test/comm-tests/package.json
diff --git a/test/test.js b/test/comm-tests/test.js
similarity index 100%
rename from test/test.js
rename to test/comm-tests/test.js
diff --git a/test/test_challenge.js b/test/comm-tests/test_challenge.js
similarity index 100%
rename from test/test_challenge.js
rename to test/comm-tests/test_challenge.js
diff --git a/test/local-cluster/.dockerignore b/test/local-cluster/.dockerignore
new file mode 100644
index 00000000..7c2995c6
--- /dev/null
+++ b/test/local-cluster/.dockerignore
@@ -0,0 +1 @@
+hpcluster
\ No newline at end of file
diff --git a/test/local-cluster/.gitignore b/test/local-cluster/.gitignore
new file mode 100644
index 00000000..b469312f
--- /dev/null
+++ b/test/local-cluster/.gitignore
@@ -0,0 +1,2 @@
+bin
+hpcluster
\ No newline at end of file
diff --git a/Dockerfile b/test/local-cluster/Dockerfile
similarity index 71%
rename from Dockerfile
rename to test/local-cluster/Dockerfile
index c88d5142..2bdb0cff 100644
--- a/Dockerfile
+++ b/test/local-cluster/Dockerfile
@@ -4,12 +4,12 @@ FROM node:10.17.0-buster-slim
 
 # Install fuse.
 # Copy fuse shared library and register it.
-COPY ./libfuse3.so.3 /usr/local/lib/
+COPY ./bin/libfuse3.so.3 /usr/local/lib/
 RUN ldconfig
-COPY ./fusermount3 /usr/local/bin/
+COPY ./bin/fusermount3 /usr/local/bin/
 
 # hpcore binary is copied to /hp directory withtin the docker image.
 WORKDIR /hp
-COPY ./build/hpcore .
-COPY ./build/hpstatemon .
+COPY ./bin/hpcore .
+COPY ./bin/hpstatemon .
 ENTRYPOINT ["/hp/hpcore"]
\ No newline at end of file
diff --git a/cluster-create.sh b/test/local-cluster/cluster-create.sh
similarity index 91%
rename from cluster-create.sh
rename to test/local-cluster/cluster-create.sh
index c112406b..19839dbf 100755
--- a/cluster-create.sh
+++ b/test/local-cluster/cluster-create.sh
@@ -28,7 +28,7 @@ do
     let pubport=8080+$n
 
     # Create contract dir named "node<i>"
-    ../build/hpcore new "node${n}" > /dev/null 2>&1
+    ../bin/hpcore new "node${n}" > /dev/null 2>&1
 
     pushd ./node$n/cfg > /dev/null 2>&1
 
@@ -65,8 +65,8 @@ do
 
     # Copy the contract executable and appbill.
     mkdir ./node$n/bin
-    cp ../examples/echocontract/contract.js ./node$n/bin/contract.js
-    cp ../build/appbill ./node$n/bin/
+    cp ../../../examples/echo_contract/contract.js ./node$n/bin/contract.js
+    cp ../bin/appbill ./node$n/bin/
 done
 
 # Function to generate JSON array string while skiping a given index.
@@ -114,12 +114,12 @@ done
 for (( i=1; i<=$ncount; i++ ))
 do
 
-    sudo mkdir -p ./node$i/statehist/0/data/ > /dev/null 2>&1
+    mkdir -p ./node$i/statehist/0/data/ > /dev/null 2>&1
 
     # Load credit balance for user for testing purposes.
     pushd ./node$i/statehist/0/data/ > /dev/null 2>&1
     >appbill.table
-    ../../../../../build/appbill --credit "705bf26354ee4c63c0e5d5d883c07cefc3196d049bd3825f827eb3bc23ead035" 10000
+    ../../../../../bin/appbill --credit "705bf26354ee4c63c0e5d5d883c07cefc3196d049bd3825f827eb3bc23ead035" 10000
     popd > /dev/null 2>&1
 
     # Copy any more initial state files for testing.
diff --git a/cluster-start.sh b/test/local-cluster/cluster-start.sh
similarity index 100%
rename from cluster-start.sh
rename to test/local-cluster/cluster-start.sh
diff --git a/consensus-test-continuous.sh b/test/local-cluster/consensus-test-continuous.sh
similarity index 100%
rename from consensus-test-continuous.sh
rename to test/local-cluster/consensus-test-continuous.sh
diff --git a/consensus-test-loop.sh b/test/local-cluster/consensus-test-loop.sh
similarity index 100%
rename from consensus-test-loop.sh
rename to test/local-cluster/consensus-test-loop.sh
diff --git a/test/vm-cluster/cluster.sh b/test/vm-cluster/cluster.sh
index 83a4016b..9cad4c20 100755
--- a/test/vm-cluster/cluster.sh
+++ b/test/vm-cluster/cluster.sh
@@ -1,5 +1,9 @@
 #!/bin/bash
+
+# VM login password must exist in vmpass.txt
 vmpass=$(cat vmpass.txt)
+# List vm IP addresses of the cluster must exist in iplist.txt
+# (This list will be treated as the node numbers 1,2.3... from topmost IP to the bottom)
 readarray -t vmips < iplist.txt
 
 vmcount=${#vmips[@]}
@@ -9,12 +13,22 @@ hpcore=$(realpath ../..)
 
 if [ "$mode" = "new" ] || [ "$mode" = "update" ] || [ "$mode" = "run" ] || [ "$mode" = "check" ] || \
    [ "$mode" = "connect" ] || [ "$mode" = "kill" ] || [ "$mode" = "reboot" ] || [ "$mode" = "ssh" ]; then
-    echo ""
+    echo "mode: $mode"
 else
     echo "Invalid command. [ new | update | run <N> | check <N> | connect <N> | kill <N> | reboot <N> | ssh <N> <custom command> ] expected."
     exit 1
 fi
 
+# Command modes:
+# new - Install hot pocket dependencies and hot pocket to each vm.
+# update - Deploy updated hot pocket binaries into each vm.
+# run - Run hot pocket of specified vm node.
+# check - Check hot pocket running status of specified vm node.
+# connect - Connect to hot pocket console output (if running) of specified vm node.
+# kill - Kill hot pocket (if running) of specified vm node.
+# reboot - Reboot specified vm node.
+# ssh - Open up an ssh terminal for the specified vm node.
+
 if [ $mode = "run" ]; then
     let nodeid=$2-1
     vmip=${vmips[$nodeid]}
@@ -74,6 +88,8 @@ if [ $mode = "update" ]; then
     exit 0
 fi
 
+# Following code will only be executed in 'new' mode.
+
 for (( i=0; i<$vmcount; i++ ))
 do
     vmip=${vmips[i]}
diff --git a/test/vm-cluster/setup-vm.sh b/test/vm-cluster/setup-vm.sh
index 6f2a9322..242e2f1b 100755
--- a/test/vm-cluster/setup-vm.sh
+++ b/test/vm-cluster/setup-vm.sh
@@ -12,10 +12,10 @@ if [ $mode = "new" ]; then
 
     sshpass -p $vmpass scp $hpcore/build/hpcore \
                             $hpcore/build/hpstatemon \
-                            $hpcore/examples/echocontract/contract.js \
-                            $hpcore/examples/rndcontract/rnd_contract \
-                            /usr/local/lib/x86_64-linux-gnu/libfuse3.so.3 \
-                            /usr/local/bin/fusermount3 \
+                            $hpcore/examples/echo_contract/contract.js \
+                            $hpcore/examples/random_contract/rnd_contract \
+                            $hpcore/fusebin/libfuse3.so.3 \
+                            $hpcore/fusebin/fusermount3 \
                             ./consensus-test-continuous.sh \
                             ./setup-hp.sh \
                             geveo@$vmip:~/
@@ -25,8 +25,8 @@ if [ $mode = "new" ]; then
 else
     sshpass -p $vmpass scp $hpcore/build/hpcore \
                             $hpcore/build/hpstatemon \
-                            $hpcore/examples/echocontract/contract.js \
-                            $hpcore/examples/rndcontract/rnd_contract \
+                            $hpcore/examples/echo_contract/contract.js \
+                            $hpcore/examples/random_contract/rnd_contract \
                             ./consensus-test-continuous.sh \
                             geveo@$vmip:~/
 fi
\ No newline at end of file