Reorganized local and remote cluster setup scripts. (#75)

.dockerignore

@@ -1,5 +0,0 @@
-**/**
-!build/hpcore
-!build/hpstatemon
-!libfuse3.so.3
-!fusermount3

.gitignore

@@ -9,5 +9,4 @@ hpsupport.dir
 **/Makefile
 **/CMakeCache.txt
 **/cmake_install.cmake
-**/CMakeFiles/**
-hpcluster
+**/CMakeFiles

CMakeLists.txt

@@ -150,7 +150,10 @@ target_precompile_headers(hpstatemon REUSE_FROM hpsupport)
 # Create docker image from hpcore build output with 'make docker'
 # Requires docker to be runnable without 'sudo'
 add_custom_target(docker
-  COMMAND docker build -t hpcore:latest .
+  COMMAND mkdir -p ./test/local-cluster/bin
+  COMMAND cp ./build/hpcore ./build/hpstatemon ./build/appbill ./test/local-cluster/bin/
+  COMMAND cp ./fusebin/fusermount3 ./fusebin/libfuse3.so.3 ./test/local-cluster/bin/
+  COMMAND docker build -t hpcore:latest ./test/local-cluster
 )
 set_target_properties(docker PROPERTIES EXCLUDE_FROM_ALL TRUE)
 add_dependencies(docker hpcore)

examples/echo_contract/Dockerfile

@@ -0,0 +1,17 @@
+FROM hpcore:latest as hpcore-nodejs
+
+RUN apt-get update
+RUN apt-get install -y build-essential
+RUN apt-get install -y curl
+RUN curl -sL https://deb.nodesource.com/setup_10.x | bash -
+RUN apt-get install -y nodejs
+
+FROM hpcore-nodejs
+
+RUN /hp/hpcore new /echo_contract
+COPY ./examples/echo_contract/contract.js /echo_contract/bin/contract.js
+RUN mv /echo_contract/cfg/hp.cfg /echo_contract/cfg/hp.json
+RUN node -p "JSON.stringify({...require('/echo_contract/cfg/hp.json'), binary:'/usr/bin/node', binargs:'./bin/contract.js' }, null, 2)" > /echo_contract/cfg/hp.cfg
+RUN rm /echo_contract/cfg/hp.json
+
+ENTRYPOINT ["/hp/hpcore", "run", "/echo_contract"]

examples/echocontract/Dockerfile

@@ -1,17 +0,0 @@
-FROM hpcore:latest as hpcore-nodejs
-
-RUN apt-get update
-RUN apt-get install -y build-essential
-RUN apt-get install -y curl
-RUN curl -sL https://deb.nodesource.com/setup_10.x | bash -
-RUN apt-get install -y nodejs
-
-FROM hpcore-nodejs
-
-RUN /hp/hpcore new /echocontract
-COPY ./examples/echocontract/contract.js /echocontract/bin/contract.js
-RUN mv /echocontract/cfg/hp.cfg /echocontract/cfg/hp.json
-RUN node -p "JSON.stringify({...require('/echocontract/cfg/hp.json'), binary:'/usr/bin/node', binargs:'./bin/contract.js' }, null, 2)" > /echocontract/cfg/hp.cfg
-RUN rm /echocontract/cfg/hp.json
-
-ENTRYPOINT ["/hp/hpcore", "run", "/echocontract"]

examples/hpclient/.gitignore

@@ -1 +1 @@
-node_modules/**
+node_modules

examples/random_contract/rnd_contrac.c

@@ -0,0 +1,82 @@
+/*
+ * This is a simple program that makes a series of random modifications to an arbitrarily large specified file
+ * designed as part of a test for file-ptrace and merkle tree generation and updating for hotpocket
+ * Originally from: https://github.com/codetsunami/file-ptracer/blob/master/rnd_contract.c
+ */
+
+// Compile with: gcc rnd_contract.c -o rnd_contract -g
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+int main(int argc, char **argv)
+{
+
+    if (argc != 3)
+        return fprintf(stderr, "usage %s <file path to randomly edit> <max no. of block modifications>\n", argv[0]);
+
+    // Read from input ts
+    char buf[128];
+    read(STDIN_FILENO, buf, 128);
+
+    char tsbuf[14];
+    memcpy(tsbuf, &buf[100], 13);
+    tsbuf[13] = '\0';
+    int ts = atoi(tsbuf);
+    //printf("args input: %.13s\n", tsbuf);
+
+    srand(ts);
+
+    FILE *f = fopen(argv[1], "rb+");
+    if (!f)
+        return fprintf(stderr, "could not open file %s\n", argv[1]);
+
+    int max_mod = strtol(argv[2], NULL, 10);
+
+    int fd = fileno(f);
+
+    fseek(f, 0L, SEEK_END);
+    size_t len = ftell(f);
+
+    // we don't need to rewind because now we'll use pwrite
+
+    // pick a random number of modications to make between 1 and 50
+    int mods = rand() % max_mod + 1;
+
+    pid_t pid = getpid();
+
+    for (int i = 0; i < mods; ++i)
+    {
+        // pick a random file offset
+        size_t offset = rand() % len;
+
+        // pick a random number of bytes to write, up to 100
+        size_t bytestowrite = rand() % 100;
+
+        char buf[100];
+
+        // write the bytes to a buffer
+        for (int n = 0; n < bytestowrite; ++n)
+            buf[n] = rand() % 0xff;
+
+        // write the buffer to the random file location
+        int n = pwrite(fd, buf, bytestowrite, offset);
+        if (!n)
+            continue;
+        int start_block = offset / (4 * 1024 * 1024);
+        int end_block = (offset + bytestowrite) / (4 * 1024 * 1024);
+        for (int i = start_block; i <= end_block; ++i)
+        {
+            //printf("@@@ pid %d wrote to block %d ... %d bytes\n", pid, i, n);
+            fflush(stdout);
+        }
+    }
+
+    // done!
+
+    fclose(f);
+}

fusebin/readme.txt

@@ -0,0 +1,2 @@
+Files in this directory contains binaries required for fuse integration for Ubuntu/Debian environment.
+This is used for test cluster setup scripts.

reset.sh

@@ -1,17 +0,0 @@
-#!/bin/bash
-
-nodes=3
-sudo ./cluster-create.sh $nodes
-WD=`pwd`
-# Setup initial state data for all nodes but one.
-for (( i=1; i<$nodes; i++ ))
-do
-    
-    sudo mkdir -p ~/hpcore/hpcluster/node$i/statehist/0/data/
-    pushd ~/hpcore/hpcluster/node$i/statehist/0/data/
-    >appbill.table
-    $WD/build/appbill --credit 705bf26354ee4c63c0e5d5d883c07cefc3196d049bd3825f827eb3bc23ead035 10000
-    popd
-    #sudo cp -r ~/Downloads/big.mkv ~/hpcore/hpcluster/node$i/statehist/0/data/
-
-done

test/.gitignore

@@ -1 +0,0 @@
-node_modules/**

test/comm-tests/.gitignore

@@ -0,0 +1 @@
+node_modules

test/local-cluster/.dockerignore

@@ -0,0 +1 @@
+hpcluster

test/local-cluster/.gitignore

@@ -0,0 +1,2 @@
+bin
+hpcluster

test/local-cluster/Dockerfile

@@ -4,12 +4,12 @@ FROM node:10.17.0-buster-slim
 
 # Install fuse.
 # Copy fuse shared library and register it.
-COPY ./libfuse3.so.3 /usr/local/lib/
+COPY ./bin/libfuse3.so.3 /usr/local/lib/
 RUN ldconfig
-COPY ./fusermount3 /usr/local/bin/
+COPY ./bin/fusermount3 /usr/local/bin/
 
 # hpcore binary is copied to /hp directory withtin the docker image.
 WORKDIR /hp
-COPY ./build/hpcore .
-COPY ./build/hpstatemon .
+COPY ./bin/hpcore .
+COPY ./bin/hpstatemon .
 ENTRYPOINT ["/hp/hpcore"]

test/local-cluster/cluster-create.sh

@@ -28,7 +28,7 @@ do
     let pubport=8080+$n
 
     # Create contract dir named "node<i>"
-    ../build/hpcore new "node${n}" > /dev/null 2>&1
+    ../bin/hpcore new "node${n}" > /dev/null 2>&1
 
     pushd ./node$n/cfg > /dev/null 2>&1
 
@@ -65,8 +65,8 @@ do
 
     # Copy the contract executable and appbill.
     mkdir ./node$n/bin
-    cp ../examples/echocontract/contract.js ./node$n/bin/contract.js
-    cp ../build/appbill ./node$n/bin/
+    cp ../../../examples/echo_contract/contract.js ./node$n/bin/contract.js
+    cp ../bin/appbill ./node$n/bin/
 done
 
 # Function to generate JSON array string while skiping a given index.
@@ -114,12 +114,12 @@ done
 for (( i=1; i<=$ncount; i++ ))
 do
 
-    sudo mkdir -p ./node$i/statehist/0/data/ > /dev/null 2>&1
+    mkdir -p ./node$i/statehist/0/data/ > /dev/null 2>&1
 
     # Load credit balance for user for testing purposes.
     pushd ./node$i/statehist/0/data/ > /dev/null 2>&1
     >appbill.table
-    ../../../../../build/appbill --credit "705bf26354ee4c63c0e5d5d883c07cefc3196d049bd3825f827eb3bc23ead035" 10000
+    ../../../../../bin/appbill --credit "705bf26354ee4c63c0e5d5d883c07cefc3196d049bd3825f827eb3bc23ead035" 10000
     popd > /dev/null 2>&1
 
     # Copy any more initial state files for testing.

test/vm-cluster/cluster.sh

@@ -1,5 +1,9 @@
 #!/bin/bash
+
+# VM login password must exist in vmpass.txt
 vmpass=$(cat vmpass.txt)
+# List vm IP addresses of the cluster must exist in iplist.txt
+# (This list will be treated as the node numbers 1,2.3... from topmost IP to the bottom)
 readarray -t vmips < iplist.txt
 
 vmcount=${#vmips[@]}
@@ -9,12 +13,22 @@ hpcore=$(realpath ../..)
 
 if [ "$mode" = "new" ] || [ "$mode" = "update" ] || [ "$mode" = "run" ] || [ "$mode" = "check" ] || \
    [ "$mode" = "connect" ] || [ "$mode" = "kill" ] || [ "$mode" = "reboot" ] || [ "$mode" = "ssh" ]; then
-    echo ""
+    echo "mode: $mode"
 else
     echo "Invalid command. [ new | update | run <N> | check <N> | connect <N> | kill <N> | reboot <N> | ssh <N> <custom command> ] expected."
     exit 1
 fi
 
+# Command modes:
+# new - Install hot pocket dependencies and hot pocket to each vm.
+# update - Deploy updated hot pocket binaries into each vm.
+# run - Run hot pocket of specified vm node.
+# check - Check hot pocket running status of specified vm node.
+# connect - Connect to hot pocket console output (if running) of specified vm node.
+# kill - Kill hot pocket (if running) of specified vm node.
+# reboot - Reboot specified vm node.
+# ssh - Open up an ssh terminal for the specified vm node.
+
 if [ $mode = "run" ]; then
     let nodeid=$2-1
     vmip=${vmips[$nodeid]}
@@ -74,6 +88,8 @@ if [ $mode = "update" ]; then
     exit 0
 fi
 
+# Following code will only be executed in 'new' mode.
+
 for (( i=0; i<$vmcount; i++ ))
 do
     vmip=${vmips[i]}

test/vm-cluster/setup-vm.sh

@@ -12,10 +12,10 @@ if [ $mode = "new" ]; then
 
     sshpass -p $vmpass scp $hpcore/build/hpcore \
                             $hpcore/build/hpstatemon \
-                            $hpcore/examples/echocontract/contract.js \
-                            $hpcore/examples/rndcontract/rnd_contract \
-                            /usr/local/lib/x86_64-linux-gnu/libfuse3.so.3 \
-                            /usr/local/bin/fusermount3 \
+                            $hpcore/examples/echo_contract/contract.js \
+                            $hpcore/examples/random_contract/rnd_contract \
+                            $hpcore/fusebin/libfuse3.so.3 \
+                            $hpcore/fusebin/fusermount3 \
                             ./consensus-test-continuous.sh \
                             ./setup-hp.sh \
                             geveo@$vmip:~/
@@ -25,8 +25,8 @@ if [ $mode = "new" ]; then
 else
     sshpass -p $vmpass scp $hpcore/build/hpcore \
                             $hpcore/build/hpstatemon \
-                            $hpcore/examples/echocontract/contract.js \
-                            $hpcore/examples/rndcontract/rnd_contract \
+                            $hpcore/examples/echo_contract/contract.js \
+                            $hpcore/examples/random_contract/rnd_contract \
                             ./consensus-test-continuous.sh \
                             geveo@$vmip:~/
 fi