Large cluster optimizations. (#348)

* Added sync log to streamer.
* Fixed ledger closing attempt while syncing.
* Added diagnostic contract.
* Reset to stage 0 on unreliable votes.
* Reduced peer msg age threshold.
* Added health tracking.
* Weakly-connected detection improvement.
* Increased version 0.5.1.
* Improved client lib server version check.
* Added health logging support to text client.
* Added weakly connected status in status response.
* Increased max peers limits when serializing.
* Local docker cluster manual ip.
* Updated vultr script vm region order.
* Sync status reporting improvement.
* Added milliseconds to logging.

examples/js_client/lib/hp-client-lib.js

@@ -22,7 +22,7 @@
         TextDecoder = util.TextDecoder;
     }
 
-    const supportedHpVersion = "0.5.0";
+    const supportedHpVersion = "0.5.";
     const serverChallengeSize = 16;
     const outputValidationPassThreshold = 0.8;
     const connectionCheckIntervalMs = 1000;
@@ -52,14 +52,16 @@
         contractReadResponse: "contract_read_response",
         connectionChange: "connection_change",
         unlChange: "unl_change",
-        ledgerEvent: "ledger_event"
+        ledgerEvent: "ledger_event",
+        healthEvent: "health_event"
     }
     Object.freeze(events);
 
     /*--- Included in public interface. ---*/
     const notificationChannels = {
         unlChange: "unl_change",
-        ledgerEvent: "ledger_event"
+        ledgerEvent: "ledger_event",
+        healthEvent: "health_event"
     }
     Object.freeze(notificationChannels);
 
@@ -181,6 +183,7 @@
         // Subscribe for unl changes if we have to maintain the trusted server key checks.
         subscriptions[notificationChannels.unlChange] = trustedKeysLookup ? true : false;
         subscriptions[notificationChannels.ledgerEvent] = false;
+        subscriptions[notificationChannels.healthEvent] = false;
 
         let status = 0; //0:none, 1:connected, 2:closed
 
@@ -548,8 +551,8 @@
 
             if (connectionStatus == 0 && m.type == "user_challenge" && m.hp_version && m.contract_id) {
 
-                if (m.hp_version != supportedHpVersion) {
-                    liblog(1, `Incompatible Hot Pocket server version. Expected:${supportedHpVersion} Got:${m.hp_version}`);
+                if (!m.hp_version.startsWith(supportedHpVersion)) {
+                    liblog(1, `Incompatible Hot Pocket server version. Expected:${supportedHpVersion}* Got:${m.hp_version}`);
                     return false;
                 }
                 else if (!m.contract_id) {
@@ -665,6 +668,7 @@
                         contractExecutionEnabled: m.contract_execution_enabled,
                         readRequestsEnabled: m.read_requests_enabled,
                         isFullHistoryNode: m.is_full_history_node,
+                        weaklyConnected: m.weakly_connected,
                         currentUnl: m.current_unl.map(u => msgHelper.deserializeValue(u)),
                         peers: m.peers
                     });
@@ -691,6 +695,10 @@
                     ev.inSync = m.in_sync;
                 emitter.emit(events.ledgerEvent, ev);
             }
+            else if (m.type == "health_event") {
+                const ev = msgHelper.deserializeHealthEvent(m);
+                emitter.emit(events.healthEvent, ev);
+            }
             else if (m.type == "ledger_query_result") {
                 const resolver = ledgerQueryResolvers[m.reply_for];
                 if (resolver) {
@@ -1142,6 +1150,24 @@
                 outputHash: this.deserializeValue(l.output_hash)
             }
         }
+
+        this.deserializeHealthEvent = (m) => {
+            if (m.event === "proposal") {
+                return {
+                    event: m.event,
+                    commLatency: m.comm_latency,
+                    readLatency: m.read_latency,
+                    batchSize: m.batch_size
+                }
+            }
+            else if (m.event === "connectivity") {
+                return {
+                    event: m.event,
+                    peerCount: m.peer_count,
+                    weaklyConnected: m.weakly_connected
+                }
+            }
+        }
     }
 
     function hexToUint8Array(hexString) {

examples/js_client/text-client.js

@@ -54,12 +54,16 @@ async function main() {
 
     // This will get fired when contract sends outputs.
     hpc.on(HotPocket.events.contractOutput, (r) => {
-        r.outputs.forEach(o => console.log(`Output (ledger:${r.ledgerSeqNo})>> ${o}`));
+        r.outputs.forEach(o => {
+            const outputLog = o.length <= 512 ? o : `[Big output (${o.length / 1024} KB)]`;
+            console.log(`Output (ledger:${r.ledgerSeqNo})>> ${outputLog}`);
+        });
     })
 
     // This will get fired when contract sends a read response.
-    hpc.on(HotPocket.events.contractReadResponse, (response) => {
-        console.log("Read response>> " + response);
+    hpc.on(HotPocket.events.contractReadResponse, (o) => {
+        const outputLog = o.length <= 512 ? o : `[Big output (${o.length / 1024} KB)]`;
+        console.log("Read response>> " + outputLog);
     })
 
     // This will get fired when the unl public key list changes.
@@ -73,6 +77,11 @@ async function main() {
         console.log(ev);
     })
 
+    // This will get fired when any health event occurs (proposal stats, connectivity changes...).
+    hpc.on(HotPocket.events.healthEvent, (ev) => {
+        console.log(ev);
+    })
+
     // Establish HotPocket connection.
     if (!await hpc.connect()) {
         console.log('Connection failed.');
@@ -83,6 +92,7 @@ async function main() {
     // After connecting, we can subscribe to events from the HotPocket node.
     // await hpc.subscribe(HotPocket.notificationChannels.unlChange);
     // await hpc.subscribe(HotPocket.notificationChannels.ledgerEvent);
+    // await hpc.subscribe(HotPocket.notificationChannels.healthEvent);
 
     // start listening for stdin
     const rl = readline.createInterface({
@@ -110,7 +120,25 @@ async function main() {
                     hpc.getLedgerBySeqNo(parseInt(inp.substr(7)), true, true)
                         .then(result => console.log(result));
                 }
+                else if (inp.startsWith("health ")) {
+                    if (inp.endsWith("on"))
+                        hpc.subscribe(HotPocket.notificationChannels.healthEvent);
+                    else if (inp.endsWith("off"))
+                        hpc.unsubscribe(HotPocket.notificationChannels.healthEvent);
+                }
+                else if (inp === "stat") {
+                    hpc.getStatus().then(stat => console.log(stat));
+                }
                 else {
+
+                    if (inp.startsWith("upload ")) {
+                        const size = parseInt(inp.split(" ")[1]);
+                        if (!isNaN(size)) {
+                            inp = "A".repeat(size * 1024 * 1024);
+                            console.log("Uploading " + size + " MB payload...");
+                        }
+                    }
+
                     hpc.submitContractInput(inp).then(input => {
                         // console.log(input.hash);
                         input.submissionStatus.then(s => {

examples/nodejs_contract/diagnostic_contract.js

@@ -0,0 +1,138 @@
+const HotPocket = require("./hp-contract-lib");
+const fs = require('fs').promises;
+var seedrandom = require('seedrandom');
+
+const filename = "file.dat";
+const autofilePrefix = "autofile";
+const autofileSize = 1 * 1024 * 1024;
+
+const diagnosticContract = async (ctx) => {
+
+    // Collection of per-user promises to wait for. Each promise completes when inputs for that user is processed.
+    const userHandlers = [];
+
+    for (const user of ctx.users.list()) {
+
+        // For each user we add a promise to list of promises.
+        userHandlers.push(new Promise(async (resolve) => {
+
+            // The contract need to ensure that all outputs for a particular user is emitted
+            // in deterministic order. Hence, we are processing all inputs for each user sequentially.
+            for (const input of user.inputs) {
+
+                const buf = await ctx.users.read(input);
+                const parts = buf.toString().split(" ");
+                const mode = parts[0];
+                const data = parts[1];
+                let output = null;
+
+                if (mode === "status") {
+                    output = "Hot Pocket diagnostic contract is running.";
+                }
+                else if (mode === "file") {
+                    const param = parseInt(data);
+                    const stat = await fs.stat(filename).catch(() => { });
+
+                    if (isNaN(param)) {
+                        if (!stat)
+                            output = "File does not exist.";
+                        else
+                            output = "Current size: " + stat.size / (1024 * 1024) + " MB";
+                    }
+                    else {
+                        if (param == 0) {
+                            if (stat) {
+                                await fs.unlink(filename);
+                                output = "Deleted file.";
+                            }
+                        }
+                        else {
+                            if (!stat)
+                                await fs.writeFile(filename, "Initial");
+
+                            await fs.truncate(filename, param * 1024 * 1024);
+                            output = "Updated file size to " + param + " MB";
+                        }
+                    }
+                }
+                else if (mode === "files") {
+                    const param = parseInt(data);
+                    const autofiles = await (await fs.readdir(".")).filter(f => f.startsWith(autofilePrefix));
+
+                    if (isNaN(param)) {
+                        output = autofiles.length + " autofiles found.";
+                    }
+                    else {
+                        if (param == 0) {
+                            for (file of autofiles) {
+                                await fs.unlink(file);
+                            }
+                            output = autofiles.length + " autofiles deleted.";
+                        }
+                        else {
+                            const content = "A".repeat(autofileSize);
+                            for (let i = (autofiles.length + 1); i <= (autofiles.length + param); i++) {
+                                await fs.writeFile(autofilePrefix + i, content);
+                            }
+                            output = param + " new autofiles created. Total: " + (autofiles.length + param);
+                        }
+                    }
+                }
+                else if (mode === "download") {
+                    const param = parseFloat(data);
+                    if (!isNaN(param)) {
+                        output = "A".repeat(param * 1024 * 1024);
+                    }
+                }
+                else if (mode === "roundtime") {
+                    const param = parseInt(data);
+                    if (!isNaN(param)) {
+                        if (param >= 100) {
+                            const config = await ctx.getConfig();
+                            config.roundtime = param;
+                            await ctx.updateConfig(config)
+                            output = "Updated Roundtime to " + config.roundtime;
+                        }
+                    }
+                    else {
+                        const config = await ctx.getConfig();
+                        output = "Roundtime: " + config.roundtime;
+                    }
+                }
+                else {
+                    output = "Received unrecognized input of length " + buf.length;
+                }
+
+                if (output)
+                    await user.send(output);
+            }
+
+            // The promise gets completed when all inputs for this user are processed.
+            resolve();
+        }));
+    }
+
+    // Wait until all user promises are complete.
+    await Promise.all(userHandlers);
+
+    // Modify random file bytes (if file exists)
+    {
+        const stat = await fs.stat(filename).catch(() => { });
+        if (stat) {
+            const rng = seedrandom(ctx.lcl_hash);
+            const fh = await fs.open(filename, 'r+');
+
+            for (let i = 0; i < 3; i++) {
+                const pos = rng() * (stat.size - 50);
+                const buf = ctx.lcl_hash.substr(i * 10, 10);
+                await fh.write(buf, pos);
+            }
+
+            await fh.close();
+        }
+    }
+
+}
+
+const hpc = new HotPocket.Contract();
+hpc.init(diagnosticContract);

examples/nodejs_contract/package-lock.json

@@ -34,6 +34,11 @@
             "version": "4.0.0",
             "resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz",
             "integrity": "sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA=="
+        },
+        "seedrandom": {
+            "version": "3.0.5",
+            "resolved": "https://registry.npmjs.org/seedrandom/-/seedrandom-3.0.5.tgz",
+            "integrity": "sha512-8OwmbklUNzwezjGInmZ+2clQmExQPvomqjL7LFqOYqtmuxRgQYqOD3mHaU+MvZn5FLUeVxVfQjwLZW/n/JFuqg=="
         }
     }
 }

examples/nodejs_contract/package.json

@@ -1,9 +1,11 @@
 {
     "scripts": {
         "build-echo": "ncc build echo_contract.js -o dist/echo-contract",
-        "build-file": "ncc build file_contract.js -o dist/file-contract"
+        "build-file": "ncc build file_contract.js -o dist/file-contract",
+        "build-diag": "ncc build diagnostic_contract.js -o dist/diagnostic-contract"
     },
     "dependencies": {
-        "bson": "4.0.4"
+        "bson": "4.0.4",
+        "seedrandom": "3.0.5"
     }
 }