diff --git a/test/evernode-cluster/contract-instance-manager.js b/test/evernode-cluster/contract-instance-manager.js index e13c9f5..c2d74c2 100644 --- a/test/evernode-cluster/contract-instance-manager.js +++ b/test/evernode-cluster/contract-instance-manager.js @@ -43,6 +43,12 @@ class ContractInstanceManager { } } + async checkAliveness() { + const hpc = await this.#getHotPocketConnection(); + hpc.clear(HotPocket.events.contractOutput); + await hpc.close(); + } + async #getHotPocketConnection() { const server = `wss://${this.#ip}:${this.#userPort}` const hpc = await HotPocket.createClient([server], this.#ownerKeys, { @@ -129,20 +135,10 @@ class ContractInstanceManager { pendingNodes: [], nodes: contract.cluster.map(n => { return { - refId: n.acquire_ref_id, - contractId: n.acquire_ref_id, - createdOnLcl: 0, - host: n.host, - ip: n.ip, - name: n.name, - peerPort: parseInt(n.peer_port), pubkey: n.pubkey, - userPort: parseInt(n.user_port), - isUnl: true, - isQuorum: true, - lifeMoments: n.extended ? contract.target_moments_count : 1, - targetLifeMoments: n.extended ? contract.target_moments_count : 1, - createdMoment: n.created_moment + ip: n.ip, + peerPort: parseInt(n.peer_port), + userPort: parseInt(n.user_port) } }) }, null, 2)) diff --git a/test/evernode-cluster/index.js b/test/evernode-cluster/index.js index f8680ae..585d47a 100644 --- a/test/evernode-cluster/index.js +++ b/test/evernode-cluster/index.js @@ -6,9 +6,10 @@ const HotPocket = require('hotpocket-js-client'); const CONFIG_FILE = "config.json"; const FUNDING_EVR_PER_ROUND = 6000; -const MAX_MEMO_PEER_LIMIT = 10; +const HOST_LOG_FILE = "log.json"; +const MAX_MEMO_PEER_LIMIT = 8; const FAIL_THRESHOLD = 1; -const DEF_TIMEOUT = 60000; +const DEF_TIMEOUT = 300000; const CLUSTER_CHUNK_RATIO = 0.2; async function sleep(ms) { @@ -21,6 +22,7 @@ async function sleep(ms) { class ClusterManager { #config = {}; + #hostsLogs = {}; #evernodeService = null; #contractIdx; #instanceCount; @@ -78,6 +80,10 @@ class ClusterManager { await fs.writeFile(CONFIG_FILE, JSON.stringify(this.#config, null, 2)).catch(console.error); } + async #writeLogs() { + await fs.writeFile(HOST_LOG_FILE, JSON.stringify(this.#hostsLogs, null, 2)).catch(console.error); + } + async init() { await this.#readConfig(); this.#evernodeService = new EvernodeService(this.#config.accounts); @@ -89,6 +95,12 @@ class ClusterManager { !this.#config.accounts.blacklist_hosts.includes(h.address) && (!this.#config.accounts.preferred_hosts || !this.#config.accounts.preferred_hosts.length || this.#config.accounts.preferred_hosts.includes(h.address))) .sort(() => Math.random() - 0.5); + for (const host of this.#hosts) { + this.#hostsLogs[host.address] = { + count: 0, + errors: [] + } + } } async #getExtendingFundAmount() { @@ -136,17 +148,34 @@ class ClusterManager { ...result.instance }); this.#hosts[hostIndex].activeInstances++; + this.#hostsLogs[host.address].count++; this.#hosts[hostIndex].acquiring = false; + let attempt = 0; + while (true) { + attempt++; + try { + await this.checkAliveness(result.instance); + break; + } + catch (e) { + if (attempt > 3) + throw e; + } + await new Promise(resolve => setTimeout(resolve, 1000)); + } + + await this.#writeLogs(); return result.instance; } catch (e) { this.#hosts[hostIndex].acquiring = false; - + this.#hostsLogs[host.address].errors.push(e.reason || e); // If the reason is lack of EVRs, fund again if (e.content?.code == 'tecINSUFFICIENT_FUNDS') { await this.#evernodeService.fundTenant(FUNDING_EVR_PER_ROUND); } + await this.#writeLogs(); throw { message: `Error while creating the node ${nodeNumber} in ${host.address}.`, innerException: e }; } } @@ -282,6 +311,13 @@ class ClusterManager { } } + async checkAliveness(instance) { + let contract = this.#config.contracts[this.#contractIdx]; + const ownerKeys = await HotPocket.generateKeys(contract.owner_privatekey); + const instanceMgr = new ContractInstanceManager(ownerKeys, instance.pubkey, instance.ip, instance.user_port, instance.contractId, contract.bundle_path); + await instanceMgr.checkAliveness(); + } + async deploy() { let contract = this.#config.contracts[this.#contractIdx]; const ownerKeys = await HotPocket.generateKeys(contract.owner_privatekey); diff --git a/test/evernode-cluster/summary.js b/test/evernode-cluster/summary.js new file mode 100644 index 0000000..9c0abb4 --- /dev/null +++ b/test/evernode-cluster/summary.js @@ -0,0 +1,84 @@ +const fs = require('fs').promises; + +async function process() { + const buf = await fs.readFile('log.json').catch(console.error); + const logConf = JSON.parse(buf); + let output = []; + let blacklist = []; + let nooffer = []; + let created = []; + let preferred = []; + for (const [key, value] of Object.entries(logConf)) { + if (value.errors.length || value.count === 0) { + output.push({ + key: key, + errors: value.errors.length ? [... new Set(value.errors)] : ['Not Picked'] + }); + if (value.errors.find(e => e != 'NO_OFFER')) + blacklist.push(key); + else + nooffer.push(key) + } + else { + created.push(key); + } + } + output.sort((a, b) => a.errors[0].localeCompare(b.errors[0])); + + const logbuf = await fs.readFile('hp.log').catch(console.error); + const lines = logbuf.toString().split('\n'); + const configbuf = await fs.readFile('config.json').catch(console.error); + const config = JSON.parse(configbuf) + let times = []; + for (const line of lines.filter(l => l.includes('frm:') && l.includes('ms') && l.includes('<'))) { + let data = line.split('frm:')[1]; + data = data.slice(0, data.length - 3); + data = data.split('<'); + let host = {}; + if (data[0] == 'self') + host = config.contracts[0].cluster[0]; + else + host = config.contracts[0].cluster.find(i => i.pubkey.slice(2, 10) == data[0]); + const index = times.findIndex(h => h.key == host.host); + if (index < 0) { + times.push({ + key: host.host, + time: parseInt(data[1]) + }) + } + else if (times[index].time > parseInt(data[1])) { + times.splice(index, 1); + times.push({ + key: host.host, + time: parseInt(data[1]) + }) + } + } + for (const p of created) { + const found = times.find(h => h.key == p); + if (!found) { + times.push({ + key: p, + time: 100000000000000 + }) + } + else if (found.time < 150) { + preferred.push(p); + } + } + times.sort((a, b) => (a.time > b.time) ? 1 : ((b.time > a.time) ? -1 : 0)); + + output.push({ + blacklist: blacklist, + nooffer: nooffer, + created: created, + preferred: preferred, + times: times, + totalhostcount: Object.keys(logConf).length, + checkedhostcount: nooffer.length + created.length + blacklist.length + }) + await fs.writeFile('summary.json', JSON.stringify(output, null, 2)).catch(console.error); + +} + +process().catch(console.error); \ No newline at end of file