Heartbeat retrying was enabled in txn failures. (#339)

This commit is contained in:
Kithmini Gunawardhana
2024-02-05 19:13:19 +05:30
committed by GitHub
parent bc8240471f
commit 8cdebff7e2
5 changed files with 69 additions and 31 deletions

View File

@@ -6,7 +6,7 @@
"": {
"name": "evernode-setup-helper",
"dependencies": {
"evernode-js-client": "0.6.28",
"evernode-js-client": "0.6.31",
"ip6addr": "0.2.5",
"ripple-keypairs": "1.3.1"
}
@@ -364,9 +364,9 @@
}
},
"node_modules/evernode-js-client": {
"version": "0.6.28",
"resolved": "https://registry.npmjs.org/evernode-js-client/-/evernode-js-client-0.6.28.tgz",
"integrity": "sha512-0itl0aJlxGdtRVCAVUHQbdRxt8HjCvXU6YW9friJqv3yUa4muBelTh14WVflSua3+KWTF/f/TVsFWCMlUsu9WA==",
"version": "0.6.31",
"resolved": "https://registry.npmjs.org/evernode-js-client/-/evernode-js-client-0.6.31.tgz",
"integrity": "sha512-1jISkGlXCfIxtG7jXmXMSmiQ/2P90kye0olevQQEH7rsFNCvM5TdhvlAGiTzFRDYj5gWN+nbPSq7eVlkdgIabw==",
"dependencies": {
"elliptic": "6.5.4",
"libsodium-wrappers": "0.7.10",
@@ -1543,9 +1543,9 @@
}
},
"evernode-js-client": {
"version": "0.6.28",
"resolved": "https://registry.npmjs.org/evernode-js-client/-/evernode-js-client-0.6.28.tgz",
"integrity": "sha512-0itl0aJlxGdtRVCAVUHQbdRxt8HjCvXU6YW9friJqv3yUa4muBelTh14WVflSua3+KWTF/f/TVsFWCMlUsu9WA==",
"version": "0.6.31",
"resolved": "https://registry.npmjs.org/evernode-js-client/-/evernode-js-client-0.6.31.tgz",
"integrity": "sha512-1jISkGlXCfIxtG7jXmXMSmiQ/2P90kye0olevQQEH7rsFNCvM5TdhvlAGiTzFRDYj5gWN+nbPSq7eVlkdgIabw==",
"requires": {
"elliptic": "6.5.4",
"libsodium-wrappers": "0.7.10",

View File

@@ -4,7 +4,7 @@
"build": "ncc build index.js --minify -o dist"
},
"dependencies": {
"evernode-js-client": "0.6.28",
"evernode-js-client": "0.6.31",
"ip6addr": "0.2.5",
"ripple-keypairs": "1.3.1"
}

View File

@@ -27,6 +27,9 @@ class MessageBoard {
processing: false,
queue: []
};
#applyFeeUpliftment = false;
#hearbeatRetryDelay = 30000; // 5 mins
#feeUpliftment = 0;
constructor(configPath, secretConfigPath, dbPath, sashiCliPath, sashiDbPath, sashiConfigPath) {
this.configPath = configPath;
@@ -175,6 +178,15 @@ class MessageBoard {
}
#prepareHostClientFunctionOptions() {
let options = {}
if (this.#applyFeeUpliftment) {
options.transactionOptions = { feeUplift: this.#feeUpliftment }
}
return options;
}
// Try to acquire the lease update lock.
async #acquireConcurrencyQueue() {
await new Promise(async resolve => {
@@ -193,12 +205,14 @@ class MessageBoard {
this.#concurrencyQueue.processing = false;
}
async #queueAction(action) {
async #queueAction(action, maxAttempts = 5, delay = 0) {
await this.#acquireConcurrencyQueue();
this.#concurrencyQueue.queue.push({
callback: action,
attempts: 0
attempts: 0,
maxAttempts: maxAttempts,
delay: delay
});
await this.#releaseConcurrencyQueue();
@@ -211,11 +225,22 @@ class MessageBoard {
for (let action of this.#concurrencyQueue.queue) {
try {
await action.callback();
this.#applyFeeUpliftment = false;
this.#feeUpliftment = 0;
}
catch (e) {
if (action.attempts < 5) {
if (action.attempts < action.maxAttempts) {
action.attempts++;
toKeep.push(action);
if (this.cfg.xrpl.affordableExtraFee > 0 && (typeof e === 'string' && e.includes('tefMAX_LEDGER'))) {
this.#applyFeeUpliftment = true;
this.#feeUpliftment = Math.floor((this.cfg.xrpl.affordableExtraFee * action.attempts) / action.maxAttempts);
}
if (action.delay > 0) {
setTimeout(async () => {
toKeep.push(action);
}, action.delay);
} else
toKeep.push(action);
}
else {
console.error(e);
@@ -291,6 +316,14 @@ class MessageBoard {
// Heartbeat sender
async #sendHeartbeat() {
const momentSize = this.hostClient.config.momentSize;
const currentTimestamp = evernode.UtilHelpers.getCurrentUnixTime();
const currentMomentStartIdx = await this.hostClient.getMomentStartIndex();
const currentMomentDuration = currentTimestamp - currentMomentStartIdx;
// Drop heartbeat re-trying in last quarter.
const maxRetries = (currentMomentDuration >= Math.floor(momentSize * 0.75) && currentMomentDuration < momentSize) ?
0 :
Math.floor((momentSize - currentMomentDuration - 300) / 300);
await this.#queueAction(async () => {
let ongoingHeartbeat = false;
const currentMoment = await this.hostClient.getMoment();
@@ -323,7 +356,7 @@ class MessageBoard {
if (voteArr && voteArr.length) {
for (const vote of voteArr) {
try {
await this.hostClient.heartbeat(vote);
await this.hostClient.heartbeat(vote, this.#prepareHostClientFunctionOptions());
this.lastHeartbeatMoment = await this.hostClient.getMoment();
heartbeatSent = true;
}
@@ -335,6 +368,7 @@ class MessageBoard {
}
else {
console.error("Heartbeat tx with vote error", e);
throw e;
}
}
}
@@ -346,20 +380,22 @@ class MessageBoard {
return;
try {
await this.hostClient.heartbeat();
await this.hostClient.heartbeat({}, this.#prepareHostClientFunctionOptions());
this.lastHeartbeatMoment = await this.hostClient.getMoment();
}
catch (err) {
if (err.code === 'tecHOOK_REJECTED')
console.log("Heartbeat rejected by the hook.");
else
else {
console.log("Heartbeat tx error", err);
throw err;
}
}
finally {
ongoingHeartbeat = false;
}
}
});
}, maxRetries, this.#hearbeatRetryDelay);
}
async #expireInstance(lease, currentTime = evernode.UtilHelpers.getCurrentUnixTime()) {
@@ -473,7 +509,10 @@ class MessageBoard {
async #startHeartBeatScheduler() {
const momentSize = this.hostClient.config.momentSize;
const halfMomentSize = momentSize / 2; // Getting half of moment size
const halfMomentSize = momentSize / 2; // Getting 50% of moment size
const acceptanceLimit = Math.floor(momentSize * 0.75); // Getting 75% of moment size
const momentReserve = Math.floor(momentSize * 0.25); // Getting 25% of moment size
const timeout = momentSize * 1000; // Converting seconds to milliseconds.
const scheduler = async () => {
@@ -491,17 +530,16 @@ class MessageBoard {
// Schedule the next heartbeat based on last heartbeat occurrence.
// NOTE : Initially checks whether host has sent a heartbeat in the current moment or not.
// If schedule the next heartbeat based on its last heartbeat.
// If it is not further checks whether it is about to send the heartbeat at the second half of a moment or not.
// If the current timestamp lies in the second half of the moment, schedule the next heartbeat withing the next moment (in the its first half).
// If it is not schedule it right now.
// If it's true, then schedule the next heartbeat based on its last heartbeat.
// If not, further check whether it is about to send a heartbeat at the which state of a moment.
// If the current timestamp lies in the last quarter of the moment, then schedule the next heartbeat withing the next moment in a random slot.
// If not, schedule it right now.
const schedule = (this.lastHeartbeatMoment === currentMoment)
? momentSize - (currentTimestamp - hostInfo.lastHeartbeatIndex)
: (currentMomentDuration > halfMomentSize && currentMomentDuration < momentSize) ? halfMomentSize : 0;
: (currentMomentDuration > acceptanceLimit && currentMomentDuration < momentSize) ? Math.floor(Math.random() * (acceptanceLimit - momentReserve)) + momentReserve : 0;
// If the start index is in the beginning of the moment, delay the heartbeat scheduler 1 minute to make sure the hook timestamp is not in previous moment when accepting the heartbeat.
const startTimeout = (currentMomentDuration) < halfMomentSize ? ((schedule + 60) * 1000) : ((schedule) * 1000);
setTimeout(async () => {
await scheduler();
}, startTimeout);

View File

@@ -6,7 +6,7 @@
"": {
"name": "mb-xrpl",
"dependencies": {
"evernode-js-client": "0.6.30",
"evernode-js-client": "0.6.31",
"ip6addr": "0.2.5",
"sqlite3": "5.0.2"
},
@@ -980,9 +980,9 @@
}
},
"node_modules/evernode-js-client": {
"version": "0.6.30",
"resolved": "https://registry.npmjs.org/evernode-js-client/-/evernode-js-client-0.6.30.tgz",
"integrity": "sha512-3FyDCiDXN0Ez0+ABGu+NLI3kcqP9aoPvq+RGU0m6rxiwbttQetGfXZouUeR/4xls8/6lZLVmKItbcw72MQpQWg==",
"version": "0.6.31",
"resolved": "https://registry.npmjs.org/evernode-js-client/-/evernode-js-client-0.6.31.tgz",
"integrity": "sha512-1jISkGlXCfIxtG7jXmXMSmiQ/2P90kye0olevQQEH7rsFNCvM5TdhvlAGiTzFRDYj5gWN+nbPSq7eVlkdgIabw==",
"dependencies": {
"elliptic": "6.5.4",
"libsodium-wrappers": "0.7.10",
@@ -4010,9 +4010,9 @@
"dev": true
},
"evernode-js-client": {
"version": "0.6.30",
"resolved": "https://registry.npmjs.org/evernode-js-client/-/evernode-js-client-0.6.30.tgz",
"integrity": "sha512-3FyDCiDXN0Ez0+ABGu+NLI3kcqP9aoPvq+RGU0m6rxiwbttQetGfXZouUeR/4xls8/6lZLVmKItbcw72MQpQWg==",
"version": "0.6.31",
"resolved": "https://registry.npmjs.org/evernode-js-client/-/evernode-js-client-0.6.31.tgz",
"integrity": "sha512-1jISkGlXCfIxtG7jXmXMSmiQ/2P90kye0olevQQEH7rsFNCvM5TdhvlAGiTzFRDYj5gWN+nbPSq7eVlkdgIabw==",
"requires": {
"elliptic": "6.5.4",
"libsodium-wrappers": "0.7.10",

View File

@@ -5,7 +5,7 @@
"build": "npm run lint && ncc build app.js --minify -o dist"
},
"dependencies": {
"evernode-js-client": "0.6.30",
"evernode-js-client": "0.6.31",
"sqlite3": "5.0.2",
"ip6addr": "0.2.5"
},