From f11ebc1253cfed8bc2a80647ceead1a17d73f1d8 Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Fri, 24 Apr 2026 19:42:20 +0100 Subject: [PATCH] fix: StatsDGauge dirty init + tx_submitter sequence drift in CI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two CI failures traced to root cause: 1. rippled_jobq_job_count: 0 series — StatsDGaugeImpl declared m_dirty{false} despite the constructor comment saying "start dirty". Gauges whose value starts and stays at 0 never emitted, so Prometheus never scraped them. Fix: m_dirty{true} on the member initializer. 2. TX error rate 82.8% — the submitter tracked account sequences locally, but in a multi-node consensus network other nodes' txns advance sequences independently. After a few ledger closes the locally-tracked sequence fell behind the ledger, producing tefPAST_SEQ for every subsequent submission. Fix: refresh account sequences from account_info every 10 s during the submission loop. --- docker/telemetry/workload/tx_submitter.py | 27 +++++++++++++++++++ src/libxrpl/beast/insight/StatsDCollector.cpp | 2 +- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/docker/telemetry/workload/tx_submitter.py b/docker/telemetry/workload/tx_submitter.py index 66a9be5510..4ff015f40f 100644 --- a/docker/telemetry/workload/tx_submitter.py +++ b/docker/telemetry/workload/tx_submitter.py @@ -642,6 +642,25 @@ async def submit_transaction( logger.debug("%s error: %s", tx_type, exc) +async def _refresh_sequences( + ws: websockets.WebSocketClientProtocol, + accounts: list[Account], +) -> None: + """Re-sync account sequences from the validated ledger. + + In a consensus network, other nodes' transactions advance sequences + beyond the submitter's local tracking. Refreshing every ~10 s keeps + the local counter close to the ledger and prevents tefPAST_SEQ storms. + """ + for acct in accounts: + try: + seq = await get_account_sequence(ws, acct.account) + if seq > acct.sequence: + acct.sequence = seq + except Exception: + pass + + async def run_submitter( endpoint: str, tps: float, @@ -684,7 +703,15 @@ async def run_submitter( ) start = time.monotonic() + last_seq_refresh = start + seq_refresh_interval = 10.0 while (time.monotonic() - start) < duration: + # Periodically re-sync account sequences from the ledger so + # locally-tracked sequences don't drift behind consensus. + if (time.monotonic() - last_seq_refresh) >= seq_refresh_interval: + await _refresh_sequences(ws, accounts) + last_seq_refresh = time.monotonic() + tx_type = random.choices(tx_types, weights=tx_weights, k=1)[0] await submit_transaction(ws, tx_type, accounts, stats) await asyncio.sleep(interval) diff --git a/src/libxrpl/beast/insight/StatsDCollector.cpp b/src/libxrpl/beast/insight/StatsDCollector.cpp index 1daaa33100..1efc42d6f4 100644 --- a/src/libxrpl/beast/insight/StatsDCollector.cpp +++ b/src/libxrpl/beast/insight/StatsDCollector.cpp @@ -164,7 +164,7 @@ private: std::string m_name; GaugeImpl::value_type m_last_value{0}; GaugeImpl::value_type m_value{0}; - bool m_dirty{false}; + bool m_dirty{true}; }; //------------------------------------------------------------------------------