diff --git a/docker/telemetry/grafana/dashboards/consensus-health.json b/docker/telemetry/grafana/dashboards/consensus-health.json index a46aa60dfd..eda035e7ab 100644 --- a/docker/telemetry/grafana/dashboards/consensus-health.json +++ b/docker/telemetry/grafana/dashboards/consensus-health.json @@ -10,7 +10,7 @@ "panels": [ { "title": "Consensus Round Duration", - "description": "p95 and p50 duration of consensus accept rounds. The consensus.accept span (RCLConsensus.cpp) measures the time to process an accepted ledger including transaction application and state finalization. The span carries proposers and round_time_ms attributes. Normal range is 3-6 seconds on mainnet.", + "description": "**What:** Time to process an accepted ledger, from the moment consensus is reached through applying the transaction set and finalising state.\n**How it's computed:** 95th and 50th percentile of per-round durations over a 5-minute window, per node.\n**Reading it:** Flat p50 with a modest p95 gap is normal; both track network transaction load.\n**Healthy range:** Roughly 3-6 seconds on mainnet.\n**Watch for:** A rising p95 that pulls away from p50 means occasional slow rounds; sustained growth precedes ledger-age alarms.\n**Source:** src/xrpld/app/consensus/RCLConsensus.cpp:RCLConsensus::Adaptor::makeAcceptSpan", "type": "timeseries", "gridPos": { "h": 8, @@ -56,7 +56,7 @@ }, { "title": "Consensus Proposals Sent Rate", - "description": "Rate at which this node sends consensus proposals to the network. Sourced from the consensus.proposal.send span (RCLConsensus.cpp) which fires each time the node proposes a transaction set. The span carries xrpl.consensus.round identifying the consensus round number. A healthy proposing node should show steady proposal output.", + "description": "**What:** How often this node broadcasts its own consensus proposal (its candidate transaction set) to the network.\n**How it's computed:** Per-second rate over a 5-minute window, per node.\n**Reading it:** A proposing validator emits a steady stream, roughly one per consensus round; observing-only nodes emit none.\n**Healthy range:** About one proposal per round (~0.2-0.3/s on mainnet) for a proposing node.\n**Watch for:** A validator that drops to zero has stopped proposing and may be unhealthy or misconfigured.\n**Source:** src/xrpld/app/consensus/RCLConsensus.cpp:RCLConsensus::Adaptor::propose", "type": "timeseries", "gridPos": { "h": 8, @@ -95,7 +95,7 @@ }, { "title": "Ledger Close Duration", - "description": "p95 duration of the ledger close event. The consensus.ledger_close span (RCLConsensus.cpp) measures the time from when consensus triggers a ledger close to completion. Carries ledger_seq and consensus_mode attributes. Compare with Consensus Round Duration to understand how close timing relates to overall round time.", + "description": "**What:** Time spent closing a ledger once consensus triggers the close, up to completion.\n**How it's computed:** 95th percentile of per-close durations over a 5-minute window, per node.\n**Reading it:** Should sit at or just below the full round duration; a subset of overall round time.\n**Healthy range:** A few seconds, tracking the ~4s mainnet ledger interval.\n**Watch for:** A widening gap versus round duration points to overhead outside ledger construction.\n**Source:** src/xrpld/app/consensus/RCLConsensus.cpp:RCLConsensus::Adaptor::onClose", "type": "timeseries", "gridPos": { "h": 8, @@ -134,7 +134,7 @@ }, { "title": "Validation Send Rate", - "description": "Rate at which this node sends ledger validations to the network. Sourced from the consensus.validation.send span (RCLConsensus.cpp). Each validation confirms the node has fully validated a ledger. The span carries ledger_seq and proposing. Should closely track the ledger close rate when the node is healthy.", + "description": "**What:** How often this node issues a validation confirming it has fully validated a ledger.\n**How it's computed:** Per-second rate over a 5-minute window, per node.\n**Reading it:** Should closely track the ledger close rate on a healthy validator.\n**Healthy range:** About one per ledger (~0.25/s on mainnet).\n**Watch for:** Validations lagging closes suggests the node is falling behind on validation.\n**Source:** src/xrpld/app/consensus/RCLConsensus.cpp:RCLConsensus::Adaptor::createValidationSpan", "type": "stat", "gridPos": { "h": 8, @@ -166,7 +166,7 @@ }, { "title": "Ledger Apply Duration (doAccept)", - "description": "Time spent applying the consensus result to build a new ledger. Measured by the consensus.accept.apply span in doAccept().", + "description": "**What:** Time spent applying the agreed consensus result to build the new ledger.\n**How it's computed:** 95th and 50th percentile of per-round apply durations over a 5-minute window, per node.\n**Reading it:** Scales with the size of the transaction set being applied.\n**Healthy range:** Tens to a few hundred milliseconds under normal load.\n**Watch for:** Spikes indicate heavy transaction sets or I/O pressure while writing ledger state.\n**Source:** src/xrpld/app/consensus/RCLConsensus.cpp:RCLConsensus::Adaptor::doAccept", "type": "timeseries", "gridPos": { "h": 8, @@ -206,7 +206,7 @@ }, { "title": "Close Time Agreement", - "description": "Rate of close time agreement vs disagreement across consensus rounds. Based on close_time_correct attribute (true = validators agreed, false = agreed to disagree per avCT_CONSENSUS_PCT).", + "description": "**What:** Whether validators agreed on the ledger close time (agreed) or agreed to disagree and used a fallback (disagreed), per round.\n**How it's computed:** Per-second rate of rounds split by agreement outcome over a 5-minute window, per node.\n**Reading it:** Nearly all rounds should show agreement.\n**Healthy range:** Overwhelmingly agreement; occasional disagreement is tolerable.\n**Watch for:** A rising disagreement share signals clock drift or network latency across validators.\n**Source:** src/xrpld/app/consensus/RCLConsensus.cpp:RCLConsensus::Adaptor::doAccept", "type": "timeseries", "gridPos": { "h": 8, @@ -239,7 +239,7 @@ }, { "title": "Consensus Mode Over Time", - "description": "Breakdown of consensus ledger close events by the node's consensus mode (Proposing, Observing, Wrong Ledger, Switched Ledger). Grouped by the consensus_mode span attribute from consensus.ledger_close. A healthy validator should be predominantly in Proposing mode. Frequent Wrong Ledger or Switched Ledger indicates sync issues.", + "description": "**What:** Ledger close events split by the node's consensus mode: Proposing, Observing, Wrong Ledger, or Switched Ledger.\n**How it's computed:** Per-second rate per mode over a 5-minute window, per node.\n**Reading it:** A healthy validator stays predominantly in Proposing; a stock node stays in Observing.\n**Healthy range:** Mostly one dominant mode with negligible Wrong/Switched Ledger.\n**Watch for:** Frequent Wrong Ledger or Switched Ledger indicates the node is out of sync and at fork risk.\n**Source:** src/xrpld/app/consensus/RCLConsensus.cpp:RCLConsensus::Adaptor::onClose", "type": "timeseries", "gridPos": { "h": 8, @@ -278,7 +278,7 @@ }, { "title": "Accept vs Close Rate", - "description": "Compares the rate of consensus.accept (ledger accepted after consensus) vs consensus.ledger_close (ledger close initiated). These should track closely in a healthy network. A divergence means some close events are not completing the accept phase, potentially indicating consensus failures or timeouts.", + "description": "**What:** Rate of ledgers accepted after consensus versus ledger-close events initiated.\n**How it's computed:** Per-second rate of each over a 5-minute window, per node.\n**Reading it:** The two lines should overlap on a healthy network.\n**Healthy range:** Both near the ledger cadence (~0.25/s on mainnet) and tracking each other.\n**Watch for:** Divergence means some closes never complete the accept phase, hinting at consensus failures or timeouts.\n**Source:** src/xrpld/app/consensus/RCLConsensus.cpp:RCLConsensus::Adaptor::makeAcceptSpan, onClose", "type": "timeseries", "gridPos": { "h": 8, @@ -324,7 +324,7 @@ }, { "title": "Validation vs Close Rate", - "description": "Compares the rate of consensus.validation.send vs consensus.ledger_close. Each validated ledger should produce one validation message. If validations lag behind closes, the node may be falling behind on validation or experiencing issues with the validation pipeline.", + "description": "**What:** Rate of validations sent versus ledger-close events.\n**How it's computed:** Per-second rate of each over a 5-minute window, per node.\n**Reading it:** Each validated ledger should yield one validation, so the lines should overlap.\n**Healthy range:** Both near the ledger cadence and tracking each other.\n**Watch for:** Validations trailing closes points to a stalled or slow validation pipeline.\n**Source:** src/xrpld/app/consensus/RCLConsensus.cpp:RCLConsensus::Adaptor::createValidationSpan, onClose", "type": "timeseries", "gridPos": { "h": 8, @@ -370,7 +370,7 @@ }, { "title": "Consensus Accept Duration Heatmap", - "description": "Heatmap showing the distribution of consensus.accept span durations across histogram buckets over time. Each cell represents how many accept events fell into that duration bucket in a 5m window. Useful for detecting outlier consensus rounds that take abnormally long.", + "description": "**What:** Distribution of per-round accept durations over time, showing how round times are spread rather than just a percentile.\n**How it's computed:** Counts of rounds by duration band in each 5-minute window, per node.\n**Reading it:** A tight band around the typical round time is healthy; brighter cells mark the common duration.\n**Healthy range:** Concentrated near the 3-6 second mainnet round time.\n**Watch for:** A second band at high durations reveals recurring slow rounds hidden by averages.\n**Source:** src/xrpld/app/consensus/RCLConsensus.cpp:RCLConsensus::Adaptor::makeAcceptSpan", "type": "heatmap", "gridPos": { "h": 8, @@ -406,7 +406,7 @@ }, { "title": "Close Time: Raw Proposals (Per Node)", - "description": "Each node's raw proposed close time (close_time_self) \u2014 the unrounded wall clock value at the moment the node closed its ledger. Compare across nodes to see clock drift.", + "description": "**What:** Each node's raw, unrounded proposed close time at the instant it closed its ledger.\n**How it's computed:** Latest raw close-time value per node, plotted per round.\n**Reading it:** Compare nodes at the same round; values should cluster tightly.\n**Healthy range:** All nodes within a few seconds of each other.\n**Watch for:** A node consistently offset from the pack indicates local clock drift.\n**Source:** src/xrpld/app/consensus/RCLConsensus.cpp:RCLConsensus::Adaptor::doAccept", "type": "timeseries", "gridPos": { "h": 8, @@ -449,7 +449,7 @@ }, { "title": "Close Time: Effective / Quantized", - "description": "The consensus-agreed close time after rounding to the current resolution bin (close_time). This is the value written to the ledger header. All nodes in agreement produce the same value.", + "description": "**What:** The consensus-agreed close time after rounding to the active resolution bin, i.e. the value written to the ledger header.\n**How it's computed:** Latest effective close-time value per node, plotted per round.\n**Reading it:** All in-agreement nodes should report the identical value each round.\n**Healthy range:** Identical across agreeing nodes.\n**Watch for:** Nodes reporting different effective values are not in close-time agreement for that round.\n**Source:** src/xrpld/app/consensus/RCLConsensus.cpp:RCLConsensus::Adaptor::doAccept", "type": "timeseries", "gridPos": { "h": 8, @@ -492,7 +492,7 @@ }, { "title": "Close Time Vote Bins & Resolution", - "description": "Number of distinct close time vote bins (close_time_vote_bins) and the bin size / resolution in ms (close_resolution_ms). More bins = more clock disagreement. Resolution adapts: finer (10s) when validators agree, coarser (120s) when they disagree.", + "description": "**What:** Number of distinct close-time vote buckets and the current close-time resolution, per round.\n**How it's computed:** Latest bin count and resolution value plotted per round, per node.\n**Reading it:** Fewer bins and finer resolution mean tighter agreement; resolution coarsens automatically when validators disagree.\n**Healthy range:** Few bins with fine (about 10s) resolution during healthy operation.\n**Watch for:** Many bins or resolution jumping to its coarse ceiling signals widespread clock disagreement.\n**Source:** src/xrpld/app/consensus/RCLConsensus.cpp:RCLConsensus::Adaptor::doAccept", "type": "timeseries", "gridPos": { "h": 8, @@ -576,7 +576,7 @@ }, { "title": "Close Time Resolution Direction", - "description": "Whether close time resolution increased (coarser bins, more disagreement), decreased (finer bins, better agreement), or stayed unchanged relative to the previous ledger. Based on resolution_direction attribute.", + "description": "**What:** Whether close-time resolution coarsened, sharpened, or held steady versus the previous ledger.\n**How it's computed:** Latest direction outcome plotted per round, per node.\n**Reading it:** Mostly unchanged with occasional sharpening is healthy.\n**Healthy range:** Predominantly unchanged.\n**Watch for:** Repeated coarsening indicates worsening agreement round over round.\n**Source:** src/xrpld/app/consensus/RCLConsensus.cpp:RCLConsensus::Adaptor::doAccept", "type": "timeseries", "gridPos": { "h": 8, @@ -619,7 +619,7 @@ }, { "title": "Close Time Bin Distribution", - "description": "Distribution of raw proposed close times across quantized bins. Shows how many nodes' proposals landed in each resolution bin per consensus round. A single dominant bin indicates good clock agreement; spread across bins indicates drift or network latency.", + "description": "**What:** How raw proposed close times spread across the quantized buckets each round.\n**How it's computed:** Count of proposals landing in each bucket, stacked per round.\n**Reading it:** A single dominant bar means strong clock agreement.\n**Healthy range:** One dominant bucket per round.\n**Watch for:** Spread across many buckets points to clock drift or network latency between validators.\n**Source:** src/xrpld/app/consensus/RCLConsensus.cpp:RCLConsensus::Adaptor::doAccept", "type": "barchart", "gridPos": { "h": 8, @@ -663,7 +663,7 @@ }, { "title": "Consensus Outcome Distribution", - "description": "Distribution of consensus.accept outcomes: yes (normal), moved_on (without full agreement), expired (timeout). Non-yes outcomes indicate network stress.", + "description": "**What:** Share of consensus outcomes: normal agreement, moved on without full agreement, or expired (timed out).\n**How it's computed:** Total counts per outcome over a 5-minute window, shown as proportions.\n**Reading it:** The normal outcome should dominate overwhelmingly.\n**Healthy range:** Nearly all normal.\n**Watch for:** A growing moved-on or expired slice indicates network stress or instability.\n**Source:** src/xrpld/app/consensus/RCLConsensus.cpp:RCLConsensus::Adaptor::makeAcceptSpan", "type": "piechart", "gridPos": { "h": 8, @@ -699,7 +699,7 @@ }, { "title": "Consensus Failures Over Time", - "description": "Rate of non-normal consensus outcomes (moved_on + expired). Spikes indicate consensus instability.", + "description": "**What:** Rate of non-normal consensus outcomes (moved on plus expired) over time.\n**How it's computed:** Per-second rate of each failure outcome over a 5-minute window, per node.\n**Reading it:** Should sit at or near zero.\n**Healthy range:** Effectively zero on a stable network.\n**Watch for:** Spikes indicate consensus instability; sustained failures warrant investigation.\n**Source:** src/xrpld/app/consensus/RCLConsensus.cpp:RCLConsensus::Adaptor::makeAcceptSpan", "type": "timeseries", "gridPos": { "h": 8, @@ -745,7 +745,7 @@ }, { "title": "Consensus Round Duration (Full Round)", - "description": "p95/p50 duration of the full consensus round. The consensus.round span (RCLConsensus.cpp startRound) wraps an entire round end-to-end. Filterable by consensus mode. This is the single most important consensus-health signal; rising round time precedes ledger-age alarms.", + "description": "**What:** End-to-end duration of a complete consensus round, filterable by consensus mode.\n**How it's computed:** 95th and 50th percentile of full-round durations over a 5-minute window, per node.\n**Reading it:** The single most important consensus-health signal; watch the trend.\n**Healthy range:** Roughly 3-6 seconds on mainnet.\n**Watch for:** Rising round time precedes validated-ledger-age alarms and points to convergence trouble.\n**Source:** src/xrpld/app/consensus/RCLConsensus.cpp:RCLConsensus::Adaptor::startRoundTracing", "type": "timeseries", "gridPos": { "h": 8, @@ -791,7 +791,7 @@ }, { "title": "Consensus Phase Duration (Open vs Establish)", - "description": "p95 duration of the open phase (transaction collection) vs the establish phase (proposal convergence). The consensus.phase.open and consensus.establish spans decompose round latency, so an operator can tell whether slowness is in collecting transactions or reaching agreement.", + "description": "**What:** Duration of the open phase (collecting transactions) versus the establish phase (converging on a proposal), which together make up a round.\n**How it's computed:** 95th percentile of each phase's duration over a 5-minute window, per node.\n**Reading it:** Compare the two to see whether slowness is in gathering transactions or reaching agreement.\n**Healthy range:** Open phase near the close interval; establish phase typically shorter.\n**Watch for:** A ballooning establish phase means slow convergence; a long open phase means delayed close.\n**Source:** src/xrpld/consensus/Consensus.h:Consensus::phaseOpen, Consensus::phaseEstablish", "type": "timeseries", "gridPos": { "h": 8, @@ -837,7 +837,7 @@ }, { "title": "Position Update Duration", - "description": "p95/p50 duration of the consensus.update_positions span, which tallies disputes and updates this node's position each round. Long durations indicate heavy dispute resolution or slow convergence on close time.", + "description": "**What:** Time spent each round tallying disputes and updating this node's consensus position.\n**How it's computed:** 95th and 50th percentile of per-round durations over a 5-minute window, per node.\n**Reading it:** Grows with the number of disputed transactions being reconciled.\n**Healthy range:** A few milliseconds under normal conditions.\n**Watch for:** Long durations indicate heavy dispute resolution or slow convergence on close time.\n**Source:** src/xrpld/consensus/Consensus.h:Consensus::updateOurPositions", "type": "timeseries", "gridPos": { "h": 8, @@ -883,7 +883,7 @@ }, { "title": "Consensus Stall Rate", - "description": "Rate of consensus.check spans reporting consensus_stalled=true, broken down by stall flag. A non-zero stalled rate surfaces stall conditions before they manifest as validated-ledger-age alarms. Requires the consensus_stalled spanmetrics dimension.", + "description": "**What:** Rate of consensus health checks reporting a stalled condition versus progressing normally.\n**How it's computed:** Per-second rate split by stalled flag over a 5-minute window, per node.\n**Reading it:** The stalled line should stay at zero.\n**Healthy range:** Zero stalled checks.\n**Watch for:** Any sustained stalled rate surfaces a stall before it becomes a validated-ledger-age alarm.\n**Source:** src/xrpld/consensus/Consensus.h:Consensus::haveConsensus", "type": "timeseries", "gridPos": { "h": 8, @@ -929,7 +929,7 @@ }, { "title": "Consensus Mode-Change Rate by Target Mode", - "description": "Rate of consensus.mode_change spans broken down by the mode the node switched INTO (mode_new). Frequent switches into Wrong Ledger or Switched Ledger indicate an unstable node at fork risk. Requires the mode_new spanmetrics dimension.", + "description": "**What:** Rate of consensus mode transitions, split by the mode the node switched into.\n**How it's computed:** Per-second rate per target mode over a 5-minute window, per node.\n**Reading it:** A stable node changes mode rarely; occasional switches around startup are normal.\n**Healthy range:** Near zero once synced.\n**Watch for:** Frequent switches into Wrong Ledger or Switched Ledger mark an unstable node at fork risk.\n**Source:** src/xrpld/app/consensus/RCLConsensus.cpp:RCLConsensus::Adaptor::onModeChange", "type": "timeseries", "gridPos": { "h": 8, @@ -968,7 +968,7 @@ } ], "schemaVersion": 39, - "tags": ["consensus", "telemetry"], + "tags": ["consensus"], "templating": { "list": [ { @@ -1034,7 +1034,7 @@ { "name": "node", "label": "Node", - "description": "Filter by rippled node (service.instance.id \u2014 e.g. Node-1)", + "description": "Filter by rippled node (service.instance.id — e.g. Node-1)", "type": "query", "query": "label_values(traces_span_metrics_calls_total, exported_instance)", "datasource": { @@ -1142,6 +1142,7 @@ "from": "now-1h", "to": "now" }, + "description": "Consensus round health: round and phase durations, proposal and validation rates, close-time agreement, stall detection, and ledger-close cadence.", "title": "Consensus Health", "uid": "xrpld-consensus" } diff --git a/docker/telemetry/grafana/dashboards/ledger-operations.json b/docker/telemetry/grafana/dashboards/ledger-operations.json index ae24b4299c..6c75423b74 100644 --- a/docker/telemetry/grafana/dashboards/ledger-operations.json +++ b/docker/telemetry/grafana/dashboards/ledger-operations.json @@ -10,7 +10,7 @@ "panels": [ { "title": "Ledger Build Rate", - "description": "Rate at which new ledgers are being built. The ledger.build span (BuildLedger.cpp) wraps the entire buildLedgerImpl() function which creates a new ledger from a parent, applies transactions, flushes SHAMap nodes, and sets the accepted state. Should match the consensus close rate (~0.25/sec on mainnet with ~4s rounds).", + "description": "**What:** How often the node constructs a new ledger from its parent, applying transactions and writing state.\n**How it's computed:** Per-second rate over a 5-minute window, per node.\n**Reading it:** Should match the consensus close cadence.\n**Healthy range:** About 0.25/s on mainnet (~4s rounds).\n**Watch for:** A drop below the close rate means the node is not keeping up with ledger construction.\n**Source:** src/xrpld/app/ledger/detail/BuildLedger.cpp:buildLedgerImpl", "type": "stat", "gridPos": { "h": 8, @@ -42,7 +42,7 @@ }, { "title": "Ledger Build Duration", - "description": "p95 and p50 duration of ledger builds. Measures the full buildLedgerImpl() call including transaction application, SHAMap flushing, and ledger acceptance. The span records xrpl.ledger.seq as an attribute. Long build times indicate expensive transaction sets or I/O pressure from SHAMap flushes.", + "description": "**What:** Time to build a full ledger, including applying transactions, writing state, and finalising acceptance.\n**How it's computed:** 95th and 50th percentile of per-build durations over a 5-minute window, per node.\n**Reading it:** Scales with transaction volume and disk I/O.\n**Healthy range:** Tens to a few hundred milliseconds under typical load.\n**Watch for:** Long build times indicate expensive transaction sets or I/O pressure while flushing state.\n**Source:** src/xrpld/app/ledger/detail/BuildLedger.cpp:buildLedgerImpl", "type": "timeseries", "gridPos": { "h": 8, @@ -88,7 +88,7 @@ }, { "title": "Ledger Validation Rate", - "description": "Rate at which ledgers pass the validation threshold and are accepted as fully validated. The ledger.validate span (LedgerMaster.cpp) fires in checkAccept() only after the ledger receives sufficient trusted validations (>= quorum). Records xrpl.ledger.seq and validations (the number of validations received).", + "description": "**What:** How often ledgers reach the trusted-validation quorum and become fully validated.\n**How it's computed:** Per-second rate over a 5-minute window, per node.\n**Reading it:** Should match the build and close rates on a healthy node.\n**Healthy range:** About 0.25/s on mainnet.\n**Watch for:** A shortfall means ledgers are not accumulating enough trusted validations to be accepted.\n**Source:** src/xrpld/app/ledger/detail/LedgerMaster.cpp:LedgerMaster::checkAccept", "type": "stat", "gridPos": { "h": 8, @@ -120,7 +120,7 @@ }, { "title": "Ledger Build Duration Heatmap", - "description": "Heatmap showing the distribution of ledger.build durations across histogram buckets over time. Each cell represents the count of ledger builds that fell into that duration bucket in a 5m window. Useful for spotting occasional slow ledger builds that may not appear in percentile charts.", + "description": "**What:** Distribution of ledger-build durations over time, revealing spread beyond a single percentile.\n**How it's computed:** Counts of builds by duration band in each 5-minute window, per node.\n**Reading it:** A tight band at low durations is healthy; brighter cells mark the common build time.\n**Healthy range:** Concentrated in the low tens of milliseconds.\n**Watch for:** An occasional high-duration band flags slow builds that percentile charts may miss.\n**Source:** src/xrpld/app/ledger/detail/BuildLedger.cpp:buildLedgerImpl", "type": "heatmap", "gridPos": { "h": 8, @@ -156,7 +156,7 @@ }, { "title": "Transaction Apply Duration", - "description": "p95 and p50 duration of applying the consensus transaction set during ledger building. The tx.apply span (BuildLedger.cpp) wraps applyTransactions() which iterates through the CanonicalTXSet with multiple retry passes. Records tx_count (successful) and tx_failed (failed) as attributes.", + "description": "**What:** Time to apply the agreed transaction set into the new ledger, including retry passes.\n**How it's computed:** 95th and 50th percentile of per-ledger apply durations over a 5-minute window, per node.\n**Reading it:** Grows with the number and complexity of transactions in the set.\n**Healthy range:** Tens to a few hundred milliseconds under normal load.\n**Watch for:** Sustained high durations indicate heavy or expensive transaction sets.\n**Source:** src/xrpld/app/ledger/detail/BuildLedger.cpp:applyTransactions", "type": "timeseries", "gridPos": { "h": 8, @@ -202,7 +202,7 @@ }, { "title": "Transaction Apply Rate", - "description": "Rate of tx.apply span invocations, reflecting how frequently the transaction application phase runs during ledger building. Each ledger build triggers one tx.apply call. Should closely match the ledger build rate.", + "description": "**What:** How often the transaction-application phase runs during ledger building.\n**How it's computed:** Per-second rate over a 5-minute window, per node.\n**Reading it:** One apply pass per ledger build, so it should track the build rate.\n**Healthy range:** About 0.25/s on mainnet.\n**Watch for:** Divergence from the build rate signals an accounting or pipeline anomaly.\n**Source:** src/xrpld/app/ledger/detail/BuildLedger.cpp:applyTransactions", "type": "timeseries", "gridPos": { "h": 8, @@ -241,7 +241,7 @@ }, { "title": "Ledger Store Rate", - "description": "Rate at which ledgers are stored into the ledger history. The ledger.store span (LedgerMaster.cpp) wraps storeLedger() which inserts the ledger into the LedgerHistory cache. Records xrpl.ledger.seq. Should match the ledger build rate under normal operation.", + "description": "**What:** How often completed ledgers are written into the local ledger history.\n**How it's computed:** Per-second rate over a 5-minute window, per node.\n**Reading it:** Should match the build rate under normal operation.\n**Healthy range:** About 0.25/s on mainnet.\n**Watch for:** A store rate below the build rate means ledgers are being built but not persisted.\n**Source:** src/xrpld/app/ledger/detail/LedgerMaster.cpp:LedgerMaster::storeLedger", "type": "stat", "gridPos": { "h": 8, @@ -273,7 +273,7 @@ }, { "title": "Build vs Close Duration", - "description": "Compares p95 durations of ledger.build (the actual ledger construction in BuildLedger.cpp) vs consensus.ledger_close (the consensus close event in RCLConsensus.cpp). Build time is a subset of close time. A large gap between them indicates overhead in the consensus pipeline outside of ledger construction itself.", + "description": "**What:** Ledger construction time versus total ledger-close time; construction is a subset of close.\n**How it's computed:** 95th percentile of each duration over a 5-minute window, per node.\n**Reading it:** Build should sit at or just below close.\n**Healthy range:** Both a few hundred milliseconds or less, tracking each other.\n**Watch for:** A large gap means significant overhead in the consensus pipeline outside ledger construction.\n**Source:** src/xrpld/app/ledger/detail/BuildLedger.cpp:buildLedgerImpl, src/xrpld/app/consensus/RCLConsensus.cpp:RCLConsensus::Adaptor::onClose", "type": "timeseries", "gridPos": { "h": 8, @@ -319,7 +319,7 @@ } ], "schemaVersion": 39, - "tags": ["ledger", "telemetry"], + "tags": ["ledger"], "templating": { "list": [ { @@ -385,7 +385,7 @@ { "name": "node", "label": "Node", - "description": "Filter by rippled node (service.instance.id \u2014 e.g. Node-1)", + "description": "Filter by rippled node (service.instance.id — e.g. Node-1)", "type": "query", "query": "label_values(traces_span_metrics_calls_total, exported_instance)", "datasource": { @@ -408,6 +408,7 @@ "from": "now-1h", "to": "now" }, + "description": "Ledger lifecycle: build rate and duration, transaction apply and store rates, validation rate, and close timing.", "title": "Ledger Operations", "uid": "xrpld-ledger-ops" } diff --git a/docker/telemetry/grafana/dashboards/peer-network.json b/docker/telemetry/grafana/dashboards/peer-network.json index 091292460b..ab8c535aa7 100644 --- a/docker/telemetry/grafana/dashboards/peer-network.json +++ b/docker/telemetry/grafana/dashboards/peer-network.json @@ -2,7 +2,7 @@ "annotations": { "list": [] }, - "description": "Requires trace_peer=1 in the [telemetry] config section.", + "description": "Peer-to-peer consensus traffic: rates of proposals and validations received from peers and their split between trusted and untrusted sources.", "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, @@ -11,7 +11,7 @@ "panels": [ { "title": "Peer Proposal Receive Rate", - "description": "Rate of consensus proposals received from network peers. The peer.proposal.receive span (PeerImp.cpp) fires in onMessage(TMProposeSet) for each incoming proposal. Records xrpl.peer.id (sending peer) and proposal_trusted (whether the proposer is in our UNL). Requires trace_peer=1 in the telemetry config.", + "description": "**What:** Rate of consensus proposals arriving from connected network peers.\n**How it's computed:** Per-second rate over a 5-minute window, per node.\n**Reading it:** Reflects how many validators' proposals reach this node each round.\n**Healthy range:** Workload-dependent; scales with peer count and network activity.\n**Watch for:** A sudden drop means loss of proposal flow; a sharp sustained spike from a single peer can indicate flooding.\n**Source:** src/xrpld/overlay/detail/PeerImp.cpp:PeerImp::onMessage(TMProposeSet)", "type": "timeseries", "gridPos": { "h": 8, @@ -50,7 +50,7 @@ }, { "title": "Peer Validation Receive Rate", - "description": "Rate of ledger validations received from network peers. The peer.validation.receive span (PeerImp.cpp) fires in onMessage(TMValidation) for each incoming validation message. Records xrpl.peer.id (sending peer) and validation_trusted (whether the validator is trusted). Requires trace_peer=1 in the telemetry config.", + "description": "**What:** Rate of ledger validations arriving from connected network peers.\n**How it's computed:** Per-second rate over a 5-minute window, per node.\n**Reading it:** Reflects the volume of validation traffic reaching this node.\n**Healthy range:** Workload-dependent; scales with the number of validators and peers.\n**Watch for:** A drop starves the validation quorum; an abnormal spike from one peer can indicate abuse.\n**Source:** src/xrpld/overlay/detail/PeerImp.cpp:PeerImp::onMessage(TMValidation)", "type": "timeseries", "gridPos": { "h": 8, @@ -89,7 +89,7 @@ }, { "title": "Proposals Trusted vs Untrusted", - "description": "Pie chart showing the ratio of proposals received from trusted validators (in our UNL) vs untrusted validators. Grouped by the proposal_trusted span attribute (true/false). A healthy node connected to a well-configured UNL should see a significant portion of trusted proposals. Note: proposals that fail early validation may not have the trusted attribute set.", + "description": "**What:** Share of received proposals from trusted validators (in this node's UNL) versus untrusted sources.\n**How it's computed:** Total counts split by trust status over a 5-minute window, shown as proportions.\n**Reading it:** A well-connected node with a good UNL sees a healthy trusted share.\n**Healthy range:** A substantial trusted proportion; exact mix is workload-dependent.\n**Watch for:** A collapse of the trusted share means poor connectivity to UNL validators; a flood of untrusted proposals can indicate abuse.\n**Source:** src/xrpld/overlay/detail/PeerImp.cpp:PeerImp::onMessage(TMProposeSet)", "type": "piechart", "gridPos": { "h": 8, @@ -121,7 +121,7 @@ }, { "title": "Validations Trusted vs Untrusted", - "description": "Pie chart showing the ratio of validations received from trusted validators (in our UNL) vs untrusted validators. Grouped by the validation_trusted span attribute (true/false). Monitoring this helps detect if the node is receiving validations from the expected set of trusted validators. Note: validations that fail early checks may not have the trusted attribute set.", + "description": "**What:** Share of received validations from trusted validators (in this node's UNL) versus untrusted sources.\n**How it's computed:** Total counts split by trust status over a 5-minute window, shown as proportions.\n**Reading it:** Confirms the node is hearing from its expected trusted validator set.\n**Healthy range:** A substantial trusted proportion; exact mix is workload-dependent.\n**Watch for:** A shrinking trusted share risks the validation quorum; a surge of untrusted validations can indicate abuse.\n**Source:** src/xrpld/overlay/detail/PeerImp.cpp:PeerImp::onMessage(TMValidation)", "type": "piechart", "gridPos": { "h": 8, @@ -153,7 +153,7 @@ } ], "schemaVersion": 39, - "tags": ["peer", "telemetry"], + "tags": ["network", "peer"], "templating": { "list": [ { @@ -219,7 +219,7 @@ { "name": "node", "label": "Node", - "description": "Filter by rippled node (service.instance.id \u2014 e.g. Node-1)", + "description": "Filter by rippled node (service.instance.id — e.g. Node-1)", "type": "query", "query": "label_values(traces_span_metrics_calls_total, exported_instance)", "datasource": { diff --git a/docker/telemetry/grafana/dashboards/rpc-performance.json b/docker/telemetry/grafana/dashboards/rpc-performance.json index fba8e92c45..45db3632b7 100644 --- a/docker/telemetry/grafana/dashboards/rpc-performance.json +++ b/docker/telemetry/grafana/dashboards/rpc-performance.json @@ -10,7 +10,7 @@ "panels": [ { "title": "RPC Request Rate by Command", - "description": "Per-second rate of RPC command executions, broken down by command name (e.g. server_info, submit). Calculated as rate(traces_span_metrics_calls_total{span_name=~\"rpc.command.*\"}) over a 5m window, grouped by the command span attribute.", + "description": "**What:** Per-command throughput of RPC calls the node serves (e.g. server_info, submit, account_info).\n**How it's computed:** Per-second rate split by command name over a 5-minute window, per node.\n**Reading it:** Shows which API endpoints drive load and how demand shifts over time.\n**Healthy range:** Workload-dependent; varies with client mix.\n**Watch for:** A sudden spike on one command can indicate a runaway client or scripted abuse.\n**Source:** src/xrpld/rpc/detail/RPCHandler.cpp:callMethod", "type": "timeseries", "gridPos": { "h": 8, @@ -49,7 +49,7 @@ }, { "title": "RPC Latency P95 by Command", - "description": "95th percentile response time for each RPC command. Computed from the spanmetrics duration histogram using histogram_quantile(0.95) over rpc.command.* spans, grouped by command. High values indicate slow commands that may need optimization.", + "description": "**What:** 95th-percentile response time for each RPC command.\n**How it's computed:** 95th percentile of per-command durations over a 5-minute window, per node.\n**Reading it:** Read-only lookups should be fast; heavy commands like path finding are naturally slower.\n**Healthy range:** Sub-second for most commands; command-dependent.\n**Watch for:** Rising p95 on a normally fast command signals contention or an expensive query pattern.\n**Source:** src/xrpld/rpc/detail/RPCHandler.cpp:callMethod", "type": "timeseries", "gridPos": { "h": 8, @@ -88,7 +88,7 @@ }, { "title": "RPC Error Rate", - "description": "Percentage of RPC commands that completed with an error status, per command. Calculated as (error calls / total calls) * 100, where errors have status_code=STATUS_CODE_ERROR. Thresholds: green < 1%, yellow 1-5%, red > 5%.", + "description": "**What:** Percentage of each RPC command's calls that ended in an error.\n**How it's computed:** Error calls divided by total calls per command over a 5-minute window, per node.\n**Reading it:** Green below 1%, yellow 1-5%, red above 5%.\n**Healthy range:** Below 1% for healthy commands.\n**Watch for:** A sustained error spike on one command is consistent with a client probing or misusing that endpoint.\n**Source:** src/xrpld/rpc/detail/RPCHandler.cpp:callMethod", "type": "bargauge", "gridPos": { "h": 8, @@ -136,7 +136,7 @@ }, { "title": "RPC Latency Heatmap", - "description": "Distribution of RPC command response times across histogram buckets. Shows the density of requests at each latency level over time. Each cell represents the count of requests that fell into that duration bucket in a 5m window. Useful for spotting bimodal latency patterns.", + "description": "**What:** Distribution of RPC response times over time across all commands.\n**How it's computed:** Counts of requests by duration band in each 5-minute window, per node.\n**Reading it:** A single tight band is healthy; brighter cells mark the common latency.\n**Healthy range:** Concentrated at low latency; command-dependent.\n**Watch for:** Two separate bands (bimodal latency) reveal a slow path affecting some requests.\n**Source:** src/xrpld/rpc/detail/RPCHandler.cpp:callMethod", "type": "heatmap", "gridPos": { "h": 8, @@ -172,7 +172,7 @@ }, { "title": "Overall RPC Throughput", - "description": "Aggregate RPC throughput showing two layers of the request pipeline. rpc.http_request is the outer HTTP handler (ServerHandler.cpp) that accepts incoming connections. rpc.process is the inner processing layer (ServerHandler.cpp) that parses and dispatches. A gap between the two indicates requests being queued or rejected before processing.", + "description": "**What:** Aggregate request flow through two pipeline layers: the outer HTTP handler that accepts connections and the inner layer that parses and dispatches.\n**How it's computed:** Per-second rate of each layer over a 5-minute window, per node.\n**Reading it:** The two lines should overlap when requests flow cleanly.\n**Healthy range:** Workload-dependent; both layers tracking each other.\n**Watch for:** A gap means requests are being queued or rejected before dispatch.\n**Source:** src/xrpld/rpc/detail/ServerHandler.cpp:ServerHandler::processSession, ServerHandler::processRequest", "type": "timeseries", "gridPos": { "h": 8, @@ -218,7 +218,7 @@ }, { "title": "RPC Success vs Error", - "description": "Aggregate rate of successful vs failed RPC commands across all command types. Success = status_code UNSET (OpenTelemetry default for OK spans). Error = status_code STATUS_CODE_ERROR. A sustained error rate warrants investigation via per-command breakdown above.", + "description": "**What:** Aggregate rate of successful versus failed RPC commands across all types.\n**How it's computed:** Per-second rate of each outcome over a 5-minute window, per node.\n**Reading it:** Success should dominate; the error line should stay low.\n**Healthy range:** Error rate near zero; success rate is workload-dependent.\n**Watch for:** A sustained rise in the error line warrants drilling into the per-command breakdown.\n**Source:** src/xrpld/rpc/detail/RPCHandler.cpp:callMethod", "type": "timeseries", "gridPos": { "h": 8, @@ -264,7 +264,7 @@ }, { "title": "Top Commands by Volume", - "description": "Top 10 most frequently called RPC commands by total invocation count over the last 5 minutes. Uses topk(10, increase(calls_total)) to rank commands. Helps identify the hottest API endpoints driving load on the node.", + "description": "**What:** The ten most-called RPC commands by total invocations recently.\n**How it's computed:** Ranked total counts per command over a 5-minute window, per node.\n**Reading it:** Identifies the hottest API endpoints driving node load.\n**Healthy range:** Workload-dependent.\n**Watch for:** An unexpected command dominating the ranking can indicate scripted abuse or a misbehaving client.\n**Source:** src/xrpld/rpc/detail/RPCHandler.cpp:callMethod", "type": "bargauge", "gridPos": { "h": 8, @@ -296,7 +296,7 @@ }, { "title": "WebSocket Message Rate", - "description": "Rate of incoming WebSocket RPC messages processed by the server. Sourced from the rpc.ws_message span (ServerHandler.cpp). Only active when clients connect via WebSocket instead of HTTP. Zero is normal if only HTTP RPC is in use.", + "description": "**What:** Rate of incoming WebSocket RPC messages the server processes.\n**How it's computed:** Per-second rate over a 5-minute window, per node.\n**Reading it:** Non-zero only when clients connect over WebSocket rather than HTTP.\n**Healthy range:** Zero is normal for HTTP-only deployments; otherwise workload-dependent.\n**Watch for:** An unexpected surge points to a chatty or abusive WebSocket client.\n**Source:** src/xrpld/rpc/detail/ServerHandler.cpp:ServerHandler::processSession", "type": "stat", "gridPos": { "h": 8, @@ -328,7 +328,7 @@ }, { "title": "RPC Resource Cost by Command", - "description": "RPC commands grouped by load_type (resource cost category). High-cost categories like exception_rpc or malformed_rpc indicate problematic clients.", + "description": "**What:** RPC traffic grouped by its resource-cost category, distinguishing cheap lookups from expensive or malformed requests.\n**How it's computed:** Per-second rate per cost category over a 5-minute window.\n**Reading it:** Low-cost categories should dominate.\n**Healthy range:** Mostly low-cost traffic; workload-dependent.\n**Watch for:** A rising share of high-cost or malformed categories points to problematic or abusive clients.\n**Source:** src/xrpld/rpc/detail/RPCHandler.cpp:callMethod", "type": "timeseries", "gridPos": { "h": 8, @@ -372,7 +372,7 @@ }, { "title": "Batch vs Single RPC Requests", - "description": "Rate of batch RPC requests vs single requests. High batch rate may indicate bulk automation clients.", + "description": "**What:** Rate of batched RPC requests versus single requests.\n**How it's computed:** Per-second rate of each over a 5-minute window, per node.\n**Reading it:** Most traffic is typically single requests.\n**Healthy range:** Workload-dependent; batch share usually small.\n**Watch for:** A high batch rate may indicate bulk-automation clients or an attempt to amplify load per connection.\n**Source:** src/xrpld/rpc/detail/ServerHandler.cpp:ServerHandler::processRequest", "type": "timeseries", "gridPos": { "h": 8, @@ -418,7 +418,7 @@ } ], "schemaVersion": 39, - "tags": ["rpc", "telemetry"], + "tags": ["rpc"], "templating": { "list": [ { @@ -484,7 +484,7 @@ { "name": "node", "label": "Node", - "description": "Filter by rippled node (service.instance.id \u2014 e.g. Node-1)", + "description": "Filter by rippled node (service.instance.id — e.g. Node-1)", "type": "query", "query": "label_values(traces_span_metrics_calls_total, exported_instance)", "datasource": { @@ -528,5 +528,6 @@ "to": "now" }, "title": "RPC Performance", - "uid": "xrpld-rpc-perf" + "uid": "xrpld-rpc-perf", + "description": "Client RPC service health: per-command request rates, latency, error rates, throughput across pipeline layers, and resource cost." } diff --git a/docker/telemetry/grafana/dashboards/transaction-overview.json b/docker/telemetry/grafana/dashboards/transaction-overview.json index 6626f7da32..3acb714b34 100644 --- a/docker/telemetry/grafana/dashboards/transaction-overview.json +++ b/docker/telemetry/grafana/dashboards/transaction-overview.json @@ -10,7 +10,7 @@ "panels": [ { "title": "Transaction Processing Rate", - "description": "Rate of transactions entering the processing pipeline. tx.process (NetworkOPs.cpp) fires when a transaction is submitted locally or received from a peer and enters processTransaction(). tx.receive (PeerImp.cpp) fires when a raw transaction message arrives from a peer before deduplication.", + "description": "**What:** Rate of transactions entering the processing pipeline (submitted locally or relayed by peers) versus raw transaction messages arriving from peers before deduplication.\n**How it's computed:** Per-second rate of each over a 5-minute window, per node.\n**Reading it:** The received line sits above the processed line by the volume of duplicates filtered out.\n**Healthy range:** Workload-dependent; scales with network transaction volume.\n**Watch for:** A large and growing gap means heavy duplicate traffic; a processed-rate collapse means submissions are not being handled.\n**Source:** src/xrpld/app/misc/NetworkOPs.cpp:NetworkOPsImp::processTransaction, src/xrpld/overlay/detail/PeerImp.cpp:PeerImp::handleTransaction", "type": "timeseries", "gridPos": { "h": 8, @@ -56,7 +56,7 @@ }, { "title": "Transaction Processing Latency by Type", - "description": "Per-transaction-type processing latency (p95 and p50). Filter with $tx_type variable above.", + "description": "**What:** Per-transaction-type processing latency through the pipeline.\n**How it's computed:** 95th and 50th percentile of per-type durations over a 5-minute window, per node.\n**Reading it:** Simple payments are fast; complex types like offers or AMM operations run longer.\n**Healthy range:** Sub-millisecond to a few milliseconds; type-dependent.\n**Watch for:** A latency climb for one type signals contention or expensive processing specific to it.\n**Source:** src/xrpld/app/misc/NetworkOPs.cpp:NetworkOPsImp::processTransaction", "type": "timeseries", "gridPos": { "h": 8, @@ -107,7 +107,7 @@ }, { "title": "Transaction Path Distribution", - "description": "Breakdown of transactions by origin path. The local attribute indicates whether the transaction was submitted locally (true) or received from a peer (false). Helps understand the ratio of locally-originated vs relayed transactions.", + "description": "**What:** Split of processed transactions by origin: submitted locally versus relayed from peers.\n**How it's computed:** Total counts split by origin over a 5-minute window, shown as proportions.\n**Reading it:** Most nodes see mostly relayed traffic; a submission endpoint sees more local.\n**Healthy range:** Workload-dependent.\n**Watch for:** A surge in local submissions on a node not meant to accept them can indicate misuse.\n**Source:** src/xrpld/app/misc/NetworkOPs.cpp:NetworkOPsImp::processTransaction", "type": "piechart", "gridPos": { "h": 8, @@ -133,7 +133,7 @@ }, { "title": "Transaction Receive vs Suppressed", - "description": "Total rate of raw transaction messages received from peers (tx.receive span from PeerImp.cpp). This fires before deduplication via the HashRouter, so the difference between tx.receive and tx.process reflects suppressed duplicate transactions.", + "description": "**What:** Raw transaction messages received from peers, split by whether they were suppressed as duplicates before processing.\n**How it's computed:** Per-second rate split by suppressed flag over a 5-minute window, per node.\n**Reading it:** A healthy relay network produces a steady suppressed share as duplicates are filtered.\n**Healthy range:** Workload-dependent; suppression is expected and normal.\n**Watch for:** A sharp rise in the suppressed line can reflect gossip amplification or a peer replaying transactions.\n**Source:** src/xrpld/overlay/detail/PeerImp.cpp:PeerImp::handleTransaction", "type": "timeseries", "gridPos": { "h": 8, @@ -172,7 +172,7 @@ }, { "title": "Transaction Processing Duration Heatmap", - "description": "Heatmap showing the distribution of tx.process span durations across histogram buckets over time. Each cell represents the count of transactions that completed within that latency bucket in a 5m window. Reveals whether processing times are consistent or exhibit multi-modal patterns.", + "description": "**What:** Distribution of transaction processing times over time.\n**How it's computed:** Counts of transactions by duration band in each 5-minute window, per node.\n**Reading it:** A single tight band is healthy; brighter cells mark the common processing time.\n**Healthy range:** Concentrated at low latency.\n**Watch for:** A second high-latency band reveals a subset of transactions that are consistently slow.\n**Source:** src/xrpld/app/misc/NetworkOPs.cpp:NetworkOPsImp::processTransaction", "type": "heatmap", "gridPos": { "h": 8, @@ -208,7 +208,7 @@ }, { "title": "Transaction Apply Duration per Ledger", - "description": "p95 and p50 latency of applying the consensus transaction set to a new ledger. The tx.apply span (BuildLedger.cpp) wraps the applyTransactions() function that iterates through the CanonicalTXSet and applies each transaction to the OpenView. Long durations indicate heavy transaction sets or expensive transaction processing.", + "description": "**What:** Time to apply the agreed transaction set into a new ledger.\n**How it's computed:** 95th and 50th percentile of per-ledger apply durations over a 5-minute window, per node.\n**Reading it:** Scales with the number and complexity of transactions per ledger.\n**Healthy range:** Tens to a few hundred milliseconds under normal load.\n**Watch for:** Sustained high durations indicate heavy transaction sets or expensive processing.\n**Source:** src/xrpld/app/ledger/detail/BuildLedger.cpp:applyTransactions", "type": "timeseries", "gridPos": { "h": 8, @@ -254,7 +254,7 @@ }, { "title": "Peer Transaction Receive Rate", - "description": "Rate of transaction messages received from network peers. Sourced from the tx.receive span (PeerImp.cpp) which fires in the onMessage(TMTransaction) handler. High rates may indicate network-wide transaction volume spikes or peer flooding.", + "description": "**What:** Rate of transaction messages received from network peers.\n**How it's computed:** Per-second rate over a 5-minute window, per node.\n**Reading it:** Reflects inbound transaction volume from the network.\n**Healthy range:** Workload-dependent; scales with network activity.\n**Watch for:** A sustained spike may reflect a network-wide volume surge or a peer flooding transactions.\n**Source:** src/xrpld/overlay/detail/PeerImp.cpp:PeerImp::handleTransaction", "type": "timeseries", "gridPos": { "h": 8, @@ -293,7 +293,7 @@ }, { "title": "Transaction Apply Failed Rate", - "description": "Rate of tx.apply spans completing with error status, indicating transaction application failures during ledger building. The span records tx_failed as an attribute. Thresholds: green < 0.1/sec, yellow 0.1-1/sec, red > 1/sec. Some failures are normal (e.g. conflicting offers) but sustained high rates may indicate issues.", + "description": "**What:** Rate of transaction-apply attempts that failed while building a ledger.\n**How it's computed:** Per-second rate of failed applies over a 5-minute window, per node.\n**Reading it:** Green below 0.1/s, yellow 0.1-1/s, red above 1/s.\n**Healthy range:** Some failures (e.g. conflicting offers) are normal; keep it under 0.1/s.\n**Watch for:** A sustained high failure rate can indicate malformed transaction floods or a systemic issue.\n**Source:** src/xrpld/app/ledger/detail/BuildLedger.cpp:applyTransactions", "type": "stat", "gridPos": { "h": 8, @@ -341,7 +341,7 @@ }, { "title": "Transaction Rate by Type", - "description": "Transaction processing rate broken down by tx_type (Payment, OfferCreate, AMMDeposit, etc.). Requires tx_type dimension in spanmetrics.", + "description": "**What:** Processing rate broken down by transaction type (Payment, OfferCreate, AMM operations, etc.).\n**How it's computed:** Per-second rate per type over a 5-minute window.\n**Reading it:** Shows the transaction mix and how it shifts over time.\n**Healthy range:** Workload-dependent; usually Payment-heavy.\n**Watch for:** A sudden burst of one type is a leading indicator of spam or fee escalation.\n**Source:** src/xrpld/app/misc/NetworkOPs.cpp:NetworkOPsImp::processTransaction", "type": "timeseries", "gridPos": { "h": 8, @@ -385,7 +385,7 @@ }, { "title": "Transaction Results by Type", - "description": "Transaction result codes (ter_result) broken down by tx_type. Shows which transaction types fail most often.", + "description": "**What:** Non-success result codes broken down by transaction type, showing which types fail and how.\n**How it's computed:** Per-second rate per type and result code over a 5-minute window (successes excluded).\n**Reading it:** Occasional failures are normal; watch which types and codes dominate.\n**Healthy range:** Low and workload-dependent.\n**Watch for:** A concentration of one failure code on one type can indicate a broken client or targeted spam.\n**Source:** src/xrpld/app/misc/NetworkOPs.cpp:NetworkOPsImp::processTransaction", "type": "timeseries", "gridPos": { "h": 8, @@ -429,7 +429,7 @@ }, { "title": "TxQ Accept Status", - "description": "TxQ accept outcomes: applied (included in ledger), failed (removed), retried (kept for next round).", + "description": "**What:** Outcomes when queued transactions are considered for a ledger: applied (included), failed (removed), or retried (kept for the next round).\n**How it's computed:** Total counts per outcome over a 5-minute window, shown as proportions.\n**Reading it:** Applied should dominate when the queue is draining healthily.\n**Healthy range:** Mostly applied; workload-dependent.\n**Watch for:** A rising retried or failed share signals queue pressure and fee escalation.\n**Source:** src/xrpld/app/misc/detail/TxQ.cpp:TxQ::accept", "type": "piechart", "gridPos": { "h": 8, @@ -465,7 +465,7 @@ }, { "title": "Transactor Duration by Type (p95)", - "description": "Per-transactor execution time (tx.transactor span). Shows which transaction types are most expensive to execute.", + "description": "**What:** Per-type execution time of the transactor that actually applies each transaction's effects.\n**How it's computed:** 95th percentile of per-type durations over a 5-minute window, per node.\n**Reading it:** Reveals which transaction types are most expensive to execute.\n**Healthy range:** Sub-millisecond to a few milliseconds; type-dependent.\n**Watch for:** A climbing figure for one type points to expensive processing or contention for that type.\n**Source:** src/libxrpl/tx/Transactor.cpp:Transactor::operator()", "type": "timeseries", "gridPos": { "h": 8, @@ -509,7 +509,7 @@ }, { "title": "TxQ Enqueue Rate by Transaction Type", - "description": "Rate of txq.enqueue spans broken down by transaction type (tx_type). Shows what share of inbound demand is Payment vs OfferCreate vs other transactors, and how the mix shifts as the queue fills. A spam burst of one type is a leading indicator of fee escalation.", + "description": "**What:** Rate at which transactions are placed into the queue, split by transaction type.\n**How it's computed:** Per-second rate per type over a 5-minute window, per node.\n**Reading it:** Shows which types make up inbound demand and how the mix shifts as the queue fills.\n**Healthy range:** Workload-dependent.\n**Watch for:** A spam burst of one type here is a leading indicator of fee escalation.\n**Source:** src/xrpld/app/misc/detail/TxQ.cpp:TxQ::apply", "type": "timeseries", "gridPos": { "h": 8, @@ -548,7 +548,7 @@ }, { "title": "Queue Bypass Ratio (Direct Apply vs Enqueue)", - "description": "Ratio of transactions that applied directly to the open ledger (txq.apply_direct) versus those that had to be queued (txq.enqueue). A falling bypass ratio is the cleanest single signal the network has entered sustained fee escalation.", + "description": "**What:** Fraction of transactions that applied straight to the open ledger versus those that had to be queued.\n**How it's computed:** Ratio of direct applies to direct-plus-queued over a 5-minute window, per node.\n**Reading it:** A high fraction means the network is keeping up without escalation.\n**Healthy range:** Near 1.0 in normal conditions.\n**Watch for:** A falling fraction is the cleanest single signal the network has entered sustained fee escalation.\n**Source:** src/xrpld/app/misc/detail/TxQ.cpp:TxQ::tryDirectApply, TxQ::apply", "type": "timeseries", "gridPos": { "h": 8, @@ -587,7 +587,7 @@ }, { "title": "Queue Accept (Drain) Duration per Ledger", - "description": "p95/p50 duration of the txq.accept span, which drains queued transactions into a newly closed ledger. Rising drain time signals queue pressure at ledger close.", + "description": "**What:** Time spent draining queued transactions into a newly closed ledger.\n**How it's computed:** 95th and 50th percentile of per-ledger drain durations over a 5-minute window, per node.\n**Reading it:** Rises as the queue holds more transactions to process at close.\n**Healthy range:** A few milliseconds when the queue is light.\n**Watch for:** Rising drain time signals queue pressure at ledger close.\n**Source:** src/xrpld/app/misc/detail/TxQ.cpp:TxQ::accept", "type": "timeseries", "gridPos": { "h": 8, @@ -633,7 +633,7 @@ }, { "title": "Queue Cleanup Rate (Expired Entries)", - "description": "Rate of txq.cleanup spans, which remove expired transactions from the queue each ledger. A rising rate means submitters under-bid the escalating fee and abandoned their transactions \u2014 a demand-frustration signal distinct from acceptance throughput.", + "description": "**What:** Rate at which expired transactions are removed from the queue each ledger.\n**How it's computed:** Per-second rate over a 5-minute window, per node.\n**Reading it:** Normally low; entries expire when submitters underbid the current fee.\n**Healthy range:** Near zero in calm conditions.\n**Watch for:** A rising rate means submitters abandoned under-fee transactions, a demand-frustration signal distinct from throughput.\n**Source:** src/xrpld/app/misc/detail/TxQ.cpp:TxQ::processClosedLedger", "type": "timeseries", "gridPos": { "h": 8, @@ -672,7 +672,7 @@ }, { "title": "Tx Apply Pipeline Rate by Stage", - "description": "Span rate for each apply-pipeline stage (preflight, preclaim, apply). A drop between stages shows where transactions are filtered out. Requires the stage dimension in spanmetrics.", + "description": "**What:** Throughput of each apply-pipeline stage (preflight, preclaim, apply), showing where transactions drop out.\n**How it's computed:** Per-second rate per stage over a 5-minute window, per node.\n**Reading it:** A decline from earlier to later stages shows where transactions are filtered.\n**Healthy range:** Workload-dependent; later stages sit at or below earlier ones.\n**Watch for:** A large early-stage drop means many transactions fail basic checks, consistent with malformed floods.\n**Source:** src/libxrpl/tx/Transactor.cpp:Transactor::operator()", "type": "timeseries", "gridPos": { "h": 8, @@ -716,7 +716,7 @@ }, { "title": "Tx Apply Pipeline Latency by Stage (p95)", - "description": "95th-percentile duration of each apply-pipeline stage. Isolates which stage (preflight, preclaim, apply) dominates transaction processing time.", + "description": "**What:** 95th-percentile duration of each apply-pipeline stage (preflight, preclaim, apply).\n**How it's computed:** 95th percentile of per-stage durations over a 5-minute window, per node.\n**Reading it:** Isolates which stage dominates transaction processing time.\n**Healthy range:** Sub-millisecond to a few milliseconds per stage.\n**Watch for:** A stage that grows disproportionately points to a bottleneck in that step.\n**Source:** src/libxrpl/tx/Transactor.cpp:Transactor::operator()", "type": "timeseries", "gridPos": { "h": 8, @@ -760,7 +760,7 @@ }, { "title": "Tx Apply Pipeline Failure Rate by Stage", - "description": "Rate of apply-pipeline spans whose ter_result is not tesSUCCESS, split by stage. Shows whether failures concentrate in preflight, preclaim, or apply. Filters on ter_result rather than span status because a failing ter code completes the span normally; only thrown exceptions set an error status.", + "description": "**What:** Rate of pipeline spans ending in a non-success result, split by stage (preflight, preclaim, apply).\n**How it's computed:** Per-second rate of non-success outcomes per stage over a 5-minute window, per node.\n**Reading it:** Shows whether failures concentrate early (basic checks) or late (application).\n**Healthy range:** Low and workload-dependent.\n**Watch for:** A surge of early-stage failures is consistent with malformed or spam transaction floods.\n**Source:** src/libxrpl/tx/Transactor.cpp:Transactor::operator()", "type": "timeseries", "gridPos": { "h": 8, @@ -804,7 +804,7 @@ }, { "title": "Tx Apply Pipeline Latency by Type and Stage (p95)", - "description": "95th-percentile duration broken down by both transaction type and apply-pipeline stage. Shows, for each transaction type, which stage (preflight, preclaim, apply) dominates its latency. Higher cardinality than the by-stage view; filter with the $tx_type and $stage variables.", + "description": "**What:** 95th-percentile stage duration broken down by both transaction type and pipeline stage.\n**How it's computed:** 95th percentile of durations per type and stage over a 5-minute window, per node.\n**Reading it:** For each transaction type, shows which stage dominates its latency.\n**Healthy range:** Sub-millisecond to a few milliseconds; type- and stage-dependent.\n**Watch for:** A specific type-and-stage combination climbing points to a targeted expensive or abusive pattern.\n**Source:** src/libxrpl/tx/Transactor.cpp:Transactor::operator()", "type": "timeseries", "gridPos": { "h": 8, @@ -848,7 +848,7 @@ } ], "schemaVersion": 39, - "tags": ["transactions", "telemetry"], + "tags": ["transactions"], "templating": { "list": [ { @@ -914,7 +914,7 @@ { "name": "node", "label": "Node", - "description": "Filter by rippled node (service.instance.id \u2014 e.g. Node-1)", + "description": "Filter by rippled node (service.instance.id — e.g. Node-1)", "type": "query", "query": "label_values(traces_span_metrics_calls_total, exported_instance)", "datasource": { @@ -1029,6 +1029,7 @@ "from": "now-1h", "to": "now" }, + "description": "Transaction pipeline overview: processing and receive rates, per-type latency and outcomes, queue behaviour, and apply-stage breakdown.", "title": "Transaction Overview", "uid": "xrpld-transactions" }