From 9a4a97cdec1a0986b86e3ef9c56b73745cd9bf5d Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Fri, 3 Jul 2026 21:23:46 +0100 Subject: [PATCH 1/3] docs(telemetry): add dashboard-level descriptions for the info button Consensus Health, Ledger Operations, and Transaction Overview had no top-level description, so Grafana showed no info button. Add a one-line summary of what each dashboard shows and its data source. Co-Authored-By: Claude Opus 4.8 --- docker/telemetry/grafana/dashboards/consensus-health.json | 1 + docker/telemetry/grafana/dashboards/ledger-operations.json | 1 + docker/telemetry/grafana/dashboards/transaction-overview.json | 1 + 3 files changed, 3 insertions(+) diff --git a/docker/telemetry/grafana/dashboards/consensus-health.json b/docker/telemetry/grafana/dashboards/consensus-health.json index a46aa60dfd..eb7b180a6b 100644 --- a/docker/telemetry/grafana/dashboards/consensus-health.json +++ b/docker/telemetry/grafana/dashboards/consensus-health.json @@ -1142,6 +1142,7 @@ "from": "now-1h", "to": "now" }, + "description": "Consensus round health: round duration, proposal and validation rates, close-time agreement, and ledger-close cadence.", "title": "Consensus Health", "uid": "xrpld-consensus" } diff --git a/docker/telemetry/grafana/dashboards/ledger-operations.json b/docker/telemetry/grafana/dashboards/ledger-operations.json index ae24b4299c..f7b3e71f57 100644 --- a/docker/telemetry/grafana/dashboards/ledger-operations.json +++ b/docker/telemetry/grafana/dashboards/ledger-operations.json @@ -408,6 +408,7 @@ "from": "now-1h", "to": "now" }, + "description": "Ledger lifecycle: build rate and duration, validation rate, and close timing.", "title": "Ledger Operations", "uid": "xrpld-ledger-ops" } diff --git a/docker/telemetry/grafana/dashboards/transaction-overview.json b/docker/telemetry/grafana/dashboards/transaction-overview.json index 6626f7da32..bdbb6031d5 100644 --- a/docker/telemetry/grafana/dashboards/transaction-overview.json +++ b/docker/telemetry/grafana/dashboards/transaction-overview.json @@ -1029,6 +1029,7 @@ "from": "now-1h", "to": "now" }, + "description": "Transaction pipeline overview: processing and receive rates, per-type latency, path distribution, apply-stage breakdown, and validation outcomes.", "title": "Transaction Overview", "uid": "xrpld-transactions" } From 7befb167051fa47df909380e0b39c853c43de402 Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Fri, 3 Jul 2026 21:54:07 +0100 Subject: [PATCH 2/3] docs(telemetry): drop (System Metrics) suffix, retag, rewrite descriptions to describe data not pipeline Co-Authored-By: Claude Opus 4.8 --- .../dashboards/system-ledger-data-sync.json | 18 ++++----- .../dashboards/system-network-traffic.json | 26 ++++++------ .../dashboards/system-node-health.json | 40 +++++++++---------- .../system-overlay-traffic-detail.json | 22 +++++----- .../dashboards/system-rpc-pathfinding.json | 36 ++++++++--------- 5 files changed, 71 insertions(+), 71 deletions(-) diff --git a/docker/telemetry/grafana/dashboards/system-ledger-data-sync.json b/docker/telemetry/grafana/dashboards/system-ledger-data-sync.json index d05327f52f..7a35f9c2f5 100644 --- a/docker/telemetry/grafana/dashboards/system-ledger-data-sync.json +++ b/docker/telemetry/grafana/dashboards/system-ledger-data-sync.json @@ -2,7 +2,7 @@ "annotations": { "list": [] }, - "description": "Ledger data exchange and object fetch traffic from beast::insight System Metrics. Covers ledger sync, node data retrieval, and transaction set exchange. Requires [insight] server=otel in rippled config.", + "description": "Ledger data exchange and object-fetch traffic between this node and its peers: ledger sync, tree-node retrieval, and transaction-set exchange. Use it to see how much ledger data the node is pulling or serving and to spot catch-up activity.", "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, @@ -11,7 +11,7 @@ "panels": [ { "title": "Ledger Data Exchange (Bytes In)", - "description": "Inbound bytes for ledger data sub-categories. 'ledger_data' = aggregated ledger data, sub-types include Transaction_Set_candidate (proposed tx sets), Transaction_Node (tx tree nodes), and Account_State_Node (state tree nodes). High Account_State_Node traffic indicates state sync; high Transaction_Set_candidate indicates consensus catch-up. Sourced from TrafficCount.h ledger_data_* categories.", + "description": "**What:** Inbound bytes for ledger-data message categories, split into aggregate get/share plus the transaction-set, transaction-node, and account-state-node sub-types the node receives from peers.\n**How it's computed:** Per-category inbound byte counters, shown at their latest cumulative value per node.\n**Reading it:** Normally low and flat once synced. Account-state-node traffic dominates during state sync; transaction-set-candidate traffic dominates during consensus catch-up.\n**Healthy range:** workload-dependent; low and steady on a synced node.\n**Watch for:** Sustained high account-state or tx-node inbound bytes on a node that should be caught up (repeated re-sync, missing history), or a single peer driving all traffic.\n**Source:** src/xrpld/overlay/detail/OverlayImpl.cpp OverlayImpl ctor (TrafficGauges)", "type": "timeseries", "gridPos": { "h": 8, @@ -99,7 +99,7 @@ }, { "title": "Ledger Share/Get Traffic (Bytes)", - "description": "Legacy ledger share and get traffic by sub-type. These are the older ledger fetch protocol categories (as opposed to ledger_data_* which is the newer protocol). Sub-types: Transaction_Set_candidate, Transaction_node, Account_State_node, plus aggregate ledger_share and ledger_get. Sourced from TrafficCount.h ledger_* categories.", + "description": "**What:** Inbound bytes for the older ledger share/get message categories and their tx-set, tx-node, and account-state sub-types.\n**How it's computed:** Per-category inbound byte counters at their latest value per node.\n**Reading it:** Usually small; these legacy categories carry ledger-fetch traffic for peers using the older protocol.\n**Healthy range:** workload-dependent; low on a synced node.\n**Watch for:** Large sustained volumes indicating heavy fetch load or a peer repeatedly requesting the same data.\n**Source:** src/xrpld/overlay/detail/OverlayImpl.cpp OverlayImpl ctor (TrafficGauges)", "type": "timeseries", "gridPos": { "h": 8, @@ -187,7 +187,7 @@ }, { "title": "GetObject Traffic by Type (Bytes In)", - "description": "Object fetch traffic by object type. GetObject is the protocol for fetching specific SHAMap nodes. Types: Ledger (full ledger headers), Transaction (individual txs), Transaction_node (tx tree nodes), Account_State_node (state tree nodes), CAS (Content Addressable Storage objects), Fetch_Pack (batch fetch during catch-up), Transactions (bulk tx fetch). High Fetch_Pack traffic indicates a node is catching up. Sourced from TrafficCount.h getobject_* categories.", + "description": "**What:** Inbound bytes for object-fetch traffic broken down by object type: ledger headers, individual transactions, transaction-tree nodes, and state-tree nodes.\n**How it's computed:** Per-type inbound byte counters at their latest value per node.\n**Reading it:** Small during steady state; grows when the node fetches missing tree nodes.\n**Healthy range:** workload-dependent; low when synced.\n**Watch for:** A large share on state/tx nodes for long periods (persistent gap-filling), meaning the node keeps catching up.\n**Source:** src/xrpld/overlay/detail/OverlayImpl.cpp OverlayImpl ctor (TrafficGauges)", "type": "timeseries", "gridPos": { "h": 8, @@ -275,7 +275,7 @@ }, { "title": "GetObject Aggregate & Special Types (Bytes In)", - "description": "Aggregate getobject traffic plus special categories: CAS (Content Addressable Storage) for SHAMap node fetch, Fetch_Pack for bulk batch downloads during catch-up, Transactions for bulk tx fetch, and the aggregate getobject_get/getobject_share totals. Sourced from TrafficCount.h getobject_* categories.", + "description": "**What:** Aggregate object-fetch inbound bytes plus special buckets: content-addressed storage fetches, bulk fetch-pack downloads used during catch-up, and bulk transaction fetches.\n**How it's computed:** Per-category inbound byte counters at their latest value per node.\n**Reading it:** Fetch-pack rises sharply while catching up a range of ledgers; near zero when fully synced.\n**Healthy range:** workload-dependent; low when synced.\n**Watch for:** Continuous fetch-pack traffic (node never fully catches up) or unexpectedly high content-store volume.\n**Source:** src/xrpld/overlay/detail/OverlayImpl.cpp OverlayImpl ctor (TrafficGauges)", "type": "timeseries", "gridPos": { "h": 8, @@ -356,7 +356,7 @@ }, { "title": "GetObject Messages by Type", - "description": "Message counts for object fetch operations. Shows how many individual fetch requests and responses are exchanged per type. High message counts with low byte counts indicate small object fetches; the inverse indicates large batch transfers. Sourced from TrafficCount.h getobject_* categories.", + "description": "**What:** Count of individual object-fetch request/response messages per object type.\n**How it's computed:** Per-type inbound message counters at their latest value per node.\n**Reading it:** Many messages with few bytes means small piecemeal fetches; few messages with many bytes means large batch transfers.\n**Healthy range:** workload-dependent.\n**Watch for:** High message counts with tiny payloads sustained over time (inefficient per-node fetching).\n**Source:** src/xrpld/overlay/detail/OverlayImpl.cpp OverlayImpl ctor (TrafficGauges)", "type": "timeseries", "gridPos": { "h": 8, @@ -437,7 +437,7 @@ }, { "title": "Overlay Traffic Heatmap (All Categories, Bytes In)", - "description": "Bar gauge showing all overlay traffic categories ranked by inbound bytes. Provides a complete at-a-glance view of which protocol message types consume the most bandwidth across all 57+ traffic categories. Sourced from all TrafficCount.h categories via wildcard match.", + "description": "**What:** All overlay traffic categories ranked by inbound bytes, giving an at-a-glance view of which message types consume the most receive bandwidth.\n**How it's computed:** Top categories by latest inbound byte value across all traffic categories.\n**Reading it:** The longest bars are the biggest bandwidth consumers; on a synced node transactions, proposals, and validations usually lead.\n**Healthy range:** workload-dependent.\n**Watch for:** A single ledger-data or fetch category dominating (ongoing sync) or an unexpected category topping the list.\n**Source:** src/xrpld/overlay/detail/OverlayImpl.cpp OverlayImpl ctor (TrafficGauges)", "type": "bargauge", "gridPos": { "h": 8, @@ -493,7 +493,7 @@ } ], "schemaVersion": 39, - "tags": ["statsd", "ledger", "sync", "telemetry"], + "tags": ["ledger", "sync"], "templating": { "list": [ { @@ -582,6 +582,6 @@ "from": "now-1h", "to": "now" }, - "title": "Ledger Data & Sync (System Metrics)", + "title": "Ledger Data & Sync", "uid": "xrpld-system-ledger-sync" } diff --git a/docker/telemetry/grafana/dashboards/system-network-traffic.json b/docker/telemetry/grafana/dashboards/system-network-traffic.json index d92878d970..90ad876e53 100644 --- a/docker/telemetry/grafana/dashboards/system-network-traffic.json +++ b/docker/telemetry/grafana/dashboards/system-network-traffic.json @@ -2,7 +2,7 @@ "annotations": { "list": [] }, - "description": "Network traffic and peer metrics from beast::insight System Metrics. Requires [insight] server=otel in rippled config.", + "description": "Peer connectivity and overlay bandwidth for this node: peer counts, disconnects, total bytes and messages exchanged, and the transaction/proposal/validation traffic that flows across the peer-to-peer network.", "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, @@ -11,7 +11,7 @@ "panels": [ { "title": "Active Peers", - "description": "Number of active inbound and outbound peer connections. Sourced from Peer_Finder.Active_Inbound_Peers and Peer_Finder.Active_Outbound_Peers gauges (PeerfinderManager.cpp). A healthy mainnet node typically has 10-21 outbound and 0-85 inbound peers depending on configuration.", + "description": "**What:** Number of active inbound and outbound peer connections the node currently holds.\n**How it's computed:** Current value of the inbound and outbound active-peer counts per node.\n**Reading it:** Outbound is what the node dials out; inbound is what others open to it. Both should be stable.\n**Healthy range:** roughly 10-21 outbound and 0-85 inbound on mainnet, depending on config.\n**Watch for:** Outbound dropping toward zero (isolation) or inbound pinned at the limit with churn (connection pressure).\n**Source:** src/xrpld/peerfinder/detail/PeerfinderManager.cpp Logic Stats ctor", "type": "timeseries", "gridPos": { "h": 8, @@ -57,7 +57,7 @@ }, { "title": "Peer Disconnects", - "description": "Cumulative count of peer disconnections. Sourced from the Overlay.Peer_Disconnects gauge (OverlayImpl.h). A rising trend indicates network instability, aggressive peer management, or resource exhaustion causing connection drops.", + "description": "**What:** Cumulative count of peer connections that have dropped.\n**How it's computed:** Running total of disconnect events per node.\n**Reading it:** A flat or slowly rising line is normal; the slope matters more than the absolute value.\n**Healthy range:** workload-dependent; slow, steady growth.\n**Watch for:** Sharp step-ups in the slope (network instability, resource exhaustion, or many peers dropping the node at once).\n**Source:** src/xrpld/overlay/detail/OverlayImpl.h OverlayImpl::Stats ctor", "type": "timeseries", "gridPos": { "h": 8, @@ -96,7 +96,7 @@ }, { "title": "Total Network Bytes", - "description": "Total bytes sent and received across all peer connections. Sourced from the total.Bytes_In and total.Bytes_Out traffic category gauges (OverlayImpl.h). Provides a high-level view of network bandwidth consumption.", + "description": "**What:** Total bytes received and sent across all peer connections.\n**How it's computed:** Current cumulative in/out byte totals per node.\n**Reading it:** Overall bandwidth footprint; in and out usually track network activity together.\n**Healthy range:** workload-dependent.\n**Watch for:** Sudden sustained jumps not matched by ledger or transaction activity (relay storms or a noisy peer).\n**Source:** src/xrpld/overlay/detail/OverlayImpl.cpp OverlayImpl ctor (TrafficGauges)", "type": "timeseries", "gridPos": { "h": 8, @@ -142,7 +142,7 @@ }, { "title": "Total Network Messages", - "description": "Total messages sent and received across all peer connections. Sourced from the total.Messages_In and total.Messages_Out traffic category gauges (OverlayImpl.h). Shows the overall message throughput of the overlay network.", + "description": "**What:** Total messages received and sent across all peer connections.\n**How it's computed:** Current cumulative in/out message totals per node.\n**Reading it:** Overall message throughput of the overlay; complements the byte totals.\n**Healthy range:** workload-dependent.\n**Watch for:** Message count climbing far faster than bytes (many tiny messages, possible flooding).\n**Source:** src/xrpld/overlay/detail/OverlayImpl.cpp OverlayImpl ctor (TrafficGauges)", "type": "timeseries", "gridPos": { "h": 8, @@ -188,7 +188,7 @@ }, { "title": "Transaction Traffic", - "description": "Bytes and messages for transaction-related overlay traffic. Includes the transactions traffic category (OverlayImpl/TrafficCount.h). Spikes indicate high transaction volume on the network or transaction flooding.", + "description": "**What:** Transaction relay messages in and out, plus duplicate transaction messages received.\n**How it's computed:** Current message counts for the transaction and transaction-duplicate categories.\n**Reading it:** In/out rise with network transaction volume; duplicates are transactions the node already had.\n**Healthy range:** workload-dependent; duplicates a modest fraction of inbound.\n**Watch for:** Duplicate inbound approaching or exceeding unique inbound (redundant relay), or a sharp spike suggesting transaction flooding.\n**Source:** src/xrpld/overlay/detail/OverlayImpl.cpp OverlayImpl ctor (TrafficGauges)", "type": "timeseries", "gridPos": { "h": 8, @@ -241,7 +241,7 @@ }, { "title": "Proposal Traffic", - "description": "Messages for consensus proposal overlay traffic. Includes proposals, proposals_untrusted, and proposals_duplicate categories (TrafficCount.h). High untrusted or duplicate counts may indicate UNL misconfiguration or network spam.", + "description": "**What:** Consensus proposal messages in/out, plus untrusted and duplicate proposal messages received.\n**How it's computed:** Current message counts for the proposal, proposal-untrusted, and proposal-duplicate categories.\n**Reading it:** Trusted in/out track consensus rounds; untrusted come from validators not on this node's trusted list.\n**Healthy range:** workload-dependent; untrusted and duplicates low relative to trusted.\n**Watch for:** High untrusted (trusted-list misconfiguration) or high duplicates (inefficient relay or proposal spam).\n**Source:** src/xrpld/overlay/detail/OverlayImpl.cpp OverlayImpl ctor (TrafficGauges)", "type": "timeseries", "gridPos": { "h": 8, @@ -301,7 +301,7 @@ }, { "title": "Validation Traffic", - "description": "Messages for validation overlay traffic. Includes validations, validations_untrusted, and validations_duplicate categories (TrafficCount.h). Monitoring trusted vs untrusted validation traffic helps detect UNL health issues.", + "description": "**What:** Validation messages in/out, plus untrusted and duplicate validation messages received.\n**How it's computed:** Current message counts for the validation, validation-untrusted, and validation-duplicate categories.\n**Reading it:** Trusted validations should arrive steadily each ledger; untrusted come from non-trusted validators.\n**Healthy range:** workload-dependent; untrusted and duplicates low relative to trusted.\n**Watch for:** Rising untrusted or duplicate validations (trusted-list health issues or validation spam).\n**Source:** src/xrpld/overlay/detail/OverlayImpl.cpp OverlayImpl ctor (TrafficGauges)", "type": "timeseries", "gridPos": { "h": 8, @@ -361,7 +361,7 @@ }, { "title": "Overlay Traffic by Category (Bytes In)", - "description": "Top traffic categories by inbound bytes. Includes all 57 overlay traffic categories from TrafficCount.h. Shows which protocol message types consume the most bandwidth. Categories include transactions, proposals, validations, ledger data, getobject, and overlay overhead.", + "description": "**What:** Top overlay traffic categories ranked by inbound bytes, excluding the all-traffic total.\n**How it's computed:** Top categories by latest inbound byte value per node.\n**Reading it:** Shows which message types dominate receive bandwidth right now.\n**Healthy range:** workload-dependent; transactions, proposals, and validations typically lead on a synced node.\n**Watch for:** A fetch or ledger-data category topping the list (sync activity) or an unexpected category dominating.\n**Source:** src/xrpld/overlay/detail/OverlayImpl.cpp OverlayImpl ctor (TrafficGauges)", "type": "bargauge", "gridPos": { "h": 8, @@ -658,7 +658,7 @@ }, { "title": "Duplicate Traffic (Wasted Bandwidth)", - "description": "Rate of duplicate overlay traffic across transaction, proposal, and validation categories. Duplicate messages are messages the node has already seen and discards. High duplicate rates indicate inefficient message routing or network topology issues causing redundant relays.", + "description": "**What:** Throughput of duplicate transaction, proposal, and validation traffic: messages the node had already seen and discarded.\n**How it's computed:** Per-second rate of the duplicate byte counters for each category, in and out.\n**Reading it:** Lower is better; this is bandwidth spent on redundant relays.\n**Healthy range:** workload-dependent; a small fraction of total traffic.\n**Watch for:** Duplicate rate climbing toward the same order as useful traffic (poor relay topology or redundant flooding).\n**Source:** src/xrpld/overlay/detail/OverlayImpl.cpp OverlayImpl ctor (TrafficGauges)", "type": "timeseries", "gridPos": { "h": 8, @@ -732,7 +732,7 @@ }, { "title": "All Traffic Categories (Detail)", - "description": "Top 15 traffic categories by inbound byte rate, excluding the total aggregate. Provides a detailed timeseries view of which overlay message types are consuming the most bandwidth over time. Complements the bar gauge snapshot view in the Overlay Traffic panel.", + "description": "**What:** The busiest overlay categories by inbound byte rate over time, excluding the all-traffic total.\n**How it's computed:** Per-second inbound byte rate for the top categories, ranked.\n**Reading it:** Time-series companion to the category bar view; shows how the traffic mix shifts over the window.\n**Healthy range:** workload-dependent.\n**Watch for:** A category ramping up and staying high, or the mix suddenly changing (sync, spam, or a misbehaving peer).\n**Source:** src/xrpld/overlay/detail/OverlayImpl.cpp OverlayImpl ctor (TrafficGauges)", "type": "timeseries", "gridPos": { "h": 8, @@ -771,7 +771,7 @@ } ], "schemaVersion": 39, - "tags": ["statsd", "network", "telemetry"], + "tags": ["network"], "templating": { "list": [ { @@ -860,6 +860,6 @@ "from": "now-1h", "to": "now" }, - "title": "Network Traffic (System Metrics)", + "title": "Network Traffic", "uid": "xrpld-system-network" } diff --git a/docker/telemetry/grafana/dashboards/system-node-health.json b/docker/telemetry/grafana/dashboards/system-node-health.json index 80e8dcab1b..ea303ea136 100644 --- a/docker/telemetry/grafana/dashboards/system-node-health.json +++ b/docker/telemetry/grafana/dashboards/system-node-health.json @@ -2,7 +2,7 @@ "annotations": { "list": [] }, - "description": "Node health metrics from beast::insight System Metrics. Requires [insight] server=otel in rippled config.", + "description": "Overall health of this node: how current its ledgers are, how much time it spends fully synced, I/O and job-queue responsiveness, ledger-fetch and history-integrity signals, and job execution and wait timings.", "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, @@ -11,7 +11,7 @@ "panels": [ { "title": "Validated Ledger Age", - "description": "Age of the most recently validated ledger in seconds. Sourced from the LedgerMaster.Validated_Ledger_Age gauge (LedgerMaster.h) which is updated every collection interval via the insight hook. Values above 20s indicate the node is falling behind the network.", + "description": "**What:** Seconds since the most recently validated ledger, i.e. how far behind the network the node is.\n**How it's computed:** Current value of the validated-ledger-age gauge per node.\n**Reading it:** Lower is better; a healthy node stays within a few ledger-close intervals.\n**Healthy range:** under ~10s (the network closes a ledger every 3-5s).\n**Watch for:** Above 20s or climbing steadily (the node is falling behind or has lost sync).\n**Source:** src/xrpld/app/ledger/LedgerMaster.h LedgerMaster::Stats ctor", "type": "stat", "gridPos": { "h": 8, @@ -59,7 +59,7 @@ }, { "title": "Published Ledger Age", - "description": "Age of the most recently published ledger in seconds. Sourced from the LedgerMaster.Published_Ledger_Age gauge (LedgerMaster.h). Published ledger age should track close to validated ledger age. A growing gap indicates publish pipeline backlog.", + "description": "**What:** Seconds since the most recently published ledger (the ledger exposed to clients and subscribers).\n**How it's computed:** Current value of the published-ledger-age gauge per node.\n**Reading it:** Should track validated-ledger age closely.\n**Healthy range:** under ~10s.\n**Watch for:** Published age growing while validated age stays low (publish-pipeline backlog).\n**Source:** src/xrpld/app/ledger/LedgerMaster.h LedgerMaster::Stats ctor", "type": "stat", "gridPos": { "h": 8, @@ -107,7 +107,7 @@ }, { "title": "Operating Mode Duration", - "description": "Cumulative time spent in each operating mode (Disconnected, Connected, Syncing, Tracking, Full). Sourced from State_Accounting.*_duration gauges (NetworkOPs.cpp) which report microseconds. A healthy node should spend the vast majority of time in Full mode.", + "description": "**What:** Cumulative time the node has spent in each operating mode: Disconnected, Connected, Syncing, Tracking, Full.\n**How it's computed:** Per-mode accumulated time (microseconds) at its latest value per node.\n**Reading it:** Full should dominate and keep growing; other modes should be nearly flat.\n**Healthy range:** almost all time in Full on a healthy node.\n**Watch for:** Growth in Syncing, Connected, or Disconnected (instability or repeated resync).\n**Source:** src/xrpld/app/misc/NetworkOPs.cpp NetworkOPsImp::Stats ctor", "type": "timeseries", "gridPos": { "h": 8, @@ -160,7 +160,7 @@ ], "fieldConfig": { "defaults": { - "unit": "\u00b5s", + "unit": "µs", "custom": { "axisLabel": "Duration", "spanNulls": true, @@ -174,7 +174,7 @@ }, { "title": "Operating Mode Transitions", - "description": "Count of transitions into each operating mode. Sourced from State_Accounting.*_transitions gauges (NetworkOPs.cpp). Frequent transitions out of Full mode indicate instability. Transitions to Disconnected or Syncing warrant investigation.", + "description": "**What:** Count of transitions into each operating mode.\n**How it's computed:** Per-mode transition counters at their latest value per node.\n**Reading it:** Few transitions is good; a stable node rarely leaves Full.\n**Healthy range:** workload-dependent; low and infrequent transitions.\n**Watch for:** Frequent transitions out of Full, or any into Disconnected/Syncing (flapping).\n**Source:** src/xrpld/app/misc/NetworkOPs.cpp NetworkOPsImp::Stats ctor", "type": "timeseries", "gridPos": { "h": 8, @@ -241,7 +241,7 @@ }, { "title": "I/O Latency", - "description": "P95 and P50 of the I/O service loop latency in milliseconds. Sourced from the ios_latency event (Application.cpp) which measures how long it takes for the io_context to process a timer callback. Values above 10ms are logged; above 500ms trigger warnings. High values indicate thread pool saturation or blocking operations.", + "description": "**What:** P95 and P50 latency of the I/O service loop, i.e. how long a queued timer callback waits to run.\n**How it's computed:** 95th and 50th percentiles over a 5-minute window.\n**Reading it:** Low and flat is good; this reflects event-loop responsiveness.\n**Healthy range:** single-digit milliseconds; over ~10ms is notable, over ~500ms is bad.\n**Watch for:** Rising P95 (thread-pool saturation or blocking work on the I/O thread).\n**Source:** src/xrpld/app/main/Application.cpp ApplicationImp ctor", "type": "timeseries", "gridPos": { "h": 8, @@ -287,7 +287,7 @@ }, { "title": "Job Queue Depth", - "description": "Current number of jobs waiting in the job queue. Sourced from the job_count gauge (JobQueue.cpp). A sustained high value indicates the node cannot process work fast enough \u2014 common during ledger replay or heavy RPC load.", + "description": "**What:** Number of jobs currently waiting in the job queue.\n**How it's computed:** Current value of the job-count gauge per node.\n**Reading it:** Near zero when the node keeps up; brief spikes during heavy work are normal.\n**Healthy range:** low, returning to baseline quickly.\n**Watch for:** Sustained high depth (node cannot process work fast enough: replay, heavy RPC, or overload).\n**Source:** src/libxrpl/core/detail/JobQueue.cpp JobQueue ctor", "type": "timeseries", "gridPos": { "h": 8, @@ -326,7 +326,7 @@ }, { "title": "Ledger Fetch Rate", - "description": "Rate of ledger fetch requests initiated by the node. Sourced from the ledger_fetches counter (InboundLedgers.cpp) which increments each time the node requests a ledger from a peer. High rates indicate the node is catching up or missing ledgers.", + "description": "**What:** Rate at which the node requests ledgers from peers.\n**How it's computed:** Per-second rate of the ledger-fetches counter over 5 minutes.\n**Reading it:** Near zero when synced; rises when catching up or backfilling history.\n**Healthy range:** workload-dependent; low on a synced node.\n**Watch for:** Sustained high fetch rate on a node that should be current (repeatedly missing ledgers).\n**Source:** src/xrpld/app/ledger/detail/InboundLedgers.cpp InboundLedgersImp ctor", "type": "stat", "gridPos": { "h": 8, @@ -358,7 +358,7 @@ }, { "title": "Ledger History Mismatches", - "description": "Rate of ledger history hash mismatches. Sourced from the ledger.history.mismatch counter (LedgerHistory.cpp) which increments when a built ledger hash does not match the expected validated hash. Non-zero values indicate consensus divergence or database corruption.", + "description": "**What:** Rate of ledger-history hash mismatches: built ledgers whose hash disagrees with the validated hash.\n**How it's computed:** Per-second rate of the history-mismatch counter over 5 minutes.\n**Reading it:** Should be flat at zero.\n**Healthy range:** zero.\n**Watch for:** Any non-zero value (consensus divergence, corrupted history, or database issues).\n**Source:** src/xrpld/telemetry/MetricsRegistry.cpp incrementLedgerHistoryMismatch (caller LedgerHistory.cpp)", "type": "stat", "gridPos": { "h": 8, @@ -414,7 +414,7 @@ }, { "title": "Key Jobs Execution Time", - "description": "Execution time for critical job types at the selected quantile. Sourced from per-job-type events in JobTypeData (JobTypeData.h). Shows how long key consensus, transaction, and maintenance jobs take to execute. Spikes indicate processing bottlenecks.", + "description": "**What:** Execution time of the most important job types (accept/advance ledger, transaction, write objects, heartbeat, sweep, trusted validation/proposal, publish, client RPC, ledger data) at the selected quantile.\n**How it's computed:** Selected quantile of each job's execution-time histogram over 5 minutes.\n**Reading it:** Lower and stable is better; shows where consensus, transaction, and maintenance time goes.\n**Healthy range:** workload-dependent; most jobs in low tens of milliseconds.\n**Watch for:** One job type spiking (a specific bottleneck) or broad increases (overload).\n**Source:** include/xrpl/core/JobTypeData.h JobTypeData ctor", "type": "timeseries", "gridPos": { "h": 8, @@ -523,7 +523,7 @@ }, { "title": "Key Jobs Dequeue Wait Time", - "description": "Time spent waiting in the job queue before execution for critical job types. Sourced from per-job-type dequeue events (JobTypeData.h). High dequeue times indicate the job queue is backlogged and jobs are waiting too long to be scheduled.", + "description": "**What:** Time key jobs wait in the queue before they start running, at the selected quantile.\n**How it's computed:** Selected quantile of each job's queue-wait histogram over 5 minutes.\n**Reading it:** Low wait means the scheduler keeps up; high wait means backlog.\n**Healthy range:** workload-dependent; short waits.\n**Watch for:** Rising wait across job types (queue congestion delaying consensus and transaction work).\n**Source:** include/xrpl/core/JobTypeData.h JobTypeData ctor", "type": "timeseries", "gridPos": { "h": 8, @@ -632,7 +632,7 @@ }, { "title": "FullBelowCache Size", - "description": "Number of entries in the FullBelowCache. Sourced from the TaggedCache size gauge (TaggedCache.h) for the Node family full below cache (NodeFamily.cpp). This cache tracks which SHAMap nodes have all children present locally, avoiding redundant fetches during ledger acquisition.", + "description": "**What:** Number of entries in the FullBelowCache, which tracks tree nodes known to have all children present locally.\n**How it's computed:** Current value of the cache-size gauge per node.\n**Reading it:** Grows as the node learns complete subtrees; helps avoid redundant fetches.\n**Healthy range:** workload-dependent; stable once synced.\n**Watch for:** Collapsing to near zero repeatedly (cache thrash) alongside rising fetch traffic.\n**Source:** include/xrpl/basics/TaggedCache.h TaggedCache::Stats ctor", "type": "timeseries", "gridPos": { "h": 8, @@ -671,7 +671,7 @@ }, { "title": "FullBelowCache Hit Rate", - "description": "Hit rate percentage for the FullBelowCache. Sourced from the TaggedCache hit_rate gauge (TaggedCache.h). A high hit rate means the node is efficiently reusing cached knowledge about complete SHAMap subtrees. Low hit rates during steady state warrant investigation.", + "description": "**What:** Hit-rate percentage of the FullBelowCache.\n**How it's computed:** Current value of the cache hit-rate gauge (0-100%).\n**Reading it:** Higher is better; means the node reuses knowledge of complete subtrees.\n**Healthy range:** above ~50% in steady state.\n**Watch for:** Low hit rate during steady state (cache too small or constant re-acquisition).\n**Source:** include/xrpl/basics/TaggedCache.h TaggedCache::Stats ctor", "type": "gauge", "gridPos": { "h": 8, @@ -721,7 +721,7 @@ }, { "title": "Ledger Publish Gap", - "description": "Difference between published and validated ledger ages. Computed as Published_Ledger_Age minus Validated_Ledger_Age. A value near zero means the publish pipeline keeps up with validation. A growing gap indicates the publish pipeline is falling behind, potentially causing stale data for subscribers.", + "description": "**What:** Difference between published and validated ledger ages: how far the publish pipeline trails validation.\n**How it's computed:** Published-ledger age minus validated-ledger age, in seconds.\n**Reading it:** Near zero means publishing keeps up with validation.\n**Healthy range:** within a few seconds of zero.\n**Watch for:** A growing gap (publish backlog, stale data for subscribers).\n**Source:** src/xrpld/app/ledger/LedgerMaster.h LedgerMaster::Stats ctor", "type": "stat", "gridPos": { "h": 8, @@ -769,7 +769,7 @@ }, { "title": "State Duration Rate (Full vs Tracking)", - "description": "Rate of change of time spent in Full and Tracking operating modes, normalized to seconds. Sourced from State_Accounting duration gauges (NetworkOPs.cpp). In steady state the Full duration rate should be close to 1.0 (gaining one second of Full-mode time per wall-clock second). A drop below 1.0 means the node is spending time in other modes.", + "description": "**What:** How fast the node accrues time in Full versus Tracking mode, normalized to seconds per second.\n**How it's computed:** Per-second rate of the Full and Tracking accumulated-time gauges, scaled to seconds.\n**Reading it:** Full rate near 1.0 means the node is in Full essentially all the time.\n**Healthy range:** Full rate ~1.0, Tracking near 0.\n**Watch for:** Full rate dropping below 1.0 with Tracking rising (time being lost to non-Full modes).\n**Source:** src/xrpld/app/misc/NetworkOPs.cpp NetworkOPsImp::Stats ctor", "type": "timeseries", "gridPos": { "h": 8, @@ -815,7 +815,7 @@ }, { "title": "All Jobs Execution Time (Detail)", - "description": "Execution time for ALL non-special job types at the selected quantile. Shows the complete picture of job execution performance. Use the Key Jobs panel for a focused view of the most critical jobs.", + "description": "**What:** Execution time for all non-special job types at the selected quantile.\n**How it's computed:** Selected quantile of each job's execution-time histogram over 5 minutes.\n**Reading it:** Full breakdown of job performance; use the Key Jobs panel for the focused view.\n**Healthy range:** workload-dependent.\n**Watch for:** Any job type with a persistent upward trend, or many rising together (systemic slowdown).\n**Source:** include/xrpl/core/JobTypeData.h JobTypeData ctor", "type": "timeseries", "gridPos": { "h": 8, @@ -854,7 +854,7 @@ }, { "title": "All Jobs Dequeue Wait (Detail)", - "description": "Dequeue wait time for ALL non-special job types at the selected quantile. Shows the complete picture of job queue waiting times. High wait times across many job types indicate systemic job queue congestion.", + "description": "**What:** Queue wait time before execution for all non-special job types at the selected quantile.\n**How it's computed:** Selected quantile of each job's queue-wait histogram over 5 minutes.\n**Reading it:** Complete picture of scheduling delay across job types.\n**Healthy range:** workload-dependent; short waits.\n**Watch for:** High waits across many job types (systemic job-queue congestion).\n**Source:** include/xrpl/core/JobTypeData.h JobTypeData ctor", "type": "timeseries", "gridPos": { "h": 8, @@ -893,7 +893,7 @@ } ], "schemaVersion": 39, - "tags": ["statsd", "node-health", "telemetry"], + "tags": ["node", "health"], "templating": { "list": [ { @@ -1016,6 +1016,6 @@ "from": "now-1h", "to": "now" }, - "title": "Node Health (System Metrics)", + "title": "Node Health", "uid": "xrpld-system-node-health" } diff --git a/docker/telemetry/grafana/dashboards/system-overlay-traffic-detail.json b/docker/telemetry/grafana/dashboards/system-overlay-traffic-detail.json index 610e850b7f..7a1a47780f 100644 --- a/docker/telemetry/grafana/dashboards/system-overlay-traffic-detail.json +++ b/docker/telemetry/grafana/dashboards/system-overlay-traffic-detail.json @@ -2,7 +2,7 @@ "annotations": { "list": [] }, - "description": "Detailed overlay traffic breakdown for categories not covered by the main Network Traffic dashboard. Includes squelch, overhead, validator lists, object fetch, ledger sync, and protocol negotiation traffic. Requires [insight] server=otel in rippled config.", + "description": "Fine-grained breakdown of peer-to-peer overlay traffic beyond the main network view: squelch relay control, protocol overhead, validator-list distribution, transaction-set exchange, transaction availability, ledger-proof and replay traffic, and unclassified messages.", "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, @@ -11,7 +11,7 @@ "panels": [ { "title": "Squelch Traffic (Messages)", - "description": "Squelch-related overlay messages. Squelch is the peer traffic management protocol that suppresses redundant message forwarding. 'squelch' = squelch control messages, 'squelch_suppressed' = messages suppressed by squelch, 'squelch_ignored' = squelch directives that were ignored. High suppressed counts indicate effective bandwidth savings; high ignored counts may indicate misconfigured peers. Sourced from TrafficCount.h squelch categories.", + "description": "**What:** Squelch relay-control messages in/out, plus messages suppressed by squelch and squelch directives that were ignored. Squelch reduces redundant message forwarding between peers.\n**How it's computed:** Current message counts for the squelch, squelch-suppressed, and squelch-ignored categories, in and out.\n**Reading it:** High suppressed counts mean squelch is saving bandwidth; ignored should stay low.\n**Healthy range:** workload-dependent; suppressed far above ignored.\n**Watch for:** High ignored counts (peers not honoring squelch) or squelch traffic itself dominating.\n**Source:** src/xrpld/overlay/detail/OverlayImpl.cpp OverlayImpl ctor (TrafficGauges)", "type": "timeseries", "gridPos": { "h": 8, @@ -85,7 +85,7 @@ }, { "title": "Overhead Traffic Breakdown (Bytes)", - "description": "Overlay protocol overhead by sub-category. 'overhead' = base protocol overhead (ping, status, etc.), 'overhead_cluster' = intra-cluster communication overhead, 'overhead_manifest' = validator manifest distribution overhead. High cluster overhead may indicate frequent cluster state syncs; high manifest overhead occurs during UNL changes. Sourced from TrafficCount.h overhead categories.", + "description": "**What:** Overlay protocol overhead bytes split into base overhead, intra-cluster overhead, and validator-manifest distribution overhead.\n**How it's computed:** Current in/out byte counts for the overhead, overhead-cluster, and overhead-manifest categories.\n**Reading it:** Base overhead is routine; cluster and manifest rise around cluster syncs and manifest changes.\n**Healthy range:** workload-dependent; low and stable.\n**Watch for:** Sustained high cluster or manifest overhead (frequent cluster state churn or manifest reissue).\n**Source:** src/xrpld/overlay/detail/OverlayImpl.cpp OverlayImpl ctor (TrafficGauges)", "type": "timeseries", "gridPos": { "h": 8, @@ -159,7 +159,7 @@ }, { "title": "Validator List Traffic", - "description": "Validator list (UNL) distribution traffic. Validator lists are exchanged when peers share their trusted validator configurations. Spikes occur during UNL updates or when new peers connect. Sourced from TrafficCount.h validator_lists category.", + "description": "**What:** Bytes and messages exchanged distributing validator lists (trusted-list configuration) between peers.\n**How it's computed:** Current in/out byte and message counts for the validator-lists category.\n**Reading it:** Bursts when lists update or new peers connect; quiet otherwise.\n**Healthy range:** workload-dependent; occasional bursts.\n**Watch for:** Continuous high volume (repeated list re-fetching or churn).\n**Source:** src/xrpld/overlay/detail/OverlayImpl.cpp OverlayImpl ctor (TrafficGauges)", "type": "timeseries", "gridPos": { "h": 8, @@ -236,7 +236,7 @@ }, { "title": "Set Get/Share Traffic (Bytes)", - "description": "Transaction set get and share traffic. 'set_get' = requests to fetch transaction sets (sent during ledger close), 'set_share' = responses sharing transaction sets. High set_get traffic indicates peers frequently requesting missing transaction sets, which may signal sync delays. Sourced from TrafficCount.h set_get/set_share categories.", + "description": "**What:** Transaction-set fetch (get) and share bytes exchanged during ledger close.\n**How it's computed:** Current in/out byte counts for the set-get and set-share categories.\n**Reading it:** Some exchange each ledger is normal as peers reconcile transaction sets.\n**Healthy range:** workload-dependent.\n**Watch for:** High set-get (peers frequently missing transaction sets: possible sync delays).\n**Source:** src/xrpld/overlay/detail/OverlayImpl.cpp OverlayImpl ctor (TrafficGauges)", "type": "timeseries", "gridPos": { "h": 8, @@ -296,7 +296,7 @@ }, { "title": "Have/Requested Transactions (Messages)", - "description": "Transaction availability protocol messages. 'have_transactions' = advertisements that a peer has specific transactions available, 'requested_transactions' = explicit requests for transaction data. A high ratio of requested to have may indicate peers are behind on transaction propagation. Sourced from TrafficCount.h have_transactions/requested_transactions categories.", + "description": "**What:** Transaction-availability messages: advertisements that a peer has certain transactions, and explicit requests for transaction data.\n**How it's computed:** Current in/out message counts for the have-transactions and requested-transactions categories.\n**Reading it:** Compare requested versus have to gauge how well transactions are propagating.\n**Healthy range:** workload-dependent.\n**Watch for:** Requested far exceeding have (peers behind on transaction propagation).\n**Source:** src/xrpld/overlay/detail/OverlayImpl.cpp OverlayImpl ctor (TrafficGauges)", "type": "timeseries", "gridPos": { "h": 8, @@ -356,7 +356,7 @@ }, { "title": "Unknown / Unclassified Traffic", - "description": "Traffic that does not match any known overlay message category. Non-zero values may indicate protocol version mismatches, corrupted messages, or new message types not yet classified. Sourced from TrafficCount.h unknown category.", + "description": "**What:** Overlay traffic that matches no known message category, in bytes and messages.\n**How it's computed:** Current in/out byte and message counts for the unknown category.\n**Reading it:** Should be at or near zero.\n**Healthy range:** zero.\n**Watch for:** Any sustained non-zero value (protocol version mismatch, corrupted messages, or an unclassified new message type).\n**Source:** src/xrpld/overlay/detail/OverlayImpl.cpp OverlayImpl ctor (TrafficGauges)", "type": "timeseries", "gridPos": { "h": 8, @@ -433,7 +433,7 @@ }, { "title": "Proof Path Traffic", - "description": "Proof path request/response traffic for ledger state proof exchange. Used by peers to verify specific ledger entries without downloading the full ledger. High request volume may indicate peers validating state during catch-up. Sourced from TrafficCount.h proof_path_request/proof_path_response categories.", + "description": "**What:** Proof-path request/response bytes used to verify individual ledger entries without downloading the whole ledger.\n**How it's computed:** Current in/out byte counts for the proof-path request and response categories.\n**Reading it:** Rises when peers verify specific state, often during catch-up.\n**Healthy range:** workload-dependent.\n**Watch for:** High sustained request volume (heavy state-verification load).\n**Source:** src/xrpld/overlay/detail/OverlayImpl.cpp OverlayImpl ctor (TrafficGauges)", "type": "timeseries", "gridPos": { "h": 8, @@ -493,7 +493,7 @@ }, { "title": "Replay Delta Traffic", - "description": "Replay delta request/response traffic for ledger replay protocol. Used during catch-up to efficiently replay ledger state changes. Sourced from TrafficCount.h replay_delta_request/replay_delta_response categories.", + "description": "**What:** Replay-delta request/response bytes used to efficiently replay ledger state changes during catch-up.\n**How it's computed:** Current in/out byte counts for the replay-delta request and response categories.\n**Reading it:** Active during catch-up and replay; quiet when synced.\n**Healthy range:** workload-dependent; low when synced.\n**Watch for:** Continuous replay traffic (node repeatedly replaying rather than staying current).\n**Source:** src/xrpld/overlay/detail/OverlayImpl.cpp OverlayImpl ctor (TrafficGauges)", "type": "timeseries", "gridPos": { "h": 8, @@ -553,7 +553,7 @@ } ], "schemaVersion": 39, - "tags": ["statsd", "overlay", "network", "telemetry"], + "tags": ["network", "peer"], "templating": { "list": [ { @@ -642,6 +642,6 @@ "from": "now-1h", "to": "now" }, - "title": "Overlay Traffic Detail (System Metrics)", + "title": "Overlay Traffic Detail", "uid": "xrpld-system-overlay-detail" } diff --git a/docker/telemetry/grafana/dashboards/system-rpc-pathfinding.json b/docker/telemetry/grafana/dashboards/system-rpc-pathfinding.json index 9f84237e65..852a560519 100644 --- a/docker/telemetry/grafana/dashboards/system-rpc-pathfinding.json +++ b/docker/telemetry/grafana/dashboards/system-rpc-pathfinding.json @@ -2,7 +2,7 @@ "annotations": { "list": [] }, - "description": "RPC and pathfinding metrics from beast::insight System Metrics. Requires [insight] server=otel in rippled config.", + "description": "Client-facing RPC behavior and payment pathfinding cost on this node: request rates, response times and sizes, resource-limit warnings and drops, gRPC read-path load, and pathfinding request, compute, and discovery activity.", "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, @@ -10,8 +10,8 @@ "links": [], "panels": [ { - "title": "RPC Request Rate (System Metrics)", - "description": "Rate of RPC requests as counted by the beast::insight counter. Sourced from rpc.requests (ServerHandler.cpp) which increments on every HTTP and WebSocket RPC request. Compare with the span-based rpc.request rate in the RPC Performance dashboard for cross-validation.", + "title": "RPC Request Rate", + "description": "**What:** Rate of RPC requests handled, counting every HTTP and WebSocket call.\n**How it's computed:** Per-second rate of the RPC-requests counter over 5 minutes.\n**Reading it:** Tracks client demand on the node.\n**Healthy range:** workload-dependent.\n**Watch for:** Sudden sustained spikes (client surge or abusive polling) or a drop to zero (endpoint unavailable).\n**Source:** src/xrpld/rpc/detail/ServerHandler.cpp ServerHandler ctor", "type": "stat", "gridPos": { "h": 8, @@ -42,8 +42,8 @@ } }, { - "title": "RPC Response Time (System Metrics)", - "description": "P95 and P50 of RPC response time from the beast::insight timer. Sourced from the rpc.time event (ServerHandler.cpp) which records elapsed milliseconds for each RPC response. This measures the full HTTP handler time, not just command execution. Compare with span-based rpc.request duration.", + "title": "RPC Response Time", + "description": "**What:** P95 and P50 of end-to-end RPC handler time (full HTTP handling, not just command execution).\n**How it's computed:** 95th and 50th percentiles over a 5-minute window.\n**Reading it:** Lower is better; P95 shows the slow tail.\n**Healthy range:** workload-dependent; most methods well under a second.\n**Watch for:** Rising P95 while request rate is flat (expensive queries or contention).\n**Source:** src/xrpld/rpc/detail/ServerHandler.cpp ServerHandler ctor", "type": "timeseries", "gridPos": { "h": 8, @@ -89,7 +89,7 @@ }, { "title": "RPC Response Size", - "description": "P95 and P50 of RPC response payload size in bytes. Sourced from the rpc.size event (ServerHandler.cpp) which records the byte length of each RPC JSON response. Large responses may indicate expensive queries (e.g. account_tx with many results) or API misuse.", + "description": "**What:** P95 and P50 of RPC response payload size in bytes.\n**How it's computed:** 95th and 50th percentiles over a 5-minute window.\n**Reading it:** Larger responses cost more bandwidth and CPU to build.\n**Healthy range:** workload-dependent.\n**Watch for:** Large P95 (result-heavy queries such as broad account_tx, or API misuse).\n**Source:** src/xrpld/rpc/detail/ServerHandler.cpp ServerHandler ctor", "type": "timeseries", "gridPos": { "h": 8, @@ -135,7 +135,7 @@ }, { "title": "RPC Response Time Distribution", - "description": "Distribution of RPC response times from the beast::insight timer showing P50, P90, P95, and P99 quantiles. Sourced from the rpc.time event (ServerHandler.cpp). Useful for detecting bimodal latency or long-tail requests.", + "description": "**What:** RPC response-time spread shown as P50, P90, P95, and P99.\n**How it's computed:** Four quantiles of response time over a 5-minute window.\n**Reading it:** A wide gap between P50 and P99 signals a long latency tail.\n**Healthy range:** workload-dependent; quantiles clustered together.\n**Watch for:** P99 pulling far above P50 (bimodal latency or occasional very slow requests).\n**Source:** src/xrpld/rpc/detail/ServerHandler.cpp ServerHandler ctor", "type": "timeseries", "gridPos": { "h": 8, @@ -195,7 +195,7 @@ }, { "title": "Pathfinding Fast Duration", - "description": "P95 and P50 of fast pathfinding execution time. Sourced from the pathfind_fast event (PathRequests.h) which records the duration of the fast pathfinding algorithm. Fast pathfinding uses a simplified search that trades accuracy for speed.", + "description": "**What:** P95 and P50 execution time of the fast pathfinding search (simplified, favoring speed over completeness).\n**How it's computed:** 95th and 50th percentiles over a 5-minute window.\n**Reading it:** Lower is better; fast pathfinding should stay quick.\n**Healthy range:** workload-dependent; typically well below full pathfinding.\n**Watch for:** Rising fast-path latency (pathfinding load or complex order books).\n**Source:** src/xrpld/rpc/detail/PathRequestManager.h ctor", "type": "timeseries", "gridPos": { "h": 8, @@ -241,7 +241,7 @@ }, { "title": "Pathfinding Full Duration", - "description": "P95 and P50 of full pathfinding execution time. Sourced from the pathfind_full event (PathRequests.h) which records the duration of the exhaustive pathfinding search. Full pathfinding is more expensive and can take significantly longer than fast mode.", + "description": "**What:** P95 and P50 execution time of the full, exhaustive pathfinding search.\n**How it's computed:** 95th and 50th percentiles over a 5-minute window.\n**Reading it:** Full pathfinding is heavier and slower than fast mode.\n**Healthy range:** workload-dependent.\n**Watch for:** High or rising full-path latency (expensive path computation under subscription load).\n**Source:** src/xrpld/rpc/detail/PathRequestManager.h ctor", "type": "timeseries", "gridPos": { "h": 8, @@ -287,7 +287,7 @@ }, { "title": "Resource Warnings Rate", - "description": "Rate of resource warning events from the Resource Manager. Sourced from the warn meter (Logic.h) which increments when a consumer (peer or RPC client) exceeds the warning threshold for resource usage. A rising rate indicates aggressive clients that may need throttling. NOTE: This panel will show no data until the |m -> |c fix is applied in System MetricsCollector.cpp (Phase 6 Task 6.1).", + "description": "**What:** Rate of resource-limit warnings raised when a peer or client exceeds its usage warning threshold.\n**How it's computed:** Per-second rate of the warn counter over 5 minutes.\n**Reading it:** Occasional warnings are normal under load; a rising rate flags aggressive clients.\n**Healthy range:** workload-dependent; low.\n**Watch for:** A climbing warning rate (clients approaching limits, a precursor to drops).\n**Source:** include/xrpl/resource/detail/Logic.h Logic::Stats ctor", "type": "stat", "gridPos": { "h": 8, @@ -335,7 +335,7 @@ }, { "title": "Resource Drops Rate", - "description": "Rate of resource drop events from the Resource Manager. Sourced from the drop meter (Logic.h) which increments when a consumer is disconnected or blocked due to excessive resource usage. Non-zero values mean the node is actively rejecting abusive connections. NOTE: This panel will show no data until the |m -> |c fix is applied in System MetricsCollector.cpp (Phase 6 Task 6.1).", + "description": "**What:** Rate of resource drops: consumers disconnected or blocked for excessive usage.\n**How it's computed:** Per-second rate of the drop counter over 5 minutes.\n**Reading it:** Non-zero means the node is actively rejecting abusive connections.\n**Healthy range:** at or near zero.\n**Watch for:** Sustained non-zero drops (ongoing abuse or a misbehaving client/peer being throttled).\n**Source:** include/xrpl/resource/detail/Logic.h Logic::Stats ctor", "type": "stat", "gridPos": { "h": 8, @@ -383,7 +383,7 @@ }, { "title": "gRPC Request Rate by Method (Spans)", - "description": "Per-method gRPC call rate derived from the grpc.{Method} spans (GRPCServer.cpp). Covers the gRPC API used by reporting/Clio. Populated only when the node serves gRPC traffic.", + "description": "**What:** Per-method gRPC call rate for the gRPC read API used by reporting and Clio.\n**How it's computed:** Per-second count of gRPC calls grouped by method.\n**Reading it:** Shows which gRPC methods are called and how often; only populated when the node serves gRPC.\n**Healthy range:** workload-dependent.\n**Watch for:** A single method spiking (heavy reporting load) or unexpected methods appearing.\n**Source:** src/xrpld/rpc/GRPCServer.cpp (gRPC method handlers)", "type": "timeseries", "gridPos": { "h": 8, @@ -422,7 +422,7 @@ }, { "title": "gRPC Latency P95 by Method (Spans)", - "description": "p95 latency per gRPC method from grpc.{Method} span durations. Identifies slow gRPC read paths.", + "description": "**What:** P95 latency per gRPC method.\n**How it's computed:** 95th percentile of gRPC call duration per method over 5 minutes.\n**Reading it:** Identifies slow gRPC read paths.\n**Healthy range:** workload-dependent.\n**Watch for:** One method's P95 rising (expensive ledger reads or backend pressure).\n**Source:** src/xrpld/rpc/GRPCServer.cpp (gRPC method handlers)", "type": "timeseries", "gridPos": { "h": 8, @@ -461,7 +461,7 @@ }, { "title": "gRPC Error Rate by Status (Spans)", - "description": "Rate of gRPC spans broken down by grpc_status (success/error/resource_exhausted/failed_precondition). A rising error or resource_exhausted rate indicates gRPC clients hitting limits.", + "description": "**What:** Rate of gRPC calls broken down by result status: success, error, resource-exhausted, failed-precondition.\n**How it's computed:** Per-second count of gRPC calls grouped by status.\n**Reading it:** Errors and resource-exhausted should be a small share.\n**Healthy range:** workload-dependent; mostly success.\n**Watch for:** Rising error or resource-exhausted rate (clients hitting limits or failing reads).\n**Source:** src/xrpld/rpc/GRPCServer.cpp (gRPC method handlers)", "type": "timeseries", "gridPos": { "h": 8, @@ -500,7 +500,7 @@ }, { "title": "Pathfinding Compute Duration (Spans)", - "description": "p95/p50 of the pathfind.compute span, the per-request path computation. Complements the StatsD pathfind_fast/full timers with span-level visibility. Populated under pathfinding (book/path) RPC load.", + "description": "**What:** P95 and P50 of per-request path computation time.\n**How it's computed:** 95th and 50th percentiles of path-computation duration over 5 minutes.\n**Reading it:** Complements the fast/full timers with per-request visibility; populated under book/path RPC load.\n**Healthy range:** workload-dependent.\n**Watch for:** Rising compute time (complex paths or heavy pathfinding demand).\n**Source:** src/xrpld/rpc/detail/PathRequest.cpp PathRequest::doUpdate", "type": "timeseries", "gridPos": { "h": 8, @@ -546,7 +546,7 @@ }, { "title": "Pathfinding Request & Discovery Rate (Spans)", - "description": "Rate of pathfind.request (client path requests) and pathfind.discover (path-discovery passes) spans. Shows pathfinding demand and the discovery cost driver for subscription-heavy nodes.", + "description": "**What:** Rate of client path requests and of path-discovery passes.\n**How it's computed:** Per-second count of path-request and path-discovery operations.\n**Reading it:** Shows pathfinding demand and the discovery cost driver for subscription-heavy nodes.\n**Healthy range:** workload-dependent.\n**Watch for:** Discovery rate climbing (subscription load driving repeated path discovery).\n**Source:** src/xrpld/rpc/handlers/orderbook/PathFind.cpp doPathFind; src/xrpld/rpc/detail/PathRequest.cpp PathRequest::findPaths", "type": "timeseries", "gridPos": { "h": 8, @@ -592,7 +592,7 @@ } ], "schemaVersion": 39, - "tags": ["statsd", "rpc", "pathfinding", "telemetry"], + "tags": ["rpc", "pathfinding"], "templating": { "list": [ { @@ -701,6 +701,6 @@ "from": "now-1h", "to": "now" }, - "title": "RPC & Pathfinding (System Metrics)", + "title": "RPC & Pathfinding", "uid": "xrpld-system-rpc" } From 50377df2710dadb3d4f62b59a8157467eeb55293 Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Fri, 3 Jul 2026 21:56:08 +0100 Subject: [PATCH 3/3] docs(telemetry): rewrite dashboard descriptions and tags to describe data not pipeline Co-Authored-By: Claude Opus 4.8 --- .../grafana/dashboards/consensus-health.json | 50 +++++++++---------- .../grafana/dashboards/ledger-operations.json | 22 ++++---- .../grafana/dashboards/peer-network.json | 14 +++--- .../grafana/dashboards/rpc-performance.json | 27 +++++----- .../dashboards/transaction-overview.json | 46 ++++++++--------- 5 files changed, 80 insertions(+), 79 deletions(-) diff --git a/docker/telemetry/grafana/dashboards/consensus-health.json b/docker/telemetry/grafana/dashboards/consensus-health.json index eb7b180a6b..eda035e7ab 100644 --- a/docker/telemetry/grafana/dashboards/consensus-health.json +++ b/docker/telemetry/grafana/dashboards/consensus-health.json @@ -10,7 +10,7 @@ "panels": [ { "title": "Consensus Round Duration", - "description": "p95 and p50 duration of consensus accept rounds. The consensus.accept span (RCLConsensus.cpp) measures the time to process an accepted ledger including transaction application and state finalization. The span carries proposers and round_time_ms attributes. Normal range is 3-6 seconds on mainnet.", + "description": "**What:** Time to process an accepted ledger, from the moment consensus is reached through applying the transaction set and finalising state.\n**How it's computed:** 95th and 50th percentile of per-round durations over a 5-minute window, per node.\n**Reading it:** Flat p50 with a modest p95 gap is normal; both track network transaction load.\n**Healthy range:** Roughly 3-6 seconds on mainnet.\n**Watch for:** A rising p95 that pulls away from p50 means occasional slow rounds; sustained growth precedes ledger-age alarms.\n**Source:** src/xrpld/app/consensus/RCLConsensus.cpp:RCLConsensus::Adaptor::makeAcceptSpan", "type": "timeseries", "gridPos": { "h": 8, @@ -56,7 +56,7 @@ }, { "title": "Consensus Proposals Sent Rate", - "description": "Rate at which this node sends consensus proposals to the network. Sourced from the consensus.proposal.send span (RCLConsensus.cpp) which fires each time the node proposes a transaction set. The span carries xrpl.consensus.round identifying the consensus round number. A healthy proposing node should show steady proposal output.", + "description": "**What:** How often this node broadcasts its own consensus proposal (its candidate transaction set) to the network.\n**How it's computed:** Per-second rate over a 5-minute window, per node.\n**Reading it:** A proposing validator emits a steady stream, roughly one per consensus round; observing-only nodes emit none.\n**Healthy range:** About one proposal per round (~0.2-0.3/s on mainnet) for a proposing node.\n**Watch for:** A validator that drops to zero has stopped proposing and may be unhealthy or misconfigured.\n**Source:** src/xrpld/app/consensus/RCLConsensus.cpp:RCLConsensus::Adaptor::propose", "type": "timeseries", "gridPos": { "h": 8, @@ -95,7 +95,7 @@ }, { "title": "Ledger Close Duration", - "description": "p95 duration of the ledger close event. The consensus.ledger_close span (RCLConsensus.cpp) measures the time from when consensus triggers a ledger close to completion. Carries ledger_seq and consensus_mode attributes. Compare with Consensus Round Duration to understand how close timing relates to overall round time.", + "description": "**What:** Time spent closing a ledger once consensus triggers the close, up to completion.\n**How it's computed:** 95th percentile of per-close durations over a 5-minute window, per node.\n**Reading it:** Should sit at or just below the full round duration; a subset of overall round time.\n**Healthy range:** A few seconds, tracking the ~4s mainnet ledger interval.\n**Watch for:** A widening gap versus round duration points to overhead outside ledger construction.\n**Source:** src/xrpld/app/consensus/RCLConsensus.cpp:RCLConsensus::Adaptor::onClose", "type": "timeseries", "gridPos": { "h": 8, @@ -134,7 +134,7 @@ }, { "title": "Validation Send Rate", - "description": "Rate at which this node sends ledger validations to the network. Sourced from the consensus.validation.send span (RCLConsensus.cpp). Each validation confirms the node has fully validated a ledger. The span carries ledger_seq and proposing. Should closely track the ledger close rate when the node is healthy.", + "description": "**What:** How often this node issues a validation confirming it has fully validated a ledger.\n**How it's computed:** Per-second rate over a 5-minute window, per node.\n**Reading it:** Should closely track the ledger close rate on a healthy validator.\n**Healthy range:** About one per ledger (~0.25/s on mainnet).\n**Watch for:** Validations lagging closes suggests the node is falling behind on validation.\n**Source:** src/xrpld/app/consensus/RCLConsensus.cpp:RCLConsensus::Adaptor::createValidationSpan", "type": "stat", "gridPos": { "h": 8, @@ -166,7 +166,7 @@ }, { "title": "Ledger Apply Duration (doAccept)", - "description": "Time spent applying the consensus result to build a new ledger. Measured by the consensus.accept.apply span in doAccept().", + "description": "**What:** Time spent applying the agreed consensus result to build the new ledger.\n**How it's computed:** 95th and 50th percentile of per-round apply durations over a 5-minute window, per node.\n**Reading it:** Scales with the size of the transaction set being applied.\n**Healthy range:** Tens to a few hundred milliseconds under normal load.\n**Watch for:** Spikes indicate heavy transaction sets or I/O pressure while writing ledger state.\n**Source:** src/xrpld/app/consensus/RCLConsensus.cpp:RCLConsensus::Adaptor::doAccept", "type": "timeseries", "gridPos": { "h": 8, @@ -206,7 +206,7 @@ }, { "title": "Close Time Agreement", - "description": "Rate of close time agreement vs disagreement across consensus rounds. Based on close_time_correct attribute (true = validators agreed, false = agreed to disagree per avCT_CONSENSUS_PCT).", + "description": "**What:** Whether validators agreed on the ledger close time (agreed) or agreed to disagree and used a fallback (disagreed), per round.\n**How it's computed:** Per-second rate of rounds split by agreement outcome over a 5-minute window, per node.\n**Reading it:** Nearly all rounds should show agreement.\n**Healthy range:** Overwhelmingly agreement; occasional disagreement is tolerable.\n**Watch for:** A rising disagreement share signals clock drift or network latency across validators.\n**Source:** src/xrpld/app/consensus/RCLConsensus.cpp:RCLConsensus::Adaptor::doAccept", "type": "timeseries", "gridPos": { "h": 8, @@ -239,7 +239,7 @@ }, { "title": "Consensus Mode Over Time", - "description": "Breakdown of consensus ledger close events by the node's consensus mode (Proposing, Observing, Wrong Ledger, Switched Ledger). Grouped by the consensus_mode span attribute from consensus.ledger_close. A healthy validator should be predominantly in Proposing mode. Frequent Wrong Ledger or Switched Ledger indicates sync issues.", + "description": "**What:** Ledger close events split by the node's consensus mode: Proposing, Observing, Wrong Ledger, or Switched Ledger.\n**How it's computed:** Per-second rate per mode over a 5-minute window, per node.\n**Reading it:** A healthy validator stays predominantly in Proposing; a stock node stays in Observing.\n**Healthy range:** Mostly one dominant mode with negligible Wrong/Switched Ledger.\n**Watch for:** Frequent Wrong Ledger or Switched Ledger indicates the node is out of sync and at fork risk.\n**Source:** src/xrpld/app/consensus/RCLConsensus.cpp:RCLConsensus::Adaptor::onClose", "type": "timeseries", "gridPos": { "h": 8, @@ -278,7 +278,7 @@ }, { "title": "Accept vs Close Rate", - "description": "Compares the rate of consensus.accept (ledger accepted after consensus) vs consensus.ledger_close (ledger close initiated). These should track closely in a healthy network. A divergence means some close events are not completing the accept phase, potentially indicating consensus failures or timeouts.", + "description": "**What:** Rate of ledgers accepted after consensus versus ledger-close events initiated.\n**How it's computed:** Per-second rate of each over a 5-minute window, per node.\n**Reading it:** The two lines should overlap on a healthy network.\n**Healthy range:** Both near the ledger cadence (~0.25/s on mainnet) and tracking each other.\n**Watch for:** Divergence means some closes never complete the accept phase, hinting at consensus failures or timeouts.\n**Source:** src/xrpld/app/consensus/RCLConsensus.cpp:RCLConsensus::Adaptor::makeAcceptSpan, onClose", "type": "timeseries", "gridPos": { "h": 8, @@ -324,7 +324,7 @@ }, { "title": "Validation vs Close Rate", - "description": "Compares the rate of consensus.validation.send vs consensus.ledger_close. Each validated ledger should produce one validation message. If validations lag behind closes, the node may be falling behind on validation or experiencing issues with the validation pipeline.", + "description": "**What:** Rate of validations sent versus ledger-close events.\n**How it's computed:** Per-second rate of each over a 5-minute window, per node.\n**Reading it:** Each validated ledger should yield one validation, so the lines should overlap.\n**Healthy range:** Both near the ledger cadence and tracking each other.\n**Watch for:** Validations trailing closes points to a stalled or slow validation pipeline.\n**Source:** src/xrpld/app/consensus/RCLConsensus.cpp:RCLConsensus::Adaptor::createValidationSpan, onClose", "type": "timeseries", "gridPos": { "h": 8, @@ -370,7 +370,7 @@ }, { "title": "Consensus Accept Duration Heatmap", - "description": "Heatmap showing the distribution of consensus.accept span durations across histogram buckets over time. Each cell represents how many accept events fell into that duration bucket in a 5m window. Useful for detecting outlier consensus rounds that take abnormally long.", + "description": "**What:** Distribution of per-round accept durations over time, showing how round times are spread rather than just a percentile.\n**How it's computed:** Counts of rounds by duration band in each 5-minute window, per node.\n**Reading it:** A tight band around the typical round time is healthy; brighter cells mark the common duration.\n**Healthy range:** Concentrated near the 3-6 second mainnet round time.\n**Watch for:** A second band at high durations reveals recurring slow rounds hidden by averages.\n**Source:** src/xrpld/app/consensus/RCLConsensus.cpp:RCLConsensus::Adaptor::makeAcceptSpan", "type": "heatmap", "gridPos": { "h": 8, @@ -406,7 +406,7 @@ }, { "title": "Close Time: Raw Proposals (Per Node)", - "description": "Each node's raw proposed close time (close_time_self) \u2014 the unrounded wall clock value at the moment the node closed its ledger. Compare across nodes to see clock drift.", + "description": "**What:** Each node's raw, unrounded proposed close time at the instant it closed its ledger.\n**How it's computed:** Latest raw close-time value per node, plotted per round.\n**Reading it:** Compare nodes at the same round; values should cluster tightly.\n**Healthy range:** All nodes within a few seconds of each other.\n**Watch for:** A node consistently offset from the pack indicates local clock drift.\n**Source:** src/xrpld/app/consensus/RCLConsensus.cpp:RCLConsensus::Adaptor::doAccept", "type": "timeseries", "gridPos": { "h": 8, @@ -449,7 +449,7 @@ }, { "title": "Close Time: Effective / Quantized", - "description": "The consensus-agreed close time after rounding to the current resolution bin (close_time). This is the value written to the ledger header. All nodes in agreement produce the same value.", + "description": "**What:** The consensus-agreed close time after rounding to the active resolution bin, i.e. the value written to the ledger header.\n**How it's computed:** Latest effective close-time value per node, plotted per round.\n**Reading it:** All in-agreement nodes should report the identical value each round.\n**Healthy range:** Identical across agreeing nodes.\n**Watch for:** Nodes reporting different effective values are not in close-time agreement for that round.\n**Source:** src/xrpld/app/consensus/RCLConsensus.cpp:RCLConsensus::Adaptor::doAccept", "type": "timeseries", "gridPos": { "h": 8, @@ -492,7 +492,7 @@ }, { "title": "Close Time Vote Bins & Resolution", - "description": "Number of distinct close time vote bins (close_time_vote_bins) and the bin size / resolution in ms (close_resolution_ms). More bins = more clock disagreement. Resolution adapts: finer (10s) when validators agree, coarser (120s) when they disagree.", + "description": "**What:** Number of distinct close-time vote buckets and the current close-time resolution, per round.\n**How it's computed:** Latest bin count and resolution value plotted per round, per node.\n**Reading it:** Fewer bins and finer resolution mean tighter agreement; resolution coarsens automatically when validators disagree.\n**Healthy range:** Few bins with fine (about 10s) resolution during healthy operation.\n**Watch for:** Many bins or resolution jumping to its coarse ceiling signals widespread clock disagreement.\n**Source:** src/xrpld/app/consensus/RCLConsensus.cpp:RCLConsensus::Adaptor::doAccept", "type": "timeseries", "gridPos": { "h": 8, @@ -576,7 +576,7 @@ }, { "title": "Close Time Resolution Direction", - "description": "Whether close time resolution increased (coarser bins, more disagreement), decreased (finer bins, better agreement), or stayed unchanged relative to the previous ledger. Based on resolution_direction attribute.", + "description": "**What:** Whether close-time resolution coarsened, sharpened, or held steady versus the previous ledger.\n**How it's computed:** Latest direction outcome plotted per round, per node.\n**Reading it:** Mostly unchanged with occasional sharpening is healthy.\n**Healthy range:** Predominantly unchanged.\n**Watch for:** Repeated coarsening indicates worsening agreement round over round.\n**Source:** src/xrpld/app/consensus/RCLConsensus.cpp:RCLConsensus::Adaptor::doAccept", "type": "timeseries", "gridPos": { "h": 8, @@ -619,7 +619,7 @@ }, { "title": "Close Time Bin Distribution", - "description": "Distribution of raw proposed close times across quantized bins. Shows how many nodes' proposals landed in each resolution bin per consensus round. A single dominant bin indicates good clock agreement; spread across bins indicates drift or network latency.", + "description": "**What:** How raw proposed close times spread across the quantized buckets each round.\n**How it's computed:** Count of proposals landing in each bucket, stacked per round.\n**Reading it:** A single dominant bar means strong clock agreement.\n**Healthy range:** One dominant bucket per round.\n**Watch for:** Spread across many buckets points to clock drift or network latency between validators.\n**Source:** src/xrpld/app/consensus/RCLConsensus.cpp:RCLConsensus::Adaptor::doAccept", "type": "barchart", "gridPos": { "h": 8, @@ -663,7 +663,7 @@ }, { "title": "Consensus Outcome Distribution", - "description": "Distribution of consensus.accept outcomes: yes (normal), moved_on (without full agreement), expired (timeout). Non-yes outcomes indicate network stress.", + "description": "**What:** Share of consensus outcomes: normal agreement, moved on without full agreement, or expired (timed out).\n**How it's computed:** Total counts per outcome over a 5-minute window, shown as proportions.\n**Reading it:** The normal outcome should dominate overwhelmingly.\n**Healthy range:** Nearly all normal.\n**Watch for:** A growing moved-on or expired slice indicates network stress or instability.\n**Source:** src/xrpld/app/consensus/RCLConsensus.cpp:RCLConsensus::Adaptor::makeAcceptSpan", "type": "piechart", "gridPos": { "h": 8, @@ -699,7 +699,7 @@ }, { "title": "Consensus Failures Over Time", - "description": "Rate of non-normal consensus outcomes (moved_on + expired). Spikes indicate consensus instability.", + "description": "**What:** Rate of non-normal consensus outcomes (moved on plus expired) over time.\n**How it's computed:** Per-second rate of each failure outcome over a 5-minute window, per node.\n**Reading it:** Should sit at or near zero.\n**Healthy range:** Effectively zero on a stable network.\n**Watch for:** Spikes indicate consensus instability; sustained failures warrant investigation.\n**Source:** src/xrpld/app/consensus/RCLConsensus.cpp:RCLConsensus::Adaptor::makeAcceptSpan", "type": "timeseries", "gridPos": { "h": 8, @@ -745,7 +745,7 @@ }, { "title": "Consensus Round Duration (Full Round)", - "description": "p95/p50 duration of the full consensus round. The consensus.round span (RCLConsensus.cpp startRound) wraps an entire round end-to-end. Filterable by consensus mode. This is the single most important consensus-health signal; rising round time precedes ledger-age alarms.", + "description": "**What:** End-to-end duration of a complete consensus round, filterable by consensus mode.\n**How it's computed:** 95th and 50th percentile of full-round durations over a 5-minute window, per node.\n**Reading it:** The single most important consensus-health signal; watch the trend.\n**Healthy range:** Roughly 3-6 seconds on mainnet.\n**Watch for:** Rising round time precedes validated-ledger-age alarms and points to convergence trouble.\n**Source:** src/xrpld/app/consensus/RCLConsensus.cpp:RCLConsensus::Adaptor::startRoundTracing", "type": "timeseries", "gridPos": { "h": 8, @@ -791,7 +791,7 @@ }, { "title": "Consensus Phase Duration (Open vs Establish)", - "description": "p95 duration of the open phase (transaction collection) vs the establish phase (proposal convergence). The consensus.phase.open and consensus.establish spans decompose round latency, so an operator can tell whether slowness is in collecting transactions or reaching agreement.", + "description": "**What:** Duration of the open phase (collecting transactions) versus the establish phase (converging on a proposal), which together make up a round.\n**How it's computed:** 95th percentile of each phase's duration over a 5-minute window, per node.\n**Reading it:** Compare the two to see whether slowness is in gathering transactions or reaching agreement.\n**Healthy range:** Open phase near the close interval; establish phase typically shorter.\n**Watch for:** A ballooning establish phase means slow convergence; a long open phase means delayed close.\n**Source:** src/xrpld/consensus/Consensus.h:Consensus::phaseOpen, Consensus::phaseEstablish", "type": "timeseries", "gridPos": { "h": 8, @@ -837,7 +837,7 @@ }, { "title": "Position Update Duration", - "description": "p95/p50 duration of the consensus.update_positions span, which tallies disputes and updates this node's position each round. Long durations indicate heavy dispute resolution or slow convergence on close time.", + "description": "**What:** Time spent each round tallying disputes and updating this node's consensus position.\n**How it's computed:** 95th and 50th percentile of per-round durations over a 5-minute window, per node.\n**Reading it:** Grows with the number of disputed transactions being reconciled.\n**Healthy range:** A few milliseconds under normal conditions.\n**Watch for:** Long durations indicate heavy dispute resolution or slow convergence on close time.\n**Source:** src/xrpld/consensus/Consensus.h:Consensus::updateOurPositions", "type": "timeseries", "gridPos": { "h": 8, @@ -883,7 +883,7 @@ }, { "title": "Consensus Stall Rate", - "description": "Rate of consensus.check spans reporting consensus_stalled=true, broken down by stall flag. A non-zero stalled rate surfaces stall conditions before they manifest as validated-ledger-age alarms. Requires the consensus_stalled spanmetrics dimension.", + "description": "**What:** Rate of consensus health checks reporting a stalled condition versus progressing normally.\n**How it's computed:** Per-second rate split by stalled flag over a 5-minute window, per node.\n**Reading it:** The stalled line should stay at zero.\n**Healthy range:** Zero stalled checks.\n**Watch for:** Any sustained stalled rate surfaces a stall before it becomes a validated-ledger-age alarm.\n**Source:** src/xrpld/consensus/Consensus.h:Consensus::haveConsensus", "type": "timeseries", "gridPos": { "h": 8, @@ -929,7 +929,7 @@ }, { "title": "Consensus Mode-Change Rate by Target Mode", - "description": "Rate of consensus.mode_change spans broken down by the mode the node switched INTO (mode_new). Frequent switches into Wrong Ledger or Switched Ledger indicate an unstable node at fork risk. Requires the mode_new spanmetrics dimension.", + "description": "**What:** Rate of consensus mode transitions, split by the mode the node switched into.\n**How it's computed:** Per-second rate per target mode over a 5-minute window, per node.\n**Reading it:** A stable node changes mode rarely; occasional switches around startup are normal.\n**Healthy range:** Near zero once synced.\n**Watch for:** Frequent switches into Wrong Ledger or Switched Ledger mark an unstable node at fork risk.\n**Source:** src/xrpld/app/consensus/RCLConsensus.cpp:RCLConsensus::Adaptor::onModeChange", "type": "timeseries", "gridPos": { "h": 8, @@ -968,7 +968,7 @@ } ], "schemaVersion": 39, - "tags": ["consensus", "telemetry"], + "tags": ["consensus"], "templating": { "list": [ { @@ -1034,7 +1034,7 @@ { "name": "node", "label": "Node", - "description": "Filter by rippled node (service.instance.id \u2014 e.g. Node-1)", + "description": "Filter by rippled node (service.instance.id — e.g. Node-1)", "type": "query", "query": "label_values(traces_span_metrics_calls_total, exported_instance)", "datasource": { @@ -1142,7 +1142,7 @@ "from": "now-1h", "to": "now" }, - "description": "Consensus round health: round duration, proposal and validation rates, close-time agreement, and ledger-close cadence.", + "description": "Consensus round health: round and phase durations, proposal and validation rates, close-time agreement, stall detection, and ledger-close cadence.", "title": "Consensus Health", "uid": "xrpld-consensus" } diff --git a/docker/telemetry/grafana/dashboards/ledger-operations.json b/docker/telemetry/grafana/dashboards/ledger-operations.json index f7b3e71f57..6c75423b74 100644 --- a/docker/telemetry/grafana/dashboards/ledger-operations.json +++ b/docker/telemetry/grafana/dashboards/ledger-operations.json @@ -10,7 +10,7 @@ "panels": [ { "title": "Ledger Build Rate", - "description": "Rate at which new ledgers are being built. The ledger.build span (BuildLedger.cpp) wraps the entire buildLedgerImpl() function which creates a new ledger from a parent, applies transactions, flushes SHAMap nodes, and sets the accepted state. Should match the consensus close rate (~0.25/sec on mainnet with ~4s rounds).", + "description": "**What:** How often the node constructs a new ledger from its parent, applying transactions and writing state.\n**How it's computed:** Per-second rate over a 5-minute window, per node.\n**Reading it:** Should match the consensus close cadence.\n**Healthy range:** About 0.25/s on mainnet (~4s rounds).\n**Watch for:** A drop below the close rate means the node is not keeping up with ledger construction.\n**Source:** src/xrpld/app/ledger/detail/BuildLedger.cpp:buildLedgerImpl", "type": "stat", "gridPos": { "h": 8, @@ -42,7 +42,7 @@ }, { "title": "Ledger Build Duration", - "description": "p95 and p50 duration of ledger builds. Measures the full buildLedgerImpl() call including transaction application, SHAMap flushing, and ledger acceptance. The span records xrpl.ledger.seq as an attribute. Long build times indicate expensive transaction sets or I/O pressure from SHAMap flushes.", + "description": "**What:** Time to build a full ledger, including applying transactions, writing state, and finalising acceptance.\n**How it's computed:** 95th and 50th percentile of per-build durations over a 5-minute window, per node.\n**Reading it:** Scales with transaction volume and disk I/O.\n**Healthy range:** Tens to a few hundred milliseconds under typical load.\n**Watch for:** Long build times indicate expensive transaction sets or I/O pressure while flushing state.\n**Source:** src/xrpld/app/ledger/detail/BuildLedger.cpp:buildLedgerImpl", "type": "timeseries", "gridPos": { "h": 8, @@ -88,7 +88,7 @@ }, { "title": "Ledger Validation Rate", - "description": "Rate at which ledgers pass the validation threshold and are accepted as fully validated. The ledger.validate span (LedgerMaster.cpp) fires in checkAccept() only after the ledger receives sufficient trusted validations (>= quorum). Records xrpl.ledger.seq and validations (the number of validations received).", + "description": "**What:** How often ledgers reach the trusted-validation quorum and become fully validated.\n**How it's computed:** Per-second rate over a 5-minute window, per node.\n**Reading it:** Should match the build and close rates on a healthy node.\n**Healthy range:** About 0.25/s on mainnet.\n**Watch for:** A shortfall means ledgers are not accumulating enough trusted validations to be accepted.\n**Source:** src/xrpld/app/ledger/detail/LedgerMaster.cpp:LedgerMaster::checkAccept", "type": "stat", "gridPos": { "h": 8, @@ -120,7 +120,7 @@ }, { "title": "Ledger Build Duration Heatmap", - "description": "Heatmap showing the distribution of ledger.build durations across histogram buckets over time. Each cell represents the count of ledger builds that fell into that duration bucket in a 5m window. Useful for spotting occasional slow ledger builds that may not appear in percentile charts.", + "description": "**What:** Distribution of ledger-build durations over time, revealing spread beyond a single percentile.\n**How it's computed:** Counts of builds by duration band in each 5-minute window, per node.\n**Reading it:** A tight band at low durations is healthy; brighter cells mark the common build time.\n**Healthy range:** Concentrated in the low tens of milliseconds.\n**Watch for:** An occasional high-duration band flags slow builds that percentile charts may miss.\n**Source:** src/xrpld/app/ledger/detail/BuildLedger.cpp:buildLedgerImpl", "type": "heatmap", "gridPos": { "h": 8, @@ -156,7 +156,7 @@ }, { "title": "Transaction Apply Duration", - "description": "p95 and p50 duration of applying the consensus transaction set during ledger building. The tx.apply span (BuildLedger.cpp) wraps applyTransactions() which iterates through the CanonicalTXSet with multiple retry passes. Records tx_count (successful) and tx_failed (failed) as attributes.", + "description": "**What:** Time to apply the agreed transaction set into the new ledger, including retry passes.\n**How it's computed:** 95th and 50th percentile of per-ledger apply durations over a 5-minute window, per node.\n**Reading it:** Grows with the number and complexity of transactions in the set.\n**Healthy range:** Tens to a few hundred milliseconds under normal load.\n**Watch for:** Sustained high durations indicate heavy or expensive transaction sets.\n**Source:** src/xrpld/app/ledger/detail/BuildLedger.cpp:applyTransactions", "type": "timeseries", "gridPos": { "h": 8, @@ -202,7 +202,7 @@ }, { "title": "Transaction Apply Rate", - "description": "Rate of tx.apply span invocations, reflecting how frequently the transaction application phase runs during ledger building. Each ledger build triggers one tx.apply call. Should closely match the ledger build rate.", + "description": "**What:** How often the transaction-application phase runs during ledger building.\n**How it's computed:** Per-second rate over a 5-minute window, per node.\n**Reading it:** One apply pass per ledger build, so it should track the build rate.\n**Healthy range:** About 0.25/s on mainnet.\n**Watch for:** Divergence from the build rate signals an accounting or pipeline anomaly.\n**Source:** src/xrpld/app/ledger/detail/BuildLedger.cpp:applyTransactions", "type": "timeseries", "gridPos": { "h": 8, @@ -241,7 +241,7 @@ }, { "title": "Ledger Store Rate", - "description": "Rate at which ledgers are stored into the ledger history. The ledger.store span (LedgerMaster.cpp) wraps storeLedger() which inserts the ledger into the LedgerHistory cache. Records xrpl.ledger.seq. Should match the ledger build rate under normal operation.", + "description": "**What:** How often completed ledgers are written into the local ledger history.\n**How it's computed:** Per-second rate over a 5-minute window, per node.\n**Reading it:** Should match the build rate under normal operation.\n**Healthy range:** About 0.25/s on mainnet.\n**Watch for:** A store rate below the build rate means ledgers are being built but not persisted.\n**Source:** src/xrpld/app/ledger/detail/LedgerMaster.cpp:LedgerMaster::storeLedger", "type": "stat", "gridPos": { "h": 8, @@ -273,7 +273,7 @@ }, { "title": "Build vs Close Duration", - "description": "Compares p95 durations of ledger.build (the actual ledger construction in BuildLedger.cpp) vs consensus.ledger_close (the consensus close event in RCLConsensus.cpp). Build time is a subset of close time. A large gap between them indicates overhead in the consensus pipeline outside of ledger construction itself.", + "description": "**What:** Ledger construction time versus total ledger-close time; construction is a subset of close.\n**How it's computed:** 95th percentile of each duration over a 5-minute window, per node.\n**Reading it:** Build should sit at or just below close.\n**Healthy range:** Both a few hundred milliseconds or less, tracking each other.\n**Watch for:** A large gap means significant overhead in the consensus pipeline outside ledger construction.\n**Source:** src/xrpld/app/ledger/detail/BuildLedger.cpp:buildLedgerImpl, src/xrpld/app/consensus/RCLConsensus.cpp:RCLConsensus::Adaptor::onClose", "type": "timeseries", "gridPos": { "h": 8, @@ -319,7 +319,7 @@ } ], "schemaVersion": 39, - "tags": ["ledger", "telemetry"], + "tags": ["ledger"], "templating": { "list": [ { @@ -385,7 +385,7 @@ { "name": "node", "label": "Node", - "description": "Filter by rippled node (service.instance.id \u2014 e.g. Node-1)", + "description": "Filter by rippled node (service.instance.id — e.g. Node-1)", "type": "query", "query": "label_values(traces_span_metrics_calls_total, exported_instance)", "datasource": { @@ -408,7 +408,7 @@ "from": "now-1h", "to": "now" }, - "description": "Ledger lifecycle: build rate and duration, validation rate, and close timing.", + "description": "Ledger lifecycle: build rate and duration, transaction apply and store rates, validation rate, and close timing.", "title": "Ledger Operations", "uid": "xrpld-ledger-ops" } diff --git a/docker/telemetry/grafana/dashboards/peer-network.json b/docker/telemetry/grafana/dashboards/peer-network.json index 091292460b..ab8c535aa7 100644 --- a/docker/telemetry/grafana/dashboards/peer-network.json +++ b/docker/telemetry/grafana/dashboards/peer-network.json @@ -2,7 +2,7 @@ "annotations": { "list": [] }, - "description": "Requires trace_peer=1 in the [telemetry] config section.", + "description": "Peer-to-peer consensus traffic: rates of proposals and validations received from peers and their split between trusted and untrusted sources.", "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, @@ -11,7 +11,7 @@ "panels": [ { "title": "Peer Proposal Receive Rate", - "description": "Rate of consensus proposals received from network peers. The peer.proposal.receive span (PeerImp.cpp) fires in onMessage(TMProposeSet) for each incoming proposal. Records xrpl.peer.id (sending peer) and proposal_trusted (whether the proposer is in our UNL). Requires trace_peer=1 in the telemetry config.", + "description": "**What:** Rate of consensus proposals arriving from connected network peers.\n**How it's computed:** Per-second rate over a 5-minute window, per node.\n**Reading it:** Reflects how many validators' proposals reach this node each round.\n**Healthy range:** Workload-dependent; scales with peer count and network activity.\n**Watch for:** A sudden drop means loss of proposal flow; a sharp sustained spike from a single peer can indicate flooding.\n**Source:** src/xrpld/overlay/detail/PeerImp.cpp:PeerImp::onMessage(TMProposeSet)", "type": "timeseries", "gridPos": { "h": 8, @@ -50,7 +50,7 @@ }, { "title": "Peer Validation Receive Rate", - "description": "Rate of ledger validations received from network peers. The peer.validation.receive span (PeerImp.cpp) fires in onMessage(TMValidation) for each incoming validation message. Records xrpl.peer.id (sending peer) and validation_trusted (whether the validator is trusted). Requires trace_peer=1 in the telemetry config.", + "description": "**What:** Rate of ledger validations arriving from connected network peers.\n**How it's computed:** Per-second rate over a 5-minute window, per node.\n**Reading it:** Reflects the volume of validation traffic reaching this node.\n**Healthy range:** Workload-dependent; scales with the number of validators and peers.\n**Watch for:** A drop starves the validation quorum; an abnormal spike from one peer can indicate abuse.\n**Source:** src/xrpld/overlay/detail/PeerImp.cpp:PeerImp::onMessage(TMValidation)", "type": "timeseries", "gridPos": { "h": 8, @@ -89,7 +89,7 @@ }, { "title": "Proposals Trusted vs Untrusted", - "description": "Pie chart showing the ratio of proposals received from trusted validators (in our UNL) vs untrusted validators. Grouped by the proposal_trusted span attribute (true/false). A healthy node connected to a well-configured UNL should see a significant portion of trusted proposals. Note: proposals that fail early validation may not have the trusted attribute set.", + "description": "**What:** Share of received proposals from trusted validators (in this node's UNL) versus untrusted sources.\n**How it's computed:** Total counts split by trust status over a 5-minute window, shown as proportions.\n**Reading it:** A well-connected node with a good UNL sees a healthy trusted share.\n**Healthy range:** A substantial trusted proportion; exact mix is workload-dependent.\n**Watch for:** A collapse of the trusted share means poor connectivity to UNL validators; a flood of untrusted proposals can indicate abuse.\n**Source:** src/xrpld/overlay/detail/PeerImp.cpp:PeerImp::onMessage(TMProposeSet)", "type": "piechart", "gridPos": { "h": 8, @@ -121,7 +121,7 @@ }, { "title": "Validations Trusted vs Untrusted", - "description": "Pie chart showing the ratio of validations received from trusted validators (in our UNL) vs untrusted validators. Grouped by the validation_trusted span attribute (true/false). Monitoring this helps detect if the node is receiving validations from the expected set of trusted validators. Note: validations that fail early checks may not have the trusted attribute set.", + "description": "**What:** Share of received validations from trusted validators (in this node's UNL) versus untrusted sources.\n**How it's computed:** Total counts split by trust status over a 5-minute window, shown as proportions.\n**Reading it:** Confirms the node is hearing from its expected trusted validator set.\n**Healthy range:** A substantial trusted proportion; exact mix is workload-dependent.\n**Watch for:** A shrinking trusted share risks the validation quorum; a surge of untrusted validations can indicate abuse.\n**Source:** src/xrpld/overlay/detail/PeerImp.cpp:PeerImp::onMessage(TMValidation)", "type": "piechart", "gridPos": { "h": 8, @@ -153,7 +153,7 @@ } ], "schemaVersion": 39, - "tags": ["peer", "telemetry"], + "tags": ["network", "peer"], "templating": { "list": [ { @@ -219,7 +219,7 @@ { "name": "node", "label": "Node", - "description": "Filter by rippled node (service.instance.id \u2014 e.g. Node-1)", + "description": "Filter by rippled node (service.instance.id — e.g. Node-1)", "type": "query", "query": "label_values(traces_span_metrics_calls_total, exported_instance)", "datasource": { diff --git a/docker/telemetry/grafana/dashboards/rpc-performance.json b/docker/telemetry/grafana/dashboards/rpc-performance.json index fba8e92c45..45db3632b7 100644 --- a/docker/telemetry/grafana/dashboards/rpc-performance.json +++ b/docker/telemetry/grafana/dashboards/rpc-performance.json @@ -10,7 +10,7 @@ "panels": [ { "title": "RPC Request Rate by Command", - "description": "Per-second rate of RPC command executions, broken down by command name (e.g. server_info, submit). Calculated as rate(traces_span_metrics_calls_total{span_name=~\"rpc.command.*\"}) over a 5m window, grouped by the command span attribute.", + "description": "**What:** Per-command throughput of RPC calls the node serves (e.g. server_info, submit, account_info).\n**How it's computed:** Per-second rate split by command name over a 5-minute window, per node.\n**Reading it:** Shows which API endpoints drive load and how demand shifts over time.\n**Healthy range:** Workload-dependent; varies with client mix.\n**Watch for:** A sudden spike on one command can indicate a runaway client or scripted abuse.\n**Source:** src/xrpld/rpc/detail/RPCHandler.cpp:callMethod", "type": "timeseries", "gridPos": { "h": 8, @@ -49,7 +49,7 @@ }, { "title": "RPC Latency P95 by Command", - "description": "95th percentile response time for each RPC command. Computed from the spanmetrics duration histogram using histogram_quantile(0.95) over rpc.command.* spans, grouped by command. High values indicate slow commands that may need optimization.", + "description": "**What:** 95th-percentile response time for each RPC command.\n**How it's computed:** 95th percentile of per-command durations over a 5-minute window, per node.\n**Reading it:** Read-only lookups should be fast; heavy commands like path finding are naturally slower.\n**Healthy range:** Sub-second for most commands; command-dependent.\n**Watch for:** Rising p95 on a normally fast command signals contention or an expensive query pattern.\n**Source:** src/xrpld/rpc/detail/RPCHandler.cpp:callMethod", "type": "timeseries", "gridPos": { "h": 8, @@ -88,7 +88,7 @@ }, { "title": "RPC Error Rate", - "description": "Percentage of RPC commands that completed with an error status, per command. Calculated as (error calls / total calls) * 100, where errors have status_code=STATUS_CODE_ERROR. Thresholds: green < 1%, yellow 1-5%, red > 5%.", + "description": "**What:** Percentage of each RPC command's calls that ended in an error.\n**How it's computed:** Error calls divided by total calls per command over a 5-minute window, per node.\n**Reading it:** Green below 1%, yellow 1-5%, red above 5%.\n**Healthy range:** Below 1% for healthy commands.\n**Watch for:** A sustained error spike on one command is consistent with a client probing or misusing that endpoint.\n**Source:** src/xrpld/rpc/detail/RPCHandler.cpp:callMethod", "type": "bargauge", "gridPos": { "h": 8, @@ -136,7 +136,7 @@ }, { "title": "RPC Latency Heatmap", - "description": "Distribution of RPC command response times across histogram buckets. Shows the density of requests at each latency level over time. Each cell represents the count of requests that fell into that duration bucket in a 5m window. Useful for spotting bimodal latency patterns.", + "description": "**What:** Distribution of RPC response times over time across all commands.\n**How it's computed:** Counts of requests by duration band in each 5-minute window, per node.\n**Reading it:** A single tight band is healthy; brighter cells mark the common latency.\n**Healthy range:** Concentrated at low latency; command-dependent.\n**Watch for:** Two separate bands (bimodal latency) reveal a slow path affecting some requests.\n**Source:** src/xrpld/rpc/detail/RPCHandler.cpp:callMethod", "type": "heatmap", "gridPos": { "h": 8, @@ -172,7 +172,7 @@ }, { "title": "Overall RPC Throughput", - "description": "Aggregate RPC throughput showing two layers of the request pipeline. rpc.http_request is the outer HTTP handler (ServerHandler.cpp) that accepts incoming connections. rpc.process is the inner processing layer (ServerHandler.cpp) that parses and dispatches. A gap between the two indicates requests being queued or rejected before processing.", + "description": "**What:** Aggregate request flow through two pipeline layers: the outer HTTP handler that accepts connections and the inner layer that parses and dispatches.\n**How it's computed:** Per-second rate of each layer over a 5-minute window, per node.\n**Reading it:** The two lines should overlap when requests flow cleanly.\n**Healthy range:** Workload-dependent; both layers tracking each other.\n**Watch for:** A gap means requests are being queued or rejected before dispatch.\n**Source:** src/xrpld/rpc/detail/ServerHandler.cpp:ServerHandler::processSession, ServerHandler::processRequest", "type": "timeseries", "gridPos": { "h": 8, @@ -218,7 +218,7 @@ }, { "title": "RPC Success vs Error", - "description": "Aggregate rate of successful vs failed RPC commands across all command types. Success = status_code UNSET (OpenTelemetry default for OK spans). Error = status_code STATUS_CODE_ERROR. A sustained error rate warrants investigation via per-command breakdown above.", + "description": "**What:** Aggregate rate of successful versus failed RPC commands across all types.\n**How it's computed:** Per-second rate of each outcome over a 5-minute window, per node.\n**Reading it:** Success should dominate; the error line should stay low.\n**Healthy range:** Error rate near zero; success rate is workload-dependent.\n**Watch for:** A sustained rise in the error line warrants drilling into the per-command breakdown.\n**Source:** src/xrpld/rpc/detail/RPCHandler.cpp:callMethod", "type": "timeseries", "gridPos": { "h": 8, @@ -264,7 +264,7 @@ }, { "title": "Top Commands by Volume", - "description": "Top 10 most frequently called RPC commands by total invocation count over the last 5 minutes. Uses topk(10, increase(calls_total)) to rank commands. Helps identify the hottest API endpoints driving load on the node.", + "description": "**What:** The ten most-called RPC commands by total invocations recently.\n**How it's computed:** Ranked total counts per command over a 5-minute window, per node.\n**Reading it:** Identifies the hottest API endpoints driving node load.\n**Healthy range:** Workload-dependent.\n**Watch for:** An unexpected command dominating the ranking can indicate scripted abuse or a misbehaving client.\n**Source:** src/xrpld/rpc/detail/RPCHandler.cpp:callMethod", "type": "bargauge", "gridPos": { "h": 8, @@ -296,7 +296,7 @@ }, { "title": "WebSocket Message Rate", - "description": "Rate of incoming WebSocket RPC messages processed by the server. Sourced from the rpc.ws_message span (ServerHandler.cpp). Only active when clients connect via WebSocket instead of HTTP. Zero is normal if only HTTP RPC is in use.", + "description": "**What:** Rate of incoming WebSocket RPC messages the server processes.\n**How it's computed:** Per-second rate over a 5-minute window, per node.\n**Reading it:** Non-zero only when clients connect over WebSocket rather than HTTP.\n**Healthy range:** Zero is normal for HTTP-only deployments; otherwise workload-dependent.\n**Watch for:** An unexpected surge points to a chatty or abusive WebSocket client.\n**Source:** src/xrpld/rpc/detail/ServerHandler.cpp:ServerHandler::processSession", "type": "stat", "gridPos": { "h": 8, @@ -328,7 +328,7 @@ }, { "title": "RPC Resource Cost by Command", - "description": "RPC commands grouped by load_type (resource cost category). High-cost categories like exception_rpc or malformed_rpc indicate problematic clients.", + "description": "**What:** RPC traffic grouped by its resource-cost category, distinguishing cheap lookups from expensive or malformed requests.\n**How it's computed:** Per-second rate per cost category over a 5-minute window.\n**Reading it:** Low-cost categories should dominate.\n**Healthy range:** Mostly low-cost traffic; workload-dependent.\n**Watch for:** A rising share of high-cost or malformed categories points to problematic or abusive clients.\n**Source:** src/xrpld/rpc/detail/RPCHandler.cpp:callMethod", "type": "timeseries", "gridPos": { "h": 8, @@ -372,7 +372,7 @@ }, { "title": "Batch vs Single RPC Requests", - "description": "Rate of batch RPC requests vs single requests. High batch rate may indicate bulk automation clients.", + "description": "**What:** Rate of batched RPC requests versus single requests.\n**How it's computed:** Per-second rate of each over a 5-minute window, per node.\n**Reading it:** Most traffic is typically single requests.\n**Healthy range:** Workload-dependent; batch share usually small.\n**Watch for:** A high batch rate may indicate bulk-automation clients or an attempt to amplify load per connection.\n**Source:** src/xrpld/rpc/detail/ServerHandler.cpp:ServerHandler::processRequest", "type": "timeseries", "gridPos": { "h": 8, @@ -418,7 +418,7 @@ } ], "schemaVersion": 39, - "tags": ["rpc", "telemetry"], + "tags": ["rpc"], "templating": { "list": [ { @@ -484,7 +484,7 @@ { "name": "node", "label": "Node", - "description": "Filter by rippled node (service.instance.id \u2014 e.g. Node-1)", + "description": "Filter by rippled node (service.instance.id — e.g. Node-1)", "type": "query", "query": "label_values(traces_span_metrics_calls_total, exported_instance)", "datasource": { @@ -528,5 +528,6 @@ "to": "now" }, "title": "RPC Performance", - "uid": "xrpld-rpc-perf" + "uid": "xrpld-rpc-perf", + "description": "Client RPC service health: per-command request rates, latency, error rates, throughput across pipeline layers, and resource cost." } diff --git a/docker/telemetry/grafana/dashboards/transaction-overview.json b/docker/telemetry/grafana/dashboards/transaction-overview.json index bdbb6031d5..3acb714b34 100644 --- a/docker/telemetry/grafana/dashboards/transaction-overview.json +++ b/docker/telemetry/grafana/dashboards/transaction-overview.json @@ -10,7 +10,7 @@ "panels": [ { "title": "Transaction Processing Rate", - "description": "Rate of transactions entering the processing pipeline. tx.process (NetworkOPs.cpp) fires when a transaction is submitted locally or received from a peer and enters processTransaction(). tx.receive (PeerImp.cpp) fires when a raw transaction message arrives from a peer before deduplication.", + "description": "**What:** Rate of transactions entering the processing pipeline (submitted locally or relayed by peers) versus raw transaction messages arriving from peers before deduplication.\n**How it's computed:** Per-second rate of each over a 5-minute window, per node.\n**Reading it:** The received line sits above the processed line by the volume of duplicates filtered out.\n**Healthy range:** Workload-dependent; scales with network transaction volume.\n**Watch for:** A large and growing gap means heavy duplicate traffic; a processed-rate collapse means submissions are not being handled.\n**Source:** src/xrpld/app/misc/NetworkOPs.cpp:NetworkOPsImp::processTransaction, src/xrpld/overlay/detail/PeerImp.cpp:PeerImp::handleTransaction", "type": "timeseries", "gridPos": { "h": 8, @@ -56,7 +56,7 @@ }, { "title": "Transaction Processing Latency by Type", - "description": "Per-transaction-type processing latency (p95 and p50). Filter with $tx_type variable above.", + "description": "**What:** Per-transaction-type processing latency through the pipeline.\n**How it's computed:** 95th and 50th percentile of per-type durations over a 5-minute window, per node.\n**Reading it:** Simple payments are fast; complex types like offers or AMM operations run longer.\n**Healthy range:** Sub-millisecond to a few milliseconds; type-dependent.\n**Watch for:** A latency climb for one type signals contention or expensive processing specific to it.\n**Source:** src/xrpld/app/misc/NetworkOPs.cpp:NetworkOPsImp::processTransaction", "type": "timeseries", "gridPos": { "h": 8, @@ -107,7 +107,7 @@ }, { "title": "Transaction Path Distribution", - "description": "Breakdown of transactions by origin path. The local attribute indicates whether the transaction was submitted locally (true) or received from a peer (false). Helps understand the ratio of locally-originated vs relayed transactions.", + "description": "**What:** Split of processed transactions by origin: submitted locally versus relayed from peers.\n**How it's computed:** Total counts split by origin over a 5-minute window, shown as proportions.\n**Reading it:** Most nodes see mostly relayed traffic; a submission endpoint sees more local.\n**Healthy range:** Workload-dependent.\n**Watch for:** A surge in local submissions on a node not meant to accept them can indicate misuse.\n**Source:** src/xrpld/app/misc/NetworkOPs.cpp:NetworkOPsImp::processTransaction", "type": "piechart", "gridPos": { "h": 8, @@ -133,7 +133,7 @@ }, { "title": "Transaction Receive vs Suppressed", - "description": "Total rate of raw transaction messages received from peers (tx.receive span from PeerImp.cpp). This fires before deduplication via the HashRouter, so the difference between tx.receive and tx.process reflects suppressed duplicate transactions.", + "description": "**What:** Raw transaction messages received from peers, split by whether they were suppressed as duplicates before processing.\n**How it's computed:** Per-second rate split by suppressed flag over a 5-minute window, per node.\n**Reading it:** A healthy relay network produces a steady suppressed share as duplicates are filtered.\n**Healthy range:** Workload-dependent; suppression is expected and normal.\n**Watch for:** A sharp rise in the suppressed line can reflect gossip amplification or a peer replaying transactions.\n**Source:** src/xrpld/overlay/detail/PeerImp.cpp:PeerImp::handleTransaction", "type": "timeseries", "gridPos": { "h": 8, @@ -172,7 +172,7 @@ }, { "title": "Transaction Processing Duration Heatmap", - "description": "Heatmap showing the distribution of tx.process span durations across histogram buckets over time. Each cell represents the count of transactions that completed within that latency bucket in a 5m window. Reveals whether processing times are consistent or exhibit multi-modal patterns.", + "description": "**What:** Distribution of transaction processing times over time.\n**How it's computed:** Counts of transactions by duration band in each 5-minute window, per node.\n**Reading it:** A single tight band is healthy; brighter cells mark the common processing time.\n**Healthy range:** Concentrated at low latency.\n**Watch for:** A second high-latency band reveals a subset of transactions that are consistently slow.\n**Source:** src/xrpld/app/misc/NetworkOPs.cpp:NetworkOPsImp::processTransaction", "type": "heatmap", "gridPos": { "h": 8, @@ -208,7 +208,7 @@ }, { "title": "Transaction Apply Duration per Ledger", - "description": "p95 and p50 latency of applying the consensus transaction set to a new ledger. The tx.apply span (BuildLedger.cpp) wraps the applyTransactions() function that iterates through the CanonicalTXSet and applies each transaction to the OpenView. Long durations indicate heavy transaction sets or expensive transaction processing.", + "description": "**What:** Time to apply the agreed transaction set into a new ledger.\n**How it's computed:** 95th and 50th percentile of per-ledger apply durations over a 5-minute window, per node.\n**Reading it:** Scales with the number and complexity of transactions per ledger.\n**Healthy range:** Tens to a few hundred milliseconds under normal load.\n**Watch for:** Sustained high durations indicate heavy transaction sets or expensive processing.\n**Source:** src/xrpld/app/ledger/detail/BuildLedger.cpp:applyTransactions", "type": "timeseries", "gridPos": { "h": 8, @@ -254,7 +254,7 @@ }, { "title": "Peer Transaction Receive Rate", - "description": "Rate of transaction messages received from network peers. Sourced from the tx.receive span (PeerImp.cpp) which fires in the onMessage(TMTransaction) handler. High rates may indicate network-wide transaction volume spikes or peer flooding.", + "description": "**What:** Rate of transaction messages received from network peers.\n**How it's computed:** Per-second rate over a 5-minute window, per node.\n**Reading it:** Reflects inbound transaction volume from the network.\n**Healthy range:** Workload-dependent; scales with network activity.\n**Watch for:** A sustained spike may reflect a network-wide volume surge or a peer flooding transactions.\n**Source:** src/xrpld/overlay/detail/PeerImp.cpp:PeerImp::handleTransaction", "type": "timeseries", "gridPos": { "h": 8, @@ -293,7 +293,7 @@ }, { "title": "Transaction Apply Failed Rate", - "description": "Rate of tx.apply spans completing with error status, indicating transaction application failures during ledger building. The span records tx_failed as an attribute. Thresholds: green < 0.1/sec, yellow 0.1-1/sec, red > 1/sec. Some failures are normal (e.g. conflicting offers) but sustained high rates may indicate issues.", + "description": "**What:** Rate of transaction-apply attempts that failed while building a ledger.\n**How it's computed:** Per-second rate of failed applies over a 5-minute window, per node.\n**Reading it:** Green below 0.1/s, yellow 0.1-1/s, red above 1/s.\n**Healthy range:** Some failures (e.g. conflicting offers) are normal; keep it under 0.1/s.\n**Watch for:** A sustained high failure rate can indicate malformed transaction floods or a systemic issue.\n**Source:** src/xrpld/app/ledger/detail/BuildLedger.cpp:applyTransactions", "type": "stat", "gridPos": { "h": 8, @@ -341,7 +341,7 @@ }, { "title": "Transaction Rate by Type", - "description": "Transaction processing rate broken down by tx_type (Payment, OfferCreate, AMMDeposit, etc.). Requires tx_type dimension in spanmetrics.", + "description": "**What:** Processing rate broken down by transaction type (Payment, OfferCreate, AMM operations, etc.).\n**How it's computed:** Per-second rate per type over a 5-minute window.\n**Reading it:** Shows the transaction mix and how it shifts over time.\n**Healthy range:** Workload-dependent; usually Payment-heavy.\n**Watch for:** A sudden burst of one type is a leading indicator of spam or fee escalation.\n**Source:** src/xrpld/app/misc/NetworkOPs.cpp:NetworkOPsImp::processTransaction", "type": "timeseries", "gridPos": { "h": 8, @@ -385,7 +385,7 @@ }, { "title": "Transaction Results by Type", - "description": "Transaction result codes (ter_result) broken down by tx_type. Shows which transaction types fail most often.", + "description": "**What:** Non-success result codes broken down by transaction type, showing which types fail and how.\n**How it's computed:** Per-second rate per type and result code over a 5-minute window (successes excluded).\n**Reading it:** Occasional failures are normal; watch which types and codes dominate.\n**Healthy range:** Low and workload-dependent.\n**Watch for:** A concentration of one failure code on one type can indicate a broken client or targeted spam.\n**Source:** src/xrpld/app/misc/NetworkOPs.cpp:NetworkOPsImp::processTransaction", "type": "timeseries", "gridPos": { "h": 8, @@ -429,7 +429,7 @@ }, { "title": "TxQ Accept Status", - "description": "TxQ accept outcomes: applied (included in ledger), failed (removed), retried (kept for next round).", + "description": "**What:** Outcomes when queued transactions are considered for a ledger: applied (included), failed (removed), or retried (kept for the next round).\n**How it's computed:** Total counts per outcome over a 5-minute window, shown as proportions.\n**Reading it:** Applied should dominate when the queue is draining healthily.\n**Healthy range:** Mostly applied; workload-dependent.\n**Watch for:** A rising retried or failed share signals queue pressure and fee escalation.\n**Source:** src/xrpld/app/misc/detail/TxQ.cpp:TxQ::accept", "type": "piechart", "gridPos": { "h": 8, @@ -465,7 +465,7 @@ }, { "title": "Transactor Duration by Type (p95)", - "description": "Per-transactor execution time (tx.transactor span). Shows which transaction types are most expensive to execute.", + "description": "**What:** Per-type execution time of the transactor that actually applies each transaction's effects.\n**How it's computed:** 95th percentile of per-type durations over a 5-minute window, per node.\n**Reading it:** Reveals which transaction types are most expensive to execute.\n**Healthy range:** Sub-millisecond to a few milliseconds; type-dependent.\n**Watch for:** A climbing figure for one type points to expensive processing or contention for that type.\n**Source:** src/libxrpl/tx/Transactor.cpp:Transactor::operator()", "type": "timeseries", "gridPos": { "h": 8, @@ -509,7 +509,7 @@ }, { "title": "TxQ Enqueue Rate by Transaction Type", - "description": "Rate of txq.enqueue spans broken down by transaction type (tx_type). Shows what share of inbound demand is Payment vs OfferCreate vs other transactors, and how the mix shifts as the queue fills. A spam burst of one type is a leading indicator of fee escalation.", + "description": "**What:** Rate at which transactions are placed into the queue, split by transaction type.\n**How it's computed:** Per-second rate per type over a 5-minute window, per node.\n**Reading it:** Shows which types make up inbound demand and how the mix shifts as the queue fills.\n**Healthy range:** Workload-dependent.\n**Watch for:** A spam burst of one type here is a leading indicator of fee escalation.\n**Source:** src/xrpld/app/misc/detail/TxQ.cpp:TxQ::apply", "type": "timeseries", "gridPos": { "h": 8, @@ -548,7 +548,7 @@ }, { "title": "Queue Bypass Ratio (Direct Apply vs Enqueue)", - "description": "Ratio of transactions that applied directly to the open ledger (txq.apply_direct) versus those that had to be queued (txq.enqueue). A falling bypass ratio is the cleanest single signal the network has entered sustained fee escalation.", + "description": "**What:** Fraction of transactions that applied straight to the open ledger versus those that had to be queued.\n**How it's computed:** Ratio of direct applies to direct-plus-queued over a 5-minute window, per node.\n**Reading it:** A high fraction means the network is keeping up without escalation.\n**Healthy range:** Near 1.0 in normal conditions.\n**Watch for:** A falling fraction is the cleanest single signal the network has entered sustained fee escalation.\n**Source:** src/xrpld/app/misc/detail/TxQ.cpp:TxQ::tryDirectApply, TxQ::apply", "type": "timeseries", "gridPos": { "h": 8, @@ -587,7 +587,7 @@ }, { "title": "Queue Accept (Drain) Duration per Ledger", - "description": "p95/p50 duration of the txq.accept span, which drains queued transactions into a newly closed ledger. Rising drain time signals queue pressure at ledger close.", + "description": "**What:** Time spent draining queued transactions into a newly closed ledger.\n**How it's computed:** 95th and 50th percentile of per-ledger drain durations over a 5-minute window, per node.\n**Reading it:** Rises as the queue holds more transactions to process at close.\n**Healthy range:** A few milliseconds when the queue is light.\n**Watch for:** Rising drain time signals queue pressure at ledger close.\n**Source:** src/xrpld/app/misc/detail/TxQ.cpp:TxQ::accept", "type": "timeseries", "gridPos": { "h": 8, @@ -633,7 +633,7 @@ }, { "title": "Queue Cleanup Rate (Expired Entries)", - "description": "Rate of txq.cleanup spans, which remove expired transactions from the queue each ledger. A rising rate means submitters under-bid the escalating fee and abandoned their transactions \u2014 a demand-frustration signal distinct from acceptance throughput.", + "description": "**What:** Rate at which expired transactions are removed from the queue each ledger.\n**How it's computed:** Per-second rate over a 5-minute window, per node.\n**Reading it:** Normally low; entries expire when submitters underbid the current fee.\n**Healthy range:** Near zero in calm conditions.\n**Watch for:** A rising rate means submitters abandoned under-fee transactions, a demand-frustration signal distinct from throughput.\n**Source:** src/xrpld/app/misc/detail/TxQ.cpp:TxQ::processClosedLedger", "type": "timeseries", "gridPos": { "h": 8, @@ -672,7 +672,7 @@ }, { "title": "Tx Apply Pipeline Rate by Stage", - "description": "Span rate for each apply-pipeline stage (preflight, preclaim, apply). A drop between stages shows where transactions are filtered out. Requires the stage dimension in spanmetrics.", + "description": "**What:** Throughput of each apply-pipeline stage (preflight, preclaim, apply), showing where transactions drop out.\n**How it's computed:** Per-second rate per stage over a 5-minute window, per node.\n**Reading it:** A decline from earlier to later stages shows where transactions are filtered.\n**Healthy range:** Workload-dependent; later stages sit at or below earlier ones.\n**Watch for:** A large early-stage drop means many transactions fail basic checks, consistent with malformed floods.\n**Source:** src/libxrpl/tx/Transactor.cpp:Transactor::operator()", "type": "timeseries", "gridPos": { "h": 8, @@ -716,7 +716,7 @@ }, { "title": "Tx Apply Pipeline Latency by Stage (p95)", - "description": "95th-percentile duration of each apply-pipeline stage. Isolates which stage (preflight, preclaim, apply) dominates transaction processing time.", + "description": "**What:** 95th-percentile duration of each apply-pipeline stage (preflight, preclaim, apply).\n**How it's computed:** 95th percentile of per-stage durations over a 5-minute window, per node.\n**Reading it:** Isolates which stage dominates transaction processing time.\n**Healthy range:** Sub-millisecond to a few milliseconds per stage.\n**Watch for:** A stage that grows disproportionately points to a bottleneck in that step.\n**Source:** src/libxrpl/tx/Transactor.cpp:Transactor::operator()", "type": "timeseries", "gridPos": { "h": 8, @@ -760,7 +760,7 @@ }, { "title": "Tx Apply Pipeline Failure Rate by Stage", - "description": "Rate of apply-pipeline spans whose ter_result is not tesSUCCESS, split by stage. Shows whether failures concentrate in preflight, preclaim, or apply. Filters on ter_result rather than span status because a failing ter code completes the span normally; only thrown exceptions set an error status.", + "description": "**What:** Rate of pipeline spans ending in a non-success result, split by stage (preflight, preclaim, apply).\n**How it's computed:** Per-second rate of non-success outcomes per stage over a 5-minute window, per node.\n**Reading it:** Shows whether failures concentrate early (basic checks) or late (application).\n**Healthy range:** Low and workload-dependent.\n**Watch for:** A surge of early-stage failures is consistent with malformed or spam transaction floods.\n**Source:** src/libxrpl/tx/Transactor.cpp:Transactor::operator()", "type": "timeseries", "gridPos": { "h": 8, @@ -804,7 +804,7 @@ }, { "title": "Tx Apply Pipeline Latency by Type and Stage (p95)", - "description": "95th-percentile duration broken down by both transaction type and apply-pipeline stage. Shows, for each transaction type, which stage (preflight, preclaim, apply) dominates its latency. Higher cardinality than the by-stage view; filter with the $tx_type and $stage variables.", + "description": "**What:** 95th-percentile stage duration broken down by both transaction type and pipeline stage.\n**How it's computed:** 95th percentile of durations per type and stage over a 5-minute window, per node.\n**Reading it:** For each transaction type, shows which stage dominates its latency.\n**Healthy range:** Sub-millisecond to a few milliseconds; type- and stage-dependent.\n**Watch for:** A specific type-and-stage combination climbing points to a targeted expensive or abusive pattern.\n**Source:** src/libxrpl/tx/Transactor.cpp:Transactor::operator()", "type": "timeseries", "gridPos": { "h": 8, @@ -848,7 +848,7 @@ } ], "schemaVersion": 39, - "tags": ["transactions", "telemetry"], + "tags": ["transactions"], "templating": { "list": [ { @@ -914,7 +914,7 @@ { "name": "node", "label": "Node", - "description": "Filter by rippled node (service.instance.id \u2014 e.g. Node-1)", + "description": "Filter by rippled node (service.instance.id — e.g. Node-1)", "type": "query", "query": "label_values(traces_span_metrics_calls_total, exported_instance)", "datasource": { @@ -1029,7 +1029,7 @@ "from": "now-1h", "to": "now" }, - "description": "Transaction pipeline overview: processing and receive rates, per-type latency, path distribution, apply-stage breakdown, and validation outcomes.", + "description": "Transaction pipeline overview: processing and receive rates, per-type latency and outcomes, queue behaviour, and apply-stage breakdown.", "title": "Transaction Overview", "uid": "xrpld-transactions" }