From d5763420f95106641dddc3d61c0345e05cc65cd4 Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Fri, 7 Aug 2026 11:54:48 +0100 Subject: [PATCH] style(telemetry): lead each dashboard row with its single-value panels Guideline 8 asks for gauges and stats at the top. Seven dashboards had them scattered below charts, so the reader met a wall of time series before the at-a-glance numbers that give those series context. Stats, gauges and bar gauges now come first within each row section. The move is deliberately scoped to inside a section: shifting a panel across a row boundary would change which category it belongs to. Panels keep their relative order otherwise, so the reading sequence within each group is unchanged. Whole panel objects are cut and re-spliced as raw text, so their contents stay byte-identical and only gridPos and id are recomputed. Verified per dashboard: panel count unchanged, no panel lost, every targets block byte-identical, ids still 1..N, and no row section left with a stat below a chart. --- .../grafana/dashboards/consensus-health.json | 80 ++-- .../grafana/dashboards/ledger-data-sync.json | 178 ++++---- .../grafana/dashboards/ledger-operations.json | 90 ++-- .../grafana/dashboards/node-health.json | 414 +++++++++--------- .../grafana/dashboards/rpc-pathfinding.json | 218 ++++----- .../grafana/dashboards/rpc-performance.json | 340 +++++++------- .../dashboards/transaction-overview.json | 130 +++--- 7 files changed, 725 insertions(+), 725 deletions(-) diff --git a/docker/telemetry/grafana/dashboards/consensus-health.json b/docker/telemetry/grafana/dashboards/consensus-health.json index 0c127ddcb3..8bda530684 100644 --- a/docker/telemetry/grafana/dashboards/consensus-health.json +++ b/docker/telemetry/grafana/dashboards/consensus-health.json @@ -87,6 +87,43 @@ "panels": [], "id": 3 }, + { + "title": "Validation Send Rate [$xrpl_network_type]", + "description": "**What:** How often this node issues a validation confirming it has fully validated a ledger.\n**How it's computed:** Per-second rate over a 5-minute window, per node.\n**Reading it:** Should closely track the ledger close rate on a healthy validator.\n**Healthy range:** About one per ledger (~0.25/s on mainnet).\n**Watch for:** Validations lagging closes suggests the node is falling behind on validation.\n**Source:** src/xrpld/app/consensus/RCLConsensus.cpp:RCLConsensus::Adaptor::createValidationSpan", + "type": "stat", + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 10 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "sum by (service_instance_id) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", consensus_mode=~\"$consensus_mode\", span_name=\"consensus.validation.send\"}[$__rate_interval]))", + "interval": "15s", + "legendFormat": "Validations / Sec [{{service_instance_id}}]" + } + ], + "fieldConfig": { + "defaults": { + "unit": "suffix: validations/s" + }, + "overrides": [] + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 4 + }, { "title": "Consensus Proposals Sent Rate", "description": "**What:** How often this node broadcasts its own consensus proposal (its candidate transaction set) to the network.\n**How it's computed:** Per-second rate over a 5-minute window, per node.\n**Reading it:** A proposing validator emits a steady stream, roughly one per consensus round; observing-only nodes emit none.\n**Healthy range:** About one proposal per round (~0.2-0.3/s on mainnet) for a proposing node.\n**Watch for:** A validator that drops to zero has stopped proposing and may be unhealthy or misconfigured.\n**Source:** src/xrpld/app/consensus/RCLConsensus.cpp:RCLConsensus::Adaptor::propose", @@ -95,7 +132,7 @@ "h": 8, "w": 12, "x": 0, - "y": 10 + "y": 14 }, "options": { "tooltip": { @@ -130,7 +167,7 @@ }, "overrides": [] }, - "id": 4 + "id": 5 }, { "title": "Ledger Close Duration", @@ -140,7 +177,7 @@ "h": 8, "w": 12, "x": 12, - "y": 10 + "y": 14 }, "options": { "tooltip": { @@ -175,43 +212,6 @@ }, "overrides": [] }, - "id": 5 - }, - { - "title": "Validation Send Rate [$xrpl_network_type]", - "description": "**What:** How often this node issues a validation confirming it has fully validated a ledger.\n**How it's computed:** Per-second rate over a 5-minute window, per node.\n**Reading it:** Should closely track the ledger close rate on a healthy validator.\n**Healthy range:** About one per ledger (~0.25/s on mainnet).\n**Watch for:** Validations lagging closes suggests the node is falling behind on validation.\n**Source:** src/xrpld/app/consensus/RCLConsensus.cpp:RCLConsensus::Adaptor::createValidationSpan", - "type": "stat", - "gridPos": { - "h": 4, - "w": 12, - "x": 0, - "y": 18 - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "expr": "sum by (service_instance_id) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", consensus_mode=~\"$consensus_mode\", span_name=\"consensus.validation.send\"}[$__rate_interval]))", - "interval": "15s", - "legendFormat": "Validations / Sec [{{service_instance_id}}]" - } - ], - "fieldConfig": { - "defaults": { - "unit": "suffix: validations/s" - }, - "overrides": [] - }, - "repeat": "xrpl_network_type", - "repeatDirection": "h", - "maxPerRow": 2, "id": 6 }, { diff --git a/docker/telemetry/grafana/dashboards/ledger-data-sync.json b/docker/telemetry/grafana/dashboards/ledger-data-sync.json index 3532137aef..7de2692b78 100644 --- a/docker/telemetry/grafana/dashboards/ledger-data-sync.json +++ b/docker/telemetry/grafana/dashboards/ledger-data-sync.json @@ -9,6 +9,67 @@ "id": null, "links": [], "panels": [ + { + "title": "Overlay Traffic Heatmap (All Categories, Bytes In) [$xrpl_network_type]", + "description": "###### What this is:\n*All overlay traffic categories ranked by inbound bytes, giving an at-a-glance view of which message types consume the most receive bandwidth.*\n\n###### How it's computed:\n*Top categories by latest inbound byte value across all traffic categories. Each bar is labelled with its traffic category followed by the node identity; the shared `_bytes_in` suffix is dropped from the category name because the panel already reports inbound bytes.*\n\n###### Reading it:\n*The longest bars are the biggest bandwidth consumers; on a synced node transactions, proposals, and validations usually lead.*\n\n###### Healthy range:\n*workload-dependent.*\n\n###### Watch for:\n*A single ledger-data or fetch category dominating (ongoing sync) or an unexpected category topping the list.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", + "type": "bargauge", + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 0 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "displayMode": "gradient", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(topk(20, {service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", __name__=~\".*_bytes_in\", __name__!~\"total_.*\"}), \"series\", \"$1\", \"__name__\", \"(.*)_bytes_in\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", + "unit": "decbytes", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1048576 + }, + { + "color": "red", + "value": 104857600 + } + ] + } + }, + "overrides": [] + }, + "id": 1, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 + }, { "title": "Ledger Data \u2014 Ledger", "description": "###### What this is:\n*Inbound bytes for ledger-data message categories, split into aggregate get/share plus the transaction-set, transaction-node, and account-state-node sub-types the node receives from peers.*\n\n###### How it's computed:\n*Per-category inbound byte rate per node.*\n\n###### Reading it:\n*Normally low and flat once synced. Account-state-node traffic dominates during state sync; transaction-set-candidate traffic dominates during consensus catch-up.*\n\n###### Healthy range:\n*workload-dependent; low and steady on a synced node.*\n\n###### Watch for:\n*Sustained high account-state or tx-node inbound bytes on a node that should be caught up (repeated re-sync, missing history), or a single peer driving all traffic.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", @@ -17,7 +78,7 @@ "h": 8, "w": 12, "x": 0, - "y": 0 + "y": 4 }, "options": { "tooltip": { @@ -57,7 +118,7 @@ }, "overrides": [] }, - "id": 1 + "id": 2 }, { "title": "Ledger Data \u2014 Transaction", @@ -67,7 +128,7 @@ "h": 8, "w": 12, "x": 12, - "y": 0 + "y": 4 }, "options": { "tooltip": { @@ -119,7 +180,7 @@ }, "overrides": [] }, - "id": 2 + "id": 3 }, { "title": "Ledger Data \u2014 Account State", @@ -129,7 +190,7 @@ "h": 8, "w": 12, "x": 0, - "y": 8 + "y": 12 }, "options": { "tooltip": { @@ -169,7 +230,7 @@ }, "overrides": [] }, - "id": 3 + "id": 4 }, { "title": "Ledger Traffic \u2014 Ledger", @@ -179,7 +240,7 @@ "h": 8, "w": 12, "x": 12, - "y": 8 + "y": 12 }, "options": { "tooltip": { @@ -219,7 +280,7 @@ }, "overrides": [] }, - "id": 4 + "id": 5 }, { "title": "Ledger Traffic \u2014 Transaction", @@ -229,7 +290,7 @@ "h": 8, "w": 12, "x": 0, - "y": 16 + "y": 20 }, "options": { "tooltip": { @@ -281,7 +342,7 @@ }, "overrides": [] }, - "id": 5 + "id": 6 }, { "title": "Ledger Traffic \u2014 Account State", @@ -291,7 +352,7 @@ "h": 8, "w": 12, "x": 12, - "y": 16 + "y": 20 }, "options": { "tooltip": { @@ -331,7 +392,7 @@ }, "overrides": [] }, - "id": 6 + "id": 7 }, { "title": "GetObject \u2014 Ledger", @@ -341,7 +402,7 @@ "h": 8, "w": 12, "x": 0, - "y": 24 + "y": 28 }, "options": { "tooltip": { @@ -381,7 +442,7 @@ }, "overrides": [] }, - "id": 7 + "id": 8 }, { "title": "GetObject \u2014 Transaction", @@ -391,7 +452,7 @@ "h": 8, "w": 12, "x": 12, - "y": 24 + "y": 28 }, "options": { "tooltip": { @@ -443,7 +504,7 @@ }, "overrides": [] }, - "id": 8 + "id": 9 }, { "title": "GetObject \u2014 Account State", @@ -453,7 +514,7 @@ "h": 8, "w": 12, "x": 0, - "y": 32 + "y": 36 }, "options": { "tooltip": { @@ -493,7 +554,7 @@ }, "overrides": [] }, - "id": 9 + "id": 10 }, { "title": "GetObject Messages \u2014 Ledger", @@ -503,7 +564,7 @@ "h": 8, "w": 12, "x": 12, - "y": 32 + "y": 36 }, "options": { "tooltip": { @@ -537,7 +598,7 @@ }, "overrides": [] }, - "id": 10 + "id": 11 }, { "title": "GetObject Messages \u2014 Transaction", @@ -547,7 +608,7 @@ "h": 8, "w": 12, "x": 0, - "y": 40 + "y": 44 }, "options": { "tooltip": { @@ -587,7 +648,7 @@ }, "overrides": [] }, - "id": 11 + "id": 12 }, { "title": "GetObject Messages \u2014 Account State", @@ -597,7 +658,7 @@ "h": 8, "w": 12, "x": 12, - "y": 40 + "y": 44 }, "options": { "tooltip": { @@ -631,7 +692,7 @@ }, "overrides": [] }, - "id": 12 + "id": 13 }, { "title": "GetObject Messages \u2014 Specials", @@ -641,7 +702,7 @@ "h": 8, "w": 12, "x": 0, - "y": 48 + "y": 52 }, "options": { "tooltip": { @@ -687,7 +748,7 @@ }, "overrides": [] }, - "id": 13 + "id": 14 }, { "title": "GetObject \u2014 Specials", @@ -697,7 +758,7 @@ "h": 8, "w": 12, "x": 12, - "y": 48 + "y": 52 }, "options": { "tooltip": { @@ -767,68 +828,7 @@ }, "overrides": [] }, - "id": 14 - }, - { - "title": "Overlay Traffic Heatmap (All Categories, Bytes In) [$xrpl_network_type]", - "description": "###### What this is:\n*All overlay traffic categories ranked by inbound bytes, giving an at-a-glance view of which message types consume the most receive bandwidth.*\n\n###### How it's computed:\n*Top categories by latest inbound byte value across all traffic categories. Each bar is labelled with its traffic category followed by the node identity; the shared `_bytes_in` suffix is dropped from the category name because the panel already reports inbound bytes.*\n\n###### Reading it:\n*The longest bars are the biggest bandwidth consumers; on a synced node transactions, proposals, and validations usually lead.*\n\n###### Healthy range:\n*workload-dependent.*\n\n###### Watch for:\n*A single ledger-data or fetch category dominating (ongoing sync) or an unexpected category topping the list.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`", - "type": "bargauge", - "gridPos": { - "h": 4, - "w": 12, - "x": 0, - "y": 56 - }, - "options": { - "tooltip": { - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - }, - "displayMode": "gradient", - "orientation": "horizontal", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "expr": "label_replace(topk(20, {service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", __name__=~\".*_bytes_in\", __name__!~\"total_.*\"}), \"series\", \"$1\", \"__name__\", \"(.*)_bytes_in\")" - } - ], - "fieldConfig": { - "defaults": { - "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "decbytes", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 1048576 - }, - { - "color": "red", - "value": 104857600 - } - ] - } - }, - "overrides": [] - }, - "id": 15, - "repeat": "xrpl_network_type", - "repeatDirection": "h", - "maxPerRow": 2 + "id": 15 }, { "title": "Sync Diagnostics", diff --git a/docker/telemetry/grafana/dashboards/ledger-operations.json b/docker/telemetry/grafana/dashboards/ledger-operations.json index 4c04a97c66..e4b8f02b3d 100644 --- a/docker/telemetry/grafana/dashboards/ledger-operations.json +++ b/docker/telemetry/grafana/dashboards/ledger-operations.json @@ -58,6 +58,43 @@ "maxPerRow": 2, "id": 2 }, + { + "title": "Ledger Validation Rate [$xrpl_network_type]", + "description": "**What:** How often ledgers reach the trusted-validation quorum and become fully validated.\n**How it's computed:** Per-second rate over a 5-minute window, per node.\n**Reading it:** Should match the build and close rates on a healthy node.\n**Healthy range:** About 0.25/s on mainnet.\n**Watch for:** A shortfall means ledgers are not accumulating enough trusted validations to be accepted.\n**Source:** src/xrpld/app/ledger/detail/LedgerMaster.cpp:LedgerMaster::checkAccept", + "type": "stat", + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 1 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "sum by (service_instance_id) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", span_name=\"ledger.validate\"}[$__rate_interval]))", + "interval": "15s", + "legendFormat": "Validations / Sec [{{service_instance_id}}]" + } + ], + "fieldConfig": { + "defaults": { + "unit": "suffix: ledgers/s" + }, + "overrides": [] + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 3 + }, { "title": "Ledger Build Duration", "description": "**What:** Time to build a full ledger, including applying transactions, writing state, and finalising acceptance.\n**How it's computed:** 95th and 50th percentile of per-build durations over a 5-minute window, per node.\n**Reading it:** Scales with transaction volume and disk I/O.\n**Healthy range:** Tens to a few hundred milliseconds under typical load.\n**Watch for:** Long build times indicate expensive transaction sets or I/O pressure while flushing state.\n**Source:** src/xrpld/app/ledger/detail/BuildLedger.cpp:buildLedgerImpl", @@ -109,43 +146,6 @@ }, "overrides": [] }, - "id": 3 - }, - { - "title": "Ledger Validation Rate [$xrpl_network_type]", - "description": "**What:** How often ledgers reach the trusted-validation quorum and become fully validated.\n**How it's computed:** Per-second rate over a 5-minute window, per node.\n**Reading it:** Should match the build and close rates on a healthy node.\n**Healthy range:** About 0.25/s on mainnet.\n**Watch for:** A shortfall means ledgers are not accumulating enough trusted validations to be accepted.\n**Source:** src/xrpld/app/ledger/detail/LedgerMaster.cpp:LedgerMaster::checkAccept", - "type": "stat", - "gridPos": { - "h": 4, - "w": 12, - "x": 0, - "y": 13 - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "expr": "sum by (service_instance_id) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", span_name=\"ledger.validate\"}[$__rate_interval]))", - "interval": "15s", - "legendFormat": "Validations / Sec [{{service_instance_id}}]" - } - ], - "fieldConfig": { - "defaults": { - "unit": "suffix: ledgers/s" - }, - "overrides": [] - }, - "repeat": "xrpl_network_type", - "repeatDirection": "h", - "maxPerRow": 2, "id": 4 }, { @@ -155,8 +155,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 17 + "x": 12, + "y": 5 }, "options": { "tooltip": { @@ -197,7 +197,7 @@ "h": 1, "w": 24, "x": 0, - "y": 25 + "y": 13 }, "collapsed": false, "panels": [], @@ -211,7 +211,7 @@ "h": 8, "w": 12, "x": 0, - "y": 26 + "y": 14 }, "options": { "tooltip": { @@ -264,7 +264,7 @@ "h": 8, "w": 12, "x": 12, - "y": 26 + "y": 14 }, "options": { "tooltip": { @@ -308,7 +308,7 @@ "h": 1, "w": 24, "x": 0, - "y": 34 + "y": 22 }, "collapsed": false, "panels": [], @@ -322,7 +322,7 @@ "h": 4, "w": 12, "x": 0, - "y": 35 + "y": 23 }, "options": { "tooltip": { @@ -359,7 +359,7 @@ "h": 8, "w": 12, "x": 0, - "y": 39 + "y": 27 }, "options": { "tooltip": { diff --git a/docker/telemetry/grafana/dashboards/node-health.json b/docker/telemetry/grafana/dashboards/node-health.json index f0a560c73a..0e11e71d9d 100644 --- a/docker/telemetry/grafana/dashboards/node-health.json +++ b/docker/telemetry/grafana/dashboards/node-health.json @@ -113,6 +113,90 @@ "maxPerRow": 2, "id": 2 }, + { + "title": "Ledger Fetch Rate [$xrpl_network_type]", + "description": "**What:** Rate at which the node requests ledgers from peers.\n**How it's computed:** Per-second rate of the ledger-fetches counter over 5 minutes.\n**Reading it:** Near zero when synced; rises when catching up or backfilling history.\n**Healthy range:** workload-dependent; low on a synced node.\n**Watch for:** Sustained high fetch rate on a node that should be current (repeatedly missing ledgers).\n**Source:** src/xrpld/app/ledger/detail/InboundLedgers.cpp InboundLedgersImp ctor", + "type": "stat", + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 4 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "rate(ledger_fetches_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[$__rate_interval])", + "legendFormat": "Fetches / Sec [{{service_instance_id}}]" + } + ], + "fieldConfig": { + "defaults": { + "unit": "suffix: fetches/s" + }, + "overrides": [] + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 3 + }, + { + "title": "Ledger History Mismatches [$xrpl_network_type]", + "description": "**What:** Rate of ledger-history hash mismatches: built ledgers whose hash disagrees with the validated hash.\n**How it's computed:** Per-second rate of the history-mismatch counter over 5 minutes.\n**Reading it:** Should be flat at zero.\n**Healthy range:** zero.\n**Watch for:** Any non-zero value (consensus divergence, corrupted history, or database issues).\n**Source:** src/xrpld/telemetry/MetricsRegistry.cpp incrementLedgerHistoryMismatch (caller LedgerHistory.cpp)", + "type": "stat", + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 4 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "rate(ledger_history_mismatch_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[$__rate_interval])", + "legendFormat": "Mismatches / Sec [{{service_instance_id}}]" + } + ], + "fieldConfig": { + "defaults": { + "unit": "suffix: mismatches/s", + "thresholds": { + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.01 + } + ] + } + }, + "overrides": [] + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 4 + }, { "title": "Operating Mode (Time Share)", "description": "**What:** Fraction of time in each operating mode (per-interval): Disconnected, Connected, Syncing, Tracking, Full.\n**How it's computed:** Per-mode duration rate divided by the sum of all five mode rates over the interval; the five shares stack to 100%.\n**Reading it:** Full should fill almost the entire band on a healthy node.\n**Healthy range:** Full near 100%; other modes near 0.\n**Watch for:** A rising share of Syncing, Connected, or Disconnected (instability or repeated resync).\n**Source:** src/xrpld/app/misc/NetworkOPs.cpp NetworkOPsImp::Stats ctor", @@ -121,7 +205,7 @@ "h": 8, "w": 12, "x": 0, - "y": 4 + "y": 8 }, "options": { "tooltip": { @@ -186,7 +270,7 @@ }, "overrides": [] }, - "id": 3 + "id": 5 }, { "title": "Operating Mode Transitions", @@ -196,7 +280,7 @@ "h": 8, "w": 12, "x": 12, - "y": 4 + "y": 8 }, "options": { "tooltip": { @@ -258,7 +342,7 @@ }, "overrides": [] }, - "id": 4 + "id": 6 }, { "title": "I/O Latency", @@ -268,7 +352,7 @@ "h": 8, "w": 12, "x": 0, - "y": 12 + "y": 16 }, "options": { "tooltip": { @@ -309,7 +393,7 @@ }, "overrides": [] }, - "id": 5 + "id": 7 }, { "title": "Job Queue Depth", @@ -319,7 +403,7 @@ "h": 8, "w": 12, "x": 12, - "y": 12 + "y": 16 }, "options": { "tooltip": { @@ -354,90 +438,6 @@ }, "overrides": [] }, - "id": 6 - }, - { - "title": "Ledger Fetch Rate [$xrpl_network_type]", - "description": "**What:** Rate at which the node requests ledgers from peers.\n**How it's computed:** Per-second rate of the ledger-fetches counter over 5 minutes.\n**Reading it:** Near zero when synced; rises when catching up or backfilling history.\n**Healthy range:** workload-dependent; low on a synced node.\n**Watch for:** Sustained high fetch rate on a node that should be current (repeatedly missing ledgers).\n**Source:** src/xrpld/app/ledger/detail/InboundLedgers.cpp InboundLedgersImp ctor", - "type": "stat", - "gridPos": { - "h": 4, - "w": 12, - "x": 0, - "y": 20 - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "expr": "rate(ledger_fetches_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[$__rate_interval])", - "legendFormat": "Fetches / Sec [{{service_instance_id}}]" - } - ], - "fieldConfig": { - "defaults": { - "unit": "suffix: fetches/s" - }, - "overrides": [] - }, - "repeat": "xrpl_network_type", - "repeatDirection": "h", - "maxPerRow": 2, - "id": 7 - }, - { - "title": "Ledger History Mismatches [$xrpl_network_type]", - "description": "**What:** Rate of ledger-history hash mismatches: built ledgers whose hash disagrees with the validated hash.\n**How it's computed:** Per-second rate of the history-mismatch counter over 5 minutes.\n**Reading it:** Should be flat at zero.\n**Healthy range:** zero.\n**Watch for:** Any non-zero value (consensus divergence, corrupted history, or database issues).\n**Source:** src/xrpld/telemetry/MetricsRegistry.cpp incrementLedgerHistoryMismatch (caller LedgerHistory.cpp)", - "type": "stat", - "gridPos": { - "h": 4, - "w": 12, - "x": 12, - "y": 20 - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "expr": "rate(ledger_history_mismatch_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[$__rate_interval])", - "legendFormat": "Mismatches / Sec [{{service_instance_id}}]" - } - ], - "fieldConfig": { - "defaults": { - "unit": "suffix: mismatches/s", - "thresholds": { - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 0.01 - } - ] - } - }, - "overrides": [] - }, - "repeat": "xrpl_network_type", - "repeatDirection": "h", - "maxPerRow": 2, "id": 8 }, { @@ -453,6 +453,112 @@ "panels": [], "id": 9 }, + { + "title": "FullBelowCache Hit Rate [$xrpl_network_type]", + "description": "**What:** Hit-rate percentage of the FullBelowCache.\n**How it's computed:** Current value of the cache hit-rate gauge (0-100%).\n**Reading it:** Higher is better; means the node reuses knowledge of complete subtrees.\n**Healthy range:** above ~50% in steady state.\n**Watch for:** Low hit rate during steady state (cache too small or constant re-acquisition).\n**Source:** include/xrpl/basics/TaggedCache.h TaggedCache::Stats ctor", + "type": "gauge", + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 25 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "node_family_full_below_cache_hit_rate{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}", + "legendFormat": "Hit Rate" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "max": 100, + "thresholds": { + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 25 + }, + { + "color": "green", + "value": 50 + } + ] + } + }, + "overrides": [] + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 10 + }, + { + "title": "Ledger Publish Gap [$xrpl_network_type]", + "description": "**What:** Difference between published and validated ledger ages: how far the publish pipeline trails validation.\n**How it's computed:** Published-ledger age minus validated-ledger age, in seconds.\n**Reading it:** Near zero means publishing keeps up with validation.\n**Healthy range:** within a few seconds of zero.\n**Watch for:** A growing gap (publish backlog, stale data for subscribers).\n**Source:** src/xrpld/app/ledger/LedgerMaster.h LedgerMaster::Stats ctor", + "type": "stat", + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 25 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "ledgermaster_published_ledger_age{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"} - ledgermaster_validated_ledger_age{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}", + "legendFormat": "Publish Gap" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "thresholds": { + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 5 + }, + { + "color": "red", + "value": 10 + } + ] + } + }, + "overrides": [] + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 11 + }, { "title": "Key Jobs Execution Time", "description": "**What:** Execution time of the most important job types (accept/advance ledger, transaction, write objects, heartbeat, sweep, trusted validation/proposal, publish, client RPC, ledger data) at the selected quantile.\n**How it's computed:** Selected quantile of each job's execution-time histogram over 5 minutes.\n**Reading it:** Lower and stable is better; shows where consensus, transaction, and maintenance time goes.\n**Healthy range:** workload-dependent; most jobs in low tens of milliseconds.\n**Watch for:** One job type spiking (a specific bottleneck) or broad increases (overload).\n**Source:** include/xrpl/core/JobTypeData.h JobTypeData ctor", @@ -461,7 +567,7 @@ "h": 8, "w": 12, "x": 0, - "y": 25 + "y": 29 }, "options": { "tooltip": { @@ -565,7 +671,7 @@ }, "overrides": [] }, - "id": 10 + "id": 12 }, { "title": "Key Jobs Dequeue Wait Time", @@ -575,7 +681,7 @@ "h": 8, "w": 12, "x": 12, - "y": 25 + "y": 29 }, "options": { "tooltip": { @@ -679,7 +785,7 @@ }, "overrides": [] }, - "id": 11 + "id": 13 }, { "title": "FullBelowCache Size", @@ -689,7 +795,7 @@ "h": 8, "w": 12, "x": 0, - "y": 33 + "y": 37 }, "options": { "tooltip": { @@ -723,112 +829,6 @@ }, "overrides": [] }, - "id": 12 - }, - { - "title": "FullBelowCache Hit Rate [$xrpl_network_type]", - "description": "**What:** Hit-rate percentage of the FullBelowCache.\n**How it's computed:** Current value of the cache hit-rate gauge (0-100%).\n**Reading it:** Higher is better; means the node reuses knowledge of complete subtrees.\n**Healthy range:** above ~50% in steady state.\n**Watch for:** Low hit rate during steady state (cache too small or constant re-acquisition).\n**Source:** include/xrpl/basics/TaggedCache.h TaggedCache::Stats ctor", - "type": "gauge", - "gridPos": { - "h": 4, - "w": 12, - "x": 0, - "y": 41 - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "expr": "node_family_full_below_cache_hit_rate{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}", - "legendFormat": "Hit Rate" - } - ], - "fieldConfig": { - "defaults": { - "unit": "percent", - "min": 0, - "max": 100, - "thresholds": { - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "yellow", - "value": 25 - }, - { - "color": "green", - "value": 50 - } - ] - } - }, - "overrides": [] - }, - "repeat": "xrpl_network_type", - "repeatDirection": "h", - "maxPerRow": 2, - "id": 13 - }, - { - "title": "Ledger Publish Gap [$xrpl_network_type]", - "description": "**What:** Difference between published and validated ledger ages: how far the publish pipeline trails validation.\n**How it's computed:** Published-ledger age minus validated-ledger age, in seconds.\n**Reading it:** Near zero means publishing keeps up with validation.\n**Healthy range:** within a few seconds of zero.\n**Watch for:** A growing gap (publish backlog, stale data for subscribers).\n**Source:** src/xrpld/app/ledger/LedgerMaster.h LedgerMaster::Stats ctor", - "type": "stat", - "gridPos": { - "h": 4, - "w": 12, - "x": 12, - "y": 41 - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "expr": "ledgermaster_published_ledger_age{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"} - ledgermaster_validated_ledger_age{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}", - "legendFormat": "Publish Gap" - } - ], - "fieldConfig": { - "defaults": { - "unit": "s", - "thresholds": { - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 5 - }, - { - "color": "red", - "value": 10 - } - ] - } - }, - "overrides": [] - }, - "repeat": "xrpl_network_type", - "repeatDirection": "h", - "maxPerRow": 2, "id": 14 }, { @@ -838,8 +838,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 45 + "x": 12, + "y": 37 }, "options": { "tooltip": { @@ -889,7 +889,7 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, + "x": 0, "y": 45 }, "options": { @@ -933,8 +933,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 53 + "x": 12, + "y": 45 }, "options": { "tooltip": { diff --git a/docker/telemetry/grafana/dashboards/rpc-pathfinding.json b/docker/telemetry/grafana/dashboards/rpc-pathfinding.json index 094f864531..32c7868e93 100644 --- a/docker/telemetry/grafana/dashboards/rpc-pathfinding.json +++ b/docker/telemetry/grafana/dashboards/rpc-pathfinding.json @@ -247,112 +247,6 @@ "panels": [], "id": 6 }, - { - "title": "Pathfinding Fast Duration", - "description": "**What:** P95 and P50 execution time of the fast pathfinding search (simplified, favoring speed over completeness).\n**How it's computed:** 95th and 50th percentiles over a 5-minute window.\n**Reading it:** Lower is better; fast pathfinding should stay quick.\n**Healthy range:** workload-dependent; typically well below full pathfinding.\n**Watch for:** Rising fast-path latency (pathfinding load or complex order books).\n**Source:** src/xrpld/rpc/detail/PathRequestManager.h ctor", - "type": "timeseries", - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 22 - }, - "options": { - "tooltip": { - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "expr": "histogram_quantile(0.95, sum by (le, service_instance_id) (rate(pathfind_fast_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", - "interval": "15s", - "legendFormat": "P95 Fast Pathfind [{{service_instance_id}}]" - }, - { - "datasource": { - "type": "prometheus" - }, - "expr": "histogram_quantile(0.5, sum by (le, service_instance_id) (rate(pathfind_fast_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", - "interval": "15s", - "legendFormat": "P50 Fast Pathfind [{{service_instance_id}}]" - } - ], - "fieldConfig": { - "defaults": { - "unit": "ms", - "custom": { - "axisLabel": "Duration (ms)", - "spanNulls": 1800000, - "insertNulls": false, - "showPoints": "auto", - "pointSize": 5, - "lineWidth": 1, - "fillOpacity": 0, - "gradientMode": "none" - } - }, - "overrides": [] - }, - "id": 7 - }, - { - "title": "Pathfinding Full Duration", - "description": "**What:** P95 and P50 execution time of the full, exhaustive pathfinding search.\n**How it's computed:** 95th and 50th percentiles over a 5-minute window.\n**Reading it:** Full pathfinding is heavier and slower than fast mode.\n**Healthy range:** workload-dependent.\n**Watch for:** High or rising full-path latency (expensive path computation under subscription load).\n**Source:** src/xrpld/rpc/detail/PathRequestManager.h ctor", - "type": "timeseries", - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 22 - }, - "options": { - "tooltip": { - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "expr": "histogram_quantile(0.95, sum by (le, service_instance_id) (rate(pathfind_full_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", - "interval": "15s", - "legendFormat": "P95 Full Pathfind [{{service_instance_id}}]" - }, - { - "datasource": { - "type": "prometheus" - }, - "expr": "histogram_quantile(0.5, sum by (le, service_instance_id) (rate(pathfind_full_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", - "interval": "15s", - "legendFormat": "P50 Full Pathfind [{{service_instance_id}}]" - } - ], - "fieldConfig": { - "defaults": { - "unit": "ms", - "custom": { - "axisLabel": "Duration (ms)", - "spanNulls": 1800000, - "insertNulls": false, - "showPoints": "auto", - "pointSize": 5, - "lineWidth": 1, - "fillOpacity": 0, - "gradientMode": "none" - } - }, - "overrides": [] - }, - "id": 8 - }, { "title": "Resource Warnings Rate [$xrpl_network_type]", "description": "**What:** Rate of resource-limit warnings raised when a peer or client exceeds its usage warning threshold.\n**How it's computed:** Per-second rate of the warn counter over 5 minutes.\n**Reading it:** Occasional warnings are normal under load; a rising rate flags aggressive clients.\n**Healthy range:** workload-dependent; low.\n**Watch for:** A climbing warning rate (clients approaching limits, a precursor to drops).\n**Source:** include/xrpl/resource/detail/Logic.h Logic::Stats ctor", @@ -361,7 +255,7 @@ "h": 4, "w": 12, "x": 0, - "y": 30 + "y": 22 }, "options": { "tooltip": { @@ -404,7 +298,7 @@ "repeat": "xrpl_network_type", "repeatDirection": "h", "maxPerRow": 2, - "id": 9 + "id": 7 }, { "title": "Resource Drops Rate [$xrpl_network_type]", @@ -414,7 +308,7 @@ "h": 4, "w": 12, "x": 12, - "y": 30 + "y": 22 }, "options": { "tooltip": { @@ -457,6 +351,112 @@ "repeat": "xrpl_network_type", "repeatDirection": "h", "maxPerRow": 2, + "id": 8 + }, + { + "title": "Pathfinding Fast Duration", + "description": "**What:** P95 and P50 execution time of the fast pathfinding search (simplified, favoring speed over completeness).\n**How it's computed:** 95th and 50th percentiles over a 5-minute window.\n**Reading it:** Lower is better; fast pathfinding should stay quick.\n**Healthy range:** workload-dependent; typically well below full pathfinding.\n**Watch for:** Rising fast-path latency (pathfinding load or complex order books).\n**Source:** src/xrpld/rpc/detail/PathRequestManager.h ctor", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 26 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le, service_instance_id) (rate(pathfind_fast_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", + "interval": "15s", + "legendFormat": "P95 Fast Pathfind [{{service_instance_id}}]" + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "histogram_quantile(0.5, sum by (le, service_instance_id) (rate(pathfind_fast_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", + "interval": "15s", + "legendFormat": "P50 Fast Pathfind [{{service_instance_id}}]" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ms", + "custom": { + "axisLabel": "Duration (ms)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" + } + }, + "overrides": [] + }, + "id": 9 + }, + { + "title": "Pathfinding Full Duration", + "description": "**What:** P95 and P50 execution time of the full, exhaustive pathfinding search.\n**How it's computed:** 95th and 50th percentiles over a 5-minute window.\n**Reading it:** Full pathfinding is heavier and slower than fast mode.\n**Healthy range:** workload-dependent.\n**Watch for:** High or rising full-path latency (expensive path computation under subscription load).\n**Source:** src/xrpld/rpc/detail/PathRequestManager.h ctor", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 26 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le, service_instance_id) (rate(pathfind_full_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", + "interval": "15s", + "legendFormat": "P95 Full Pathfind [{{service_instance_id}}]" + }, + { + "datasource": { + "type": "prometheus" + }, + "expr": "histogram_quantile(0.5, sum by (le, service_instance_id) (rate(pathfind_full_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\"}[5m])))", + "interval": "15s", + "legendFormat": "P50 Full Pathfind [{{service_instance_id}}]" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ms", + "custom": { + "axisLabel": "Duration (ms)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" + } + }, + "overrides": [] + }, "id": 10 }, { diff --git a/docker/telemetry/grafana/dashboards/rpc-performance.json b/docker/telemetry/grafana/dashboards/rpc-performance.json index f2f19ce578..aa8bac5c83 100644 --- a/docker/telemetry/grafana/dashboards/rpc-performance.json +++ b/docker/telemetry/grafana/dashboards/rpc-performance.json @@ -8,96 +8,6 @@ "id": null, "links": [], "panels": [ - { - "title": "RPC Request Rate by Command", - "description": "**What:** Per-command throughput of RPC calls the node serves (e.g. server_info, submit, account_info).\n**How it's computed:** Per-second rate split by command name over a 5-minute window, per node.\n**Reading it:** Shows which API endpoints drive load and how demand shifts over time.\n**Healthy range:** Workload-dependent; varies with client mix.\n**Watch for:** A sudden spike on one command can indicate a runaway client or scripted abuse.\n**Source:** src/xrpld/rpc/detail/RPCHandler.cpp:callMethod", - "type": "timeseries", - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc", - "maxHeight": 600 - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "expr": "sum by (command, service_instance_id) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", command=~\"$command\", span_name=~\"rpc.command.*\"}[$__rate_interval]))", - "interval": "15s", - "legendFormat": "{{command}} [{{service_instance_id}}]" - } - ], - "fieldConfig": { - "defaults": { - "unit": "reqps", - "custom": { - "axisLabel": "Requests / Sec", - "spanNulls": 1800000, - "insertNulls": false, - "showPoints": "auto", - "pointSize": 5, - "lineWidth": 1, - "fillOpacity": 0, - "gradientMode": "none" - } - }, - "overrides": [] - }, - "id": 1 - }, - { - "title": "RPC Latency P95 by Command", - "description": "**What:** 95th-percentile response time for each RPC command.\n**How it's computed:** 95th percentile of per-command durations over a 5-minute window, per node.\n**Reading it:** Read-only lookups should be fast; heavy commands like path finding are naturally slower.\n**Healthy range:** Sub-second for most commands; command-dependent.\n**Watch for:** Rising p95 on a normally fast command signals contention or an expensive query pattern.\n**Source:** src/xrpld/rpc/detail/RPCHandler.cpp:callMethod", - "type": "timeseries", - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 0 - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc", - "maxHeight": 600 - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "expr": "histogram_quantile(0.95, sum by (le, command, service_instance_id) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", command=~\"$command\", span_name=~\"rpc.command.*\"}[5m])))", - "interval": "15s", - "legendFormat": "P95 {{command}} [{{service_instance_id}}]" - } - ], - "fieldConfig": { - "defaults": { - "unit": "ms", - "custom": { - "axisLabel": "Latency (ms)", - "spanNulls": 1800000, - "insertNulls": false, - "showPoints": "auto", - "pointSize": 5, - "lineWidth": 1, - "fillOpacity": 0, - "gradientMode": "none" - } - }, - "overrides": [] - }, - "id": 2 - }, { "title": "RPC Error Rate [$xrpl_network_type]", "description": "**What:** Percentage of each RPC command's calls that ended in an error.\n**How it's computed:** Error calls divided by total calls per command over a 5-minute window, per node.\n**Reading it:** Green below 1%, yellow 1-5%, red above 5%.\n**Healthy range:** Below 1% for healthy commands.\n**Watch for:** A sustained error spike on one command is consistent with a client probing or misusing that endpoint.\n**Source:** src/xrpld/rpc/detail/RPCHandler.cpp:callMethod", @@ -106,7 +16,7 @@ "h": 4, "w": 12, "x": 0, - "y": 8 + "y": 0 }, "options": { "tooltip": { @@ -149,8 +59,172 @@ "repeat": "xrpl_network_type", "repeatDirection": "h", "maxPerRow": 2, + "id": 1 + }, + { + "title": "Top Commands by Volume [$xrpl_network_type]", + "description": "**What:** The ten most-called RPC commands by total invocations recently.\n**How it's computed:** Ranked total counts per command over a 5-minute window, per node.\n**Reading it:** Identifies the hottest API endpoints driving node load.\n**Healthy range:** Workload-dependent.\n**Watch for:** An unexpected command dominating the ranking can indicate scripted abuse or a misbehaving client.\n**Source:** src/xrpld/rpc/detail/RPCHandler.cpp:callMethod", + "type": "bargauge", + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 0 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "topk(10, sum by (command, service_instance_id) (increase(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", command=~\"$command\", span_name=~\"rpc.command.*\"}[$__rate_interval])))", + "interval": "15s", + "legendFormat": "{{command}} [{{service_instance_id}}]" + } + ], + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 2 + }, + { + "title": "WebSocket Message Rate [$xrpl_network_type]", + "description": "**What:** Rate of incoming WebSocket RPC messages the server processes.\n**How it's computed:** Per-second rate over a 5-minute window, per node.\n**Reading it:** Non-zero only when clients connect over WebSocket rather than HTTP.\n**Healthy range:** Zero is normal for HTTP-only deployments; otherwise workload-dependent.\n**Watch for:** An unexpected surge points to a chatty or abusive WebSocket client.\n**Source:** src/xrpld/rpc/detail/ServerHandler.cpp:ServerHandler::processSession", + "type": "stat", + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 4 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "sum by (service_instance_id) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", command=~\"$command\", span_name=\"rpc.ws_message\"}[$__rate_interval]))", + "interval": "15s", + "legendFormat": "WS Messages / Sec [{{service_instance_id}}]" + } + ], + "fieldConfig": { + "defaults": { + "unit": "suffix: messages/s" + }, + "overrides": [] + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, "id": 3 }, + { + "title": "RPC Request Rate by Command", + "description": "**What:** Per-command throughput of RPC calls the node serves (e.g. server_info, submit, account_info).\n**How it's computed:** Per-second rate split by command name over a 5-minute window, per node.\n**Reading it:** Shows which API endpoints drive load and how demand shifts over time.\n**Healthy range:** Workload-dependent; varies with client mix.\n**Watch for:** A sudden spike on one command can indicate a runaway client or scripted abuse.\n**Source:** src/xrpld/rpc/detail/RPCHandler.cpp:callMethod", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc", + "maxHeight": 600 + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "sum by (command, service_instance_id) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", command=~\"$command\", span_name=~\"rpc.command.*\"}[$__rate_interval]))", + "interval": "15s", + "legendFormat": "{{command}} [{{service_instance_id}}]" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "axisLabel": "Requests / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" + } + }, + "overrides": [] + }, + "id": 4 + }, + { + "title": "RPC Latency P95 by Command", + "description": "**What:** 95th-percentile response time for each RPC command.\n**How it's computed:** 95th percentile of per-command durations over a 5-minute window, per node.\n**Reading it:** Read-only lookups should be fast; heavy commands like path finding are naturally slower.\n**Healthy range:** Sub-second for most commands; command-dependent.\n**Watch for:** Rising p95 on a normally fast command signals contention or an expensive query pattern.\n**Source:** src/xrpld/rpc/detail/RPCHandler.cpp:callMethod", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc", + "maxHeight": 600 + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le, command, service_instance_id) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", command=~\"$command\", span_name=~\"rpc.command.*\"}[5m])))", + "interval": "15s", + "legendFormat": "P95 {{command}} [{{service_instance_id}}]" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ms", + "custom": { + "axisLabel": "Latency (ms)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" + } + }, + "overrides": [] + }, + "id": 5 + }, { "title": "RPC Latency Heatmap", "description": "**What:** Distribution of RPC response times over time across all commands.\n**How it's computed:** Counts of requests by duration band in each 5-minute window, per node.\n**Reading it:** A single tight band is healthy; brighter cells mark the common latency.\n**Healthy range:** Concentrated at low latency; command-dependent.\n**Watch for:** Two separate bands (bimodal latency) reveal a slow path affecting some requests.\n**Source:** src/xrpld/rpc/detail/RPCHandler.cpp:callMethod", @@ -159,7 +233,7 @@ "h": 8, "w": 12, "x": 0, - "y": 12 + "y": 16 }, "options": { "tooltip": { @@ -191,7 +265,7 @@ } } }, - "id": 4 + "id": 6 }, { "title": "Overall RPC Throughput", @@ -201,7 +275,7 @@ "h": 8, "w": 12, "x": 12, - "y": 12 + "y": 16 }, "options": { "tooltip": { @@ -244,7 +318,7 @@ }, "overrides": [] }, - "id": 5 + "id": 7 }, { "title": "RPC Success vs Error", @@ -254,7 +328,7 @@ "h": 8, "w": 12, "x": 0, - "y": 20 + "y": 24 }, "options": { "tooltip": { @@ -297,80 +371,6 @@ }, "overrides": [] }, - "id": 6 - }, - { - "title": "Top Commands by Volume [$xrpl_network_type]", - "description": "**What:** The ten most-called RPC commands by total invocations recently.\n**How it's computed:** Ranked total counts per command over a 5-minute window, per node.\n**Reading it:** Identifies the hottest API endpoints driving node load.\n**Healthy range:** Workload-dependent.\n**Watch for:** An unexpected command dominating the ranking can indicate scripted abuse or a misbehaving client.\n**Source:** src/xrpld/rpc/detail/RPCHandler.cpp:callMethod", - "type": "bargauge", - "gridPos": { - "h": 4, - "w": 12, - "x": 0, - "y": 28 - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "expr": "topk(10, sum by (command, service_instance_id) (increase(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", command=~\"$command\", span_name=~\"rpc.command.*\"}[$__rate_interval])))", - "interval": "15s", - "legendFormat": "{{command}} [{{service_instance_id}}]" - } - ], - "fieldConfig": { - "defaults": { - "unit": "none" - }, - "overrides": [] - }, - "repeat": "xrpl_network_type", - "repeatDirection": "h", - "maxPerRow": 2, - "id": 7 - }, - { - "title": "WebSocket Message Rate [$xrpl_network_type]", - "description": "**What:** Rate of incoming WebSocket RPC messages the server processes.\n**How it's computed:** Per-second rate over a 5-minute window, per node.\n**Reading it:** Non-zero only when clients connect over WebSocket rather than HTTP.\n**Healthy range:** Zero is normal for HTTP-only deployments; otherwise workload-dependent.\n**Watch for:** An unexpected surge points to a chatty or abusive WebSocket client.\n**Source:** src/xrpld/rpc/detail/ServerHandler.cpp:ServerHandler::processSession", - "type": "stat", - "gridPos": { - "h": 4, - "w": 12, - "x": 12, - "y": 28 - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "expr": "sum by (service_instance_id) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", command=~\"$command\", span_name=\"rpc.ws_message\"}[$__rate_interval]))", - "interval": "15s", - "legendFormat": "WS Messages / Sec [{{service_instance_id}}]" - } - ], - "fieldConfig": { - "defaults": { - "unit": "suffix: messages/s" - }, - "overrides": [] - }, - "repeat": "xrpl_network_type", - "repeatDirection": "h", - "maxPerRow": 2, "id": 8 }, { diff --git a/docker/telemetry/grafana/dashboards/transaction-overview.json b/docker/telemetry/grafana/dashboards/transaction-overview.json index 27e7254194..565edda59a 100644 --- a/docker/telemetry/grafana/dashboards/transaction-overview.json +++ b/docker/telemetry/grafana/dashboards/transaction-overview.json @@ -8,6 +8,60 @@ "id": null, "links": [], "panels": [ + { + "title": "Transaction Apply Failed Rate [$xrpl_network_type]", + "description": "**What:** Rate of transactor-stage applies that returned a non-success result while building a ledger.\n**How it's computed:** Per-second rate of tx.transactor spans at stage=apply whose ter_result is not tesSUCCESS, per node.\n**Reading it:** Green below 1/s, yellow 1-5/s, red above 5/s.\n**Healthy range:** A steady low non-zero rate is normal (e.g. tefPAST_SEQ, tecUNFUNDED and other benign conflicts).\n**Watch for:** A sustained jump above the normal baseline can indicate malformed transaction floods or a systemic issue.\n**Source:** src/libxrpl/tx/Transactor.cpp:Transactor::operator()", + "type": "stat", + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 0 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc", + "maxHeight": 600 + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "sum by (service_instance_id) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", span_name=\"tx.transactor\", stage=\"apply\", ter_result!~\"tesSUCCESS|\"}[$__rate_interval]))", + "interval": "15s", + "legendFormat": "Failed / Sec [{{service_instance_id}}]" + } + ], + "fieldConfig": { + "defaults": { + "unit": "suffix: transactions/s", + "thresholds": { + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 5 + } + ] + } + }, + "overrides": [] + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 1 + }, { "title": "Transaction Processing Rate", "description": "**What:** Rate of transactions entering the processing pipeline (submitted locally or relayed by peers) versus raw transaction messages arriving from peers before deduplication.\n**How it's computed:** Per-second rate of each over a 5-minute window, per node.\n**Reading it:** The received line sits above the processed line by the volume of duplicates filtered out.\n**Healthy range:** Workload-dependent; scales with network transaction volume.\n**Watch for:** A large and growing gap means heavy duplicate traffic; a processed-rate collapse means submissions are not being handled.\n**Source:** src/xrpld/app/misc/NetworkOPs.cpp:NetworkOPsImp::processTransaction, src/xrpld/overlay/detail/PeerImp.cpp:PeerImp::handleTransaction", @@ -16,7 +70,7 @@ "h": 8, "w": 12, "x": 0, - "y": 0 + "y": 4 }, "options": { "tooltip": { @@ -59,7 +113,7 @@ }, "overrides": [] }, - "id": 1 + "id": 2 }, { "title": "Transaction Processing Latency by Type", @@ -69,7 +123,7 @@ "h": 8, "w": 24, "x": 0, - "y": 8 + "y": 12 }, "options": { "tooltip": { @@ -117,7 +171,7 @@ }, "overrides": [] }, - "id": 2 + "id": 3 }, { "title": "Transaction Path Distribution", @@ -127,7 +181,7 @@ "h": 8, "w": 12, "x": 0, - "y": 16 + "y": 20 }, "options": { "tooltip": { @@ -151,7 +205,7 @@ }, "overrides": [] }, - "id": 3 + "id": 4 }, { "title": "Transaction Receive vs Suppressed", @@ -161,7 +215,7 @@ "h": 8, "w": 12, "x": 12, - "y": 16 + "y": 20 }, "options": { "tooltip": { @@ -196,7 +250,7 @@ }, "overrides": [] }, - "id": 4 + "id": 5 }, { "title": "Transaction Processing Duration Heatmap", @@ -206,7 +260,7 @@ "h": 8, "w": 12, "x": 0, - "y": 24 + "y": 28 }, "options": { "tooltip": { @@ -238,7 +292,7 @@ } } }, - "id": 5 + "id": 6 }, { "title": "Transaction Apply Duration per Ledger", @@ -248,7 +302,7 @@ "h": 8, "w": 12, "x": 12, - "y": 24 + "y": 28 }, "options": { "tooltip": { @@ -291,60 +345,6 @@ }, "overrides": [] }, - "id": 6 - }, - { - "title": "Transaction Apply Failed Rate [$xrpl_network_type]", - "description": "**What:** Rate of transactor-stage applies that returned a non-success result while building a ledger.\n**How it's computed:** Per-second rate of tx.transactor spans at stage=apply whose ter_result is not tesSUCCESS, per node.\n**Reading it:** Green below 1/s, yellow 1-5/s, red above 5/s.\n**Healthy range:** A steady low non-zero rate is normal (e.g. tefPAST_SEQ, tecUNFUNDED and other benign conflicts).\n**Watch for:** A sustained jump above the normal baseline can indicate malformed transaction floods or a systemic issue.\n**Source:** src/libxrpl/tx/Transactor.cpp:Transactor::operator()", - "type": "stat", - "gridPos": { - "h": 4, - "w": 12, - "x": 0, - "y": 32 - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc", - "maxHeight": 600 - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "expr": "sum by (service_instance_id) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", span_name=\"tx.transactor\", stage=\"apply\", ter_result!~\"tesSUCCESS|\"}[$__rate_interval]))", - "interval": "15s", - "legendFormat": "Failed / Sec [{{service_instance_id}}]" - } - ], - "fieldConfig": { - "defaults": { - "unit": "suffix: transactions/s", - "thresholds": { - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 1 - }, - { - "color": "red", - "value": 5 - } - ] - } - }, - "overrides": [] - }, - "repeat": "xrpl_network_type", - "repeatDirection": "h", - "maxPerRow": 2, "id": 7 }, {