diff --git a/docker/telemetry/grafana/dashboards/consensus-health.json b/docker/telemetry/grafana/dashboards/consensus-health.json index e77e207c10..e423e3ec07 100644 --- a/docker/telemetry/grafana/dashboards/consensus-health.json +++ b/docker/telemetry/grafana/dashboards/consensus-health.json @@ -39,11 +39,11 @@ "links": [], "panels": [ { - "title": "Validation Send Rate", + "title": "Validation Send Rate [$xrpl_network_type]", "description": "###### What this is:\n*How often this node issues a ledger validation to the network. Each validation asserts the node fully validated one ledger.*\n\n###### How it's computed:\n*Per-second rate of validation events sent, summed per node over a 5-minute window.*\n\n###### Reading it:\n*A flat line at the network's ledger cadence is normal; higher is not better and lower means the node is validating less often.*\n\n###### Healthy range:\n*Roughly one validation per closed ledger (about 0.2-0.3/s on a healthy chain).*\n\n###### Watch for:\n*A drop to zero on a validator means it stopped validating; a value well below the close rate means validation is lagging.*\n\n###### Keywords:\n- **Ledger validation** *(network event)* \u2014 the second consensus stage where the node confirms a built ledger matches the trusted validator quorum and marks it final.\n- **Ledger close interval** *(network-wide)* \u2014 the network's steady ledger rhythm \u2014 roughly one closed ledger every 3-5 seconds on Mainnet.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[RCLConsensus.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/RCLConsensus.cpp)\n\n###### Function:\n`RCLConsensus::Adaptor::createValidationSpan`\n\n###### References:\n[Ledger validation](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure#validation) \u00b7 [Ledger close interval](https://xrpl.org/docs/concepts/ledgers/ledger-close-times) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledger-validation)", "type": "stat", "gridPos": { - "h": 8, + "h": 4, "w": 12, "x": 0, "y": 0 @@ -71,7 +71,10 @@ }, "overrides": [] }, - "id": 1 + "id": 1, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 }, { "title": "Consensus Mode Over Time", @@ -80,8 +83,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 0 + "x": 0, + "y": 4 }, "options": { "tooltip": { @@ -108,13 +111,29 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] }, "id": 2 }, + { + "title": "Proposals & Validations", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 12 + }, + "collapsed": false, + "panels": [], + "id": 3 + }, { "title": "Consensus Proposals Sent Rate", "description": "###### What this is:\n*How often this node proposes its candidate transaction set to peers during consensus.*\n\n###### How it's computed:\n*Per-second rate of proposal-send events, summed per node over a 5-minute window.*\n\n###### Reading it:\n*A steady line tracking the ledger cadence is healthy; only proposing (validator) nodes show activity.*\n\n###### Healthy range:\n*Steady output in step with the close rate for a proposing node; zero for a non-proposing node.*\n\n###### Watch for:\n*A proposing validator dropping to zero, or erratic spikes suggesting repeated re-proposals within rounds.*\n\n###### Keywords:\n- **Proposal** *(network event)* \u2014 a validator's advertised set of candidate transactions for the next ledger, revised each round.\n- **Consensus** *(network event)* \u2014 the protocol by which validators agree on the next ledger's transaction set and close time.\n- **Ledger close interval** *(network-wide)* \u2014 the network's steady ledger rhythm \u2014 roughly one closed ledger every 3-5 seconds on Mainnet.\n- **Consensus mode** *(per node)* \u2014 the node's role/health in the current round: Proposing, Observing, Wrong Ledger, or Switched Ledger.\n- **Consensus round** *(network event)* \u2014 one propose-and-revise iteration of consensus; several may run before validators converge on a ledger.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[RCLConsensus.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/RCLConsensus.cpp)\n\n###### Function:\n`RCLConsensus::Adaptor::propose`\n\n###### References:\n[Proposal](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Ledger close interval](https://xrpl.org/docs/concepts/ledgers/ledger-close-times) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#proposal)", @@ -123,7 +142,7 @@ "h": 8, "w": 12, "x": 0, - "y": 8 + "y": 13 }, "options": { "tooltip": { @@ -150,12 +169,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] }, - "id": 3 + "id": 4 }, { "title": "Accept vs Close Rate", @@ -165,7 +187,7 @@ "h": 8, "w": 12, "x": 12, - "y": 8 + "y": 13 }, "options": { "tooltip": { @@ -199,12 +221,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] }, - "id": 4 + "id": 5 }, { "title": "Validation vs Close Rate", @@ -214,7 +239,7 @@ "h": 8, "w": 12, "x": 0, - "y": 16 + "y": 21 }, "options": { "tooltip": { @@ -248,12 +273,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] }, - "id": 5 + "id": 6 }, { "title": "Time to Reach Consensus (p50/p95)", @@ -263,7 +291,7 @@ "h": 8, "w": 12, "x": 12, - "y": 16 + "y": 21 }, "options": { "tooltip": { @@ -302,12 +330,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] }, - "id": 6 + "id": 7 }, { "title": "Average Time to Reach Consensus", @@ -317,7 +348,7 @@ "h": 8, "w": 12, "x": 0, - "y": 24 + "y": 29 }, "options": { "tooltip": { @@ -346,12 +377,28 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] }, - "id": 7 + "id": 8 + }, + { + "title": "Consensus Rounds", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 37 + }, + "collapsed": false, + "panels": [], + "id": 9 }, { "title": "Consensus Rounds per Ledger (Establish Count)", @@ -361,7 +408,7 @@ "h": 8, "w": 24, "x": 0, - "y": 32 + "y": 38 }, "options": { "tooltip": { @@ -395,13 +442,16 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" }, "decimals": 0 }, "overrides": [] }, - "id": 8 + "id": 10 }, { "title": "Previous Round Time per Ledger", @@ -411,7 +461,7 @@ "h": 8, "w": 12, "x": 0, - "y": 40 + "y": 46 }, "options": { "tooltip": { @@ -440,12 +490,155 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] }, - "id": 9 + "id": 11 + }, + { + "title": "Timing", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 54 + }, + "collapsed": false, + "panels": [], + "id": 12 + }, + { + "title": "Close-Time Resolution Change (per Round) [$xrpl_network_type]", + "description": "###### What this is:\n*Whether the close-time rounding granularity moved Coarser, Finer, or stayed Steady versus the previous round.*\n\n###### How it's computed:\n*Rounds are counted over the window and grouped by direction of change.*\n\n###### Reading it:\n*Mostly Steady is healthy; Coarser means widening bins (more disagreement), Finer means tightening.*\n\n###### Healthy range:\n*Predominantly Steady.*\n\n###### Watch for:\n*Frequent Coarser shifts indicate the network is repeatedly failing to agree on close time.*\n\n###### Keywords:\n- **Close-time resolution** *(network event)* \u2014 the granularity (in seconds) that close times are rounded to; widens when validators disagree.\n- **Consensus round** *(network event)* \u2014 one propose-and-revise iteration of consensus; several may run before validators converge on a ledger.\n- **Close time** *(network event)* \u2014 the timestamp validators agree to stamp on a ledger, rounded to a shared resolution.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as trace spans and stored in Tempo; the value is computed directly from those raw spans by the panel's TraceQL query (no Prometheus metric involved).*\n\n###### Source:\n[RCLConsensus.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/RCLConsensus.cpp)\n\n###### Function:\n`RCLConsensus::Adaptor::doAccept`\n\n###### References:\n[Close-time resolution](https://xrpl.org/docs/concepts/ledgers/ledger-close-times) \u00b7 [Consensus round](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#close-time-resolution)", + "type": "bargauge", + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 55 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "fixed", + "fixedColor": "blue" + } + }, + "overrides": [] + }, + "options": { + "orientation": "horizontal", + "displayMode": "gradient", + "showUnfilled": true, + "reduceOptions": { + "calcs": ["sum"], + "fields": "", + "values": false + }, + "legend": { + "showLegend": false + }, + "tooltip": {} + }, + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "${DS_TEMPO}" + }, + "queryType": "traceql", + "query": "{name=\"consensus.accept.apply\" && resource.service.instance.id=~\"$node\" && resource.service.name=~\"$service_name\" && resource.deployment.environment=~\"$deployment_environment\" && resource.xrpl.network.type=~\"$xrpl_network_type\" && span.resolution_direction=~\"$resolution_direction\"} | count_over_time() by (span.resolution_direction)", + "legendFormat": "{{span.resolution_direction}}", + "refId": "A" + } + ], + "id": 13, + "transformations": [ + { + "id": "renameByRegex", + "options": { + "regex": "increased", + "renamePattern": "Coarser (more disagreement)" + } + }, + { + "id": "renameByRegex", + "options": { + "regex": "decreased", + "renamePattern": "Finer (better agreement)" + } + }, + { + "id": "renameByRegex", + "options": { + "regex": "unchanged", + "renamePattern": "Steady" + } + } + ], + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 + }, + { + "title": "Close-Time Proposal Spread (Distinct Positions per Round) [$xrpl_network_type]", + "description": "###### What this is:\n*How spread out validators' proposed close times were, as the number of distinct positions per round.*\n\n###### How it's computed:\n*Rounds are counted over the window and grouped by their distinct-position count; each bar is one count value.*\n\n###### Reading it:\n*Weight on '1 distinct position' means everyone agreed; weight on 2 or more means the proposals split.*\n\n###### Healthy range:\n*Most rounds at a single distinct position.*\n\n###### Watch for:\n*A shift toward two or more distinct positions signals growing clock drift or latency across validators.*\n\n###### Keywords:\n- **Distinct positions** *(network event)* \u2014 the number of different close-time values validators proposed in a round; one means full agreement.\n- **Consensus round** *(network event)* \u2014 one propose-and-revise iteration of consensus; several may run before validators converge on a ledger.\n- **Proposal** *(network event)* \u2014 a validator's advertised set of candidate transactions for the next ledger, revised each round.\n- **Clock drift** *(network event)* \u2014 differences between validators' clocks, which spread their proposed close times.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as trace spans and stored in Tempo; the value is computed directly from those raw spans by the panel's TraceQL query (no Prometheus metric involved).*\n\n###### Source:\n[RCLConsensus.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/RCLConsensus.cpp)\n\n###### Function:\n`RCLConsensus::Adaptor::doAccept`\n\n###### References:\n[Distinct positions](https://xrpl.org/docs/concepts/ledgers/ledger-close-times) \u00b7 [Consensus round](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#distinct-positions)", + "type": "bargauge", + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 55 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "fixed", + "fixedColor": "blue" + } + }, + "overrides": [] + }, + "options": { + "orientation": "horizontal", + "displayMode": "gradient", + "showUnfilled": true, + "reduceOptions": { + "calcs": ["sum"], + "fields": "", + "values": false + }, + "legend": { + "showLegend": false + }, + "tooltip": {} + }, + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "${DS_TEMPO}" + }, + "queryType": "traceql", + "query": "{name=\"consensus.accept.apply\" && resource.service.instance.id=~\"$node\" && resource.service.name=~\"$service_name\" && resource.deployment.environment=~\"$deployment_environment\" && resource.xrpl.network.type=~\"$xrpl_network_type\"} | count_over_time() by (span.close_time_vote_bins)", + "legendFormat": "{{span.close_time_vote_bins}} Distinct Position(s)", + "refId": "A" + } + ], + "id": 14, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 }, { "title": "Position Update Duration", @@ -454,8 +647,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 40 + "x": 0, + "y": 59 }, "options": { "tooltip": { @@ -482,12 +675,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] }, - "id": 10 + "id": 15 }, { "title": "Ledger Close Duration", @@ -496,8 +692,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 48 + "x": 12, + "y": 59 }, "options": { "tooltip": { @@ -524,12 +720,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] }, - "id": 11 + "id": 16 }, { "title": "Ledger Apply Duration (doAccept)", @@ -538,8 +737,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 48 + "x": 0, + "y": 67 }, "targets": [ { @@ -559,12 +758,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] }, - "id": 12, + "id": 17, "options": { "tooltip": { "mode": "multi", @@ -580,8 +782,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 56 + "x": 12, + "y": 67 }, "options": { "tooltip": { @@ -610,7 +812,7 @@ "unit": "ms" } }, - "id": 13 + "id": 18 }, { "title": "Close-Time Agreement Rate (Agreed vs Disagreed)", @@ -619,8 +821,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 56 + "x": 0, + "y": 75 }, "targets": [ { @@ -640,12 +842,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] }, - "id": 14, + "id": 19, "options": { "tooltip": { "mode": "multi", @@ -661,8 +866,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 64 + "x": 12, + "y": 75 }, "fieldConfig": { "defaults": { @@ -672,7 +877,10 @@ "pointSize": 5, "showPoints": "auto", "axisLabel": "Count / Milliseconds", - "spanNulls": 1800000 + "spanNulls": 1800000, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" }, "unit": "short" }, @@ -745,128 +953,20 @@ "refId": "B" } ], - "id": 15 + "id": 20 }, { - "title": "Close-Time Resolution Change (per Round)", - "description": "###### What this is:\n*Whether the close-time rounding granularity moved Coarser, Finer, or stayed Steady versus the previous round.*\n\n###### How it's computed:\n*Rounds are counted over the window and grouped by direction of change.*\n\n###### Reading it:\n*Mostly Steady is healthy; Coarser means widening bins (more disagreement), Finer means tightening.*\n\n###### Healthy range:\n*Predominantly Steady.*\n\n###### Watch for:\n*Frequent Coarser shifts indicate the network is repeatedly failing to agree on close time.*\n\n###### Keywords:\n- **Close-time resolution** *(network event)* \u2014 the granularity (in seconds) that close times are rounded to; widens when validators disagree.\n- **Consensus round** *(network event)* \u2014 one propose-and-revise iteration of consensus; several may run before validators converge on a ledger.\n- **Close time** *(network event)* \u2014 the timestamp validators agree to stamp on a ledger, rounded to a shared resolution.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as trace spans and stored in Tempo; the value is computed directly from those raw spans by the panel's TraceQL query (no Prometheus metric involved).*\n\n###### Source:\n[RCLConsensus.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/RCLConsensus.cpp)\n\n###### Function:\n`RCLConsensus::Adaptor::doAccept`\n\n###### References:\n[Close-time resolution](https://xrpl.org/docs/concepts/ledgers/ledger-close-times) \u00b7 [Consensus round](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#close-time-resolution)", - "type": "bargauge", + "title": "Outcomes & Failures", + "type": "row", "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 64 - }, - "fieldConfig": { - "defaults": { - "unit": "short", - "decimals": 0, - "color": { - "mode": "fixed", - "fixedColor": "blue" - } - }, - "overrides": [] - }, - "options": { - "orientation": "horizontal", - "displayMode": "gradient", - "showUnfilled": true, - "reduceOptions": { - "calcs": ["sum"], - "fields": "", - "values": false - }, - "legend": { - "showLegend": false - }, - "tooltip": {} - }, - "targets": [ - { - "datasource": { - "type": "tempo", - "uid": "${DS_TEMPO}" - }, - "queryType": "traceql", - "query": "{name=\"consensus.accept.apply\" && resource.service.instance.id=~\"$node\" && resource.service.name=~\"$service_name\" && resource.deployment.environment=~\"$deployment_environment\" && resource.xrpl.network.type=~\"$xrpl_network_type\" && span.resolution_direction=~\"$resolution_direction\"} | count_over_time() by (span.resolution_direction)", - "legendFormat": "{{span.resolution_direction}}", - "refId": "A" - } - ], - "id": 16, - "transformations": [ - { - "id": "renameByRegex", - "options": { - "regex": "increased", - "renamePattern": "Coarser (more disagreement)" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "decreased", - "renamePattern": "Finer (better agreement)" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "unchanged", - "renamePattern": "Steady" - } - } - ] - }, - { - "title": "Close-Time Proposal Spread (Distinct Positions per Round)", - "description": "###### What this is:\n*How spread out validators' proposed close times were, as the number of distinct positions per round.*\n\n###### How it's computed:\n*Rounds are counted over the window and grouped by their distinct-position count; each bar is one count value.*\n\n###### Reading it:\n*Weight on '1 distinct position' means everyone agreed; weight on 2 or more means the proposals split.*\n\n###### Healthy range:\n*Most rounds at a single distinct position.*\n\n###### Watch for:\n*A shift toward two or more distinct positions signals growing clock drift or latency across validators.*\n\n###### Keywords:\n- **Distinct positions** *(network event)* \u2014 the number of different close-time values validators proposed in a round; one means full agreement.\n- **Consensus round** *(network event)* \u2014 one propose-and-revise iteration of consensus; several may run before validators converge on a ledger.\n- **Proposal** *(network event)* \u2014 a validator's advertised set of candidate transactions for the next ledger, revised each round.\n- **Clock drift** *(network event)* \u2014 differences between validators' clocks, which spread their proposed close times.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as trace spans and stored in Tempo; the value is computed directly from those raw spans by the panel's TraceQL query (no Prometheus metric involved).*\n\n###### Source:\n[RCLConsensus.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/RCLConsensus.cpp)\n\n###### Function:\n`RCLConsensus::Adaptor::doAccept`\n\n###### References:\n[Distinct positions](https://xrpl.org/docs/concepts/ledgers/ledger-close-times) \u00b7 [Consensus round](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#distinct-positions)", - "type": "bargauge", - "gridPos": { - "h": 8, - "w": 12, + "h": 1, + "w": 24, "x": 0, - "y": 72 + "y": 83 }, - "fieldConfig": { - "defaults": { - "unit": "short", - "decimals": 0, - "color": { - "mode": "fixed", - "fixedColor": "blue" - } - }, - "overrides": [] - }, - "options": { - "orientation": "horizontal", - "displayMode": "gradient", - "showUnfilled": true, - "reduceOptions": { - "calcs": ["sum"], - "fields": "", - "values": false - }, - "legend": { - "showLegend": false - }, - "tooltip": {} - }, - "targets": [ - { - "datasource": { - "type": "tempo", - "uid": "${DS_TEMPO}" - }, - "queryType": "traceql", - "query": "{name=\"consensus.accept.apply\" && resource.service.instance.id=~\"$node\" && resource.service.name=~\"$service_name\" && resource.deployment.environment=~\"$deployment_environment\" && resource.xrpl.network.type=~\"$xrpl_network_type\"} | count_over_time() by (span.close_time_vote_bins)", - "legendFormat": "{{span.close_time_vote_bins}} Distinct Position(s)", - "refId": "A" - } - ], - "id": 17 + "collapsed": false, + "panels": [], + "id": 21 }, { "title": "Consensus Outcome Distribution (per Round)", @@ -876,7 +976,7 @@ "h": 8, "w": 24, "x": 0, - "y": 80 + "y": 84 }, "options": { "legend": { @@ -905,7 +1005,7 @@ }, "overrides": [] }, - "id": 18 + "id": 22 }, { "title": "Consensus Failures Over Time", @@ -915,7 +1015,7 @@ "h": 8, "w": 12, "x": 0, - "y": 88 + "y": 92 }, "options": { "tooltip": { @@ -949,12 +1049,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] }, - "id": 19 + "id": 23 }, { "title": "Consensus Stall Rate", @@ -964,7 +1067,7 @@ "h": 8, "w": 12, "x": 12, - "y": 88 + "y": 92 }, "options": { "tooltip": { @@ -998,12 +1101,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] }, - "id": 20 + "id": 24 }, { "title": "Consensus Mode-Change Rate by Target Mode", @@ -1013,7 +1119,7 @@ "h": 8, "w": 12, "x": 0, - "y": 96 + "y": 100 }, "options": { "tooltip": { @@ -1040,12 +1146,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] }, - "id": 21 + "id": 25 }, { "title": "Ledger History Mismatch Rate by Reason", @@ -1055,7 +1164,7 @@ "h": 8, "w": 12, "x": 12, - "y": 96 + "y": 100 }, "options": { "tooltip": { @@ -1082,12 +1191,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] }, - "id": 22 + "id": 26 } ], "schemaVersion": 39, diff --git a/docker/telemetry/grafana/dashboards/fee-market.json b/docker/telemetry/grafana/dashboards/fee-market.json index 8d1cc6bfd8..ab4b502f04 100644 --- a/docker/telemetry/grafana/dashboards/fee-market.json +++ b/docker/telemetry/grafana/dashboards/fee-market.json @@ -39,6 +39,19 @@ "id": null, "links": [], "panels": [ + { + "title": "Queue Depth & Throughput", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "collapsed": false, + "panels": [], + "id": 1 + }, { "title": "Transaction Queue Depth", "description": "###### What this is:\n*Transactions currently waiting in the transaction queue versus the queue's maximum capacity.*\n\n###### How it's computed:\n*Instantaneous gauge readings of current queue count and configured max size.*\n\n###### Reading it:\n*Queue depth well below capacity is normal; depth approaching capacity means the node is saturating.*\n\n###### Healthy range:\n*Depth near 0 in quiet periods; workload-dependent under load.*\n\n###### Watch for:\n*Depth pinned at capacity for sustained periods, which signals demand exceeding throughput or a fee-spam burst.*\n\n###### Keywords:\n- **Transaction queue (TxQ)** *(per node)* \u2014 holds transactions that meet local cost but not the open-ledger cost, to include in a later ledger.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerTxqGauge`\n\n###### References:\n[Transaction queue (TxQ)](https://xrpl.org/docs/concepts/transactions/transaction-queue) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#transaction-queue-txq)", @@ -47,7 +60,7 @@ "h": 8, "w": 12, "x": 0, - "y": 0 + "y": 1 }, "options": { "tooltip": { @@ -79,16 +92,20 @@ "custom": { "axisLabel": "Transactions", "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, - "spanNulls": 1800000 + "lineWidth": 1, + "fillOpacity": 0, + "spanNulls": 1800000, + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" - } + }, + "decimals": 0 }, "overrides": [] - } + }, + "id": 2 }, { "title": "Transactions Per Ledger", @@ -98,7 +115,7 @@ "h": 8, "w": 12, "x": 12, - "y": 0 + "y": 1 }, "options": { "tooltip": { @@ -130,16 +147,32 @@ "custom": { "axisLabel": "Transactions", "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, - "spanNulls": 1800000 + "lineWidth": 1, + "fillOpacity": 0, + "spanNulls": 1800000, + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" } }, "overrides": [] - } + }, + "id": 3 + }, + { + "title": "Fee Escalation", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "collapsed": false, + "panels": [], + "id": 4 }, { "title": "Fee Escalation Levels", @@ -147,9 +180,9 @@ "type": "timeseries", "gridPos": { "h": 8, - "w": 24, + "w": 12, "x": 0, - "y": 8 + "y": 10 }, "options": { "tooltip": { @@ -195,20 +228,23 @@ "custom": { "axisLabel": "Fee Level", "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 5, + "lineWidth": 1, + "fillOpacity": 0, "scaleDistribution": { "type": "log", "log": 2 }, - "spanNulls": 1800000 + "spanNulls": 1800000, + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" } }, "overrides": [] - } + }, + "id": 5 }, { "title": "Load Factor Breakdown", @@ -217,8 +253,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 16 + "x": 12, + "y": 10 }, "options": { "tooltip": { @@ -264,9 +300,11 @@ "custom": { "axisLabel": "Multiplier", "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 5, - "spanNulls": 1800000 + "lineWidth": 1, + "fillOpacity": 0, + "spanNulls": 1800000, + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -289,7 +327,8 @@ } }, "overrides": [] - } + }, + "id": 6 }, { "title": "Load Factor Components", @@ -298,8 +337,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 16 + "x": 0, + "y": 18 }, "options": { "tooltip": { @@ -338,16 +377,32 @@ "custom": { "axisLabel": "Multiplier", "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 5, - "spanNulls": 1800000 + "lineWidth": 1, + "fillOpacity": 0, + "spanNulls": 1800000, + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" } }, "overrides": [] - } + }, + "id": 7 + }, + { + "title": "Queue Rejections", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 26 + }, + "collapsed": false, + "panels": [], + "id": 8 }, { "title": "Queue Abandonment Rate (Expired)", @@ -357,7 +412,7 @@ "h": 8, "w": 12, "x": 0, - "y": 24 + "y": 27 }, "options": { "tooltip": { @@ -384,11 +439,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 9 }, { "title": "Queue Admission Rejections (Dropped)", @@ -398,7 +457,7 @@ "h": 8, "w": 12, "x": 12, - "y": 24 + "y": 27 }, "options": { "tooltip": { @@ -425,11 +484,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 10 } ], "schemaVersion": 39, diff --git a/docker/telemetry/grafana/dashboards/job-queue.json b/docker/telemetry/grafana/dashboards/job-queue.json index c7e745e7e9..f509b39f60 100644 --- a/docker/telemetry/grafana/dashboards/job-queue.json +++ b/docker/telemetry/grafana/dashboards/job-queue.json @@ -10,14 +10,27 @@ "links": [], "panels": [ { - "title": "Current Job Latency (p99 Gauge)", + "title": "Current State", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "collapsed": false, + "panels": [], + "id": 1 + }, + { + "title": "Current Job Latency (p99 Gauge) [$xrpl_network_type]", "description": "###### What this is:\n*At-a-glance p99 of how long jobs wait in the queue and how long they run once started.*\n\n###### How it's computed:\n*99th percentile derived from the job wait-time and run-time histograms over the last 5 minutes.*\n\n###### Reading it:\n*Lower is better; green under 100ms, yellow to 1s, red beyond 1s.*\n\n###### Healthy range:\n*Wait and exec p99 under 100ms.*\n\n###### Watch for:\n*p99 wait climbing into the red, meaning worker threads are saturated and jobs are backing up.*\n\n###### Keywords:\n- **Job queue / job type** *(per node)* \u2014 xrpld's worker-thread pool; every unit of background work is enqueued under a named job type.\n- **Deferred job** *(per node)* \u2014 a job held back because its type is already at its concurrency limit; the leading indicator of queue backpressure.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordJobStarted / recordJobFinished`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#job-queue-job-type)", "type": "gauge", "gridPos": { - "h": 8, + "h": 4, "w": 12, "x": 0, - "y": 0 + "y": 1 }, "options": { "reduceOptions": { @@ -68,7 +81,23 @@ }, "overrides": [] }, - "id": 1 + "id": 2, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 + }, + { + "title": "Throughput", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "collapsed": false, + "panels": [], + "id": 3 }, { "title": "Job Throughput Rate (Per Second)", @@ -76,9 +105,9 @@ "type": "timeseries", "gridPos": { "h": 8, - "w": 24, + "w": 12, "x": 0, - "y": 8 + "y": 6 }, "options": { "tooltip": { @@ -113,13 +142,14 @@ "unit": "suffix: jobs/s", "custom": { "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, + "lineWidth": 1, + "fillOpacity": 0, "axisLabel": "Jobs / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -127,7 +157,7 @@ }, "overrides": [] }, - "id": 2 + "id": 4 }, { "title": "Per-Job-Type Queued Rate", @@ -137,7 +167,7 @@ "h": 8, "w": 24, "x": 0, - "y": 16 + "y": 14 }, "options": { "tooltip": { @@ -166,12 +196,13 @@ "custom": { "drawStyle": "line", "lineWidth": 1, - "fillOpacity": 5, + "fillOpacity": 0, "axisLabel": "Jobs / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -179,7 +210,7 @@ }, "overrides": [] }, - "id": 3 + "id": 5 }, { "title": "Per-Job-Type Finish Rate", @@ -189,7 +220,7 @@ "h": 8, "w": 24, "x": 0, - "y": 24 + "y": 22 }, "options": { "tooltip": { @@ -218,12 +249,13 @@ "custom": { "drawStyle": "line", "lineWidth": 1, - "fillOpacity": 5, + "fillOpacity": 0, "axisLabel": "Jobs / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -231,7 +263,20 @@ }, "overrides": [] }, - "id": 4 + "id": 6 + }, + { + "title": "Latency", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 30 + }, + "collapsed": false, + "panels": [], + "id": 7 }, { "title": "Job Queue Wait Time", @@ -241,7 +286,7 @@ "h": 8, "w": 12, "x": 0, - "y": 32 + "y": 31 }, "options": { "tooltip": { @@ -270,13 +315,14 @@ "unit": "\u00b5s", "custom": { "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 5, + "lineWidth": 1, + "fillOpacity": 0, "axisLabel": "Duration (\u00b5s)", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -284,7 +330,7 @@ }, "overrides": [] }, - "id": 5 + "id": 8 }, { "title": "Job Execution Time", @@ -294,7 +340,7 @@ "h": 8, "w": 12, "x": 12, - "y": 32 + "y": 31 }, "options": { "tooltip": { @@ -323,13 +369,14 @@ "unit": "\u00b5s", "custom": { "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 5, + "lineWidth": 1, + "fillOpacity": 0, "axisLabel": "Duration (\u00b5s)", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -337,7 +384,7 @@ }, "overrides": [] }, - "id": 6 + "id": 9 }, { "title": "Per-Job-Type Execution Time (p99)", @@ -347,7 +394,7 @@ "h": 8, "w": 24, "x": 0, - "y": 40 + "y": 39 }, "options": { "tooltip": { @@ -376,12 +423,13 @@ "custom": { "drawStyle": "line", "lineWidth": 1, - "fillOpacity": 5, + "fillOpacity": 0, "axisLabel": "Duration (\u00b5s)", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -389,7 +437,20 @@ }, "overrides": [] }, - "id": 7 + "id": 10 + }, + { + "title": "Overflow", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 47 + }, + "collapsed": false, + "panels": [], + "id": 11 }, { "title": "Transaction Overflow Rate", @@ -397,7 +458,7 @@ "type": "timeseries", "gridPos": { "h": 8, - "w": 24, + "w": 12, "x": 0, "y": 48 }, @@ -439,12 +500,13 @@ "custom": { "axisLabel": "Overflows / Min", "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, + "lineWidth": 1, + "fillOpacity": 0, "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -452,7 +514,7 @@ }, "overrides": [] }, - "id": 8 + "id": 12 } ], "schemaVersion": 39, diff --git a/docker/telemetry/grafana/dashboards/ledger-data-sync.json b/docker/telemetry/grafana/dashboards/ledger-data-sync.json index a31c2e87b8..e45ecef233 100644 --- a/docker/telemetry/grafana/dashboards/ledger-data-sync.json +++ b/docker/telemetry/grafana/dashboards/ledger-data-sync.json @@ -9,6 +9,67 @@ "id": null, "links": [], "panels": [ + { + "title": "Overlay Traffic Heatmap (All Categories, Bytes In) [$xrpl_network_type]", + "description": "###### What this is:\n*All overlay traffic categories ranked by inbound bytes, giving an at-a-glance view of which message types consume the most receive bandwidth.*\n\n###### How it's computed:\n*Top categories by latest inbound byte value across all traffic categories. Each bar is labelled with its traffic category followed by the node identity; the shared `_bytes_in` suffix is dropped from the category name because the panel already reports inbound bytes.*\n\n###### Reading it:\n*The longest bars are the biggest bandwidth consumers; on a synced node transactions, proposals, and validations usually lead.*\n\n###### Healthy range:\n*workload-dependent.*\n\n###### Watch for:\n*A single ledger-data or fetch category dominating (ongoing sync) or an unexpected category topping the list.*\n\n###### Keywords:\n- **Overlay** *(network-wide)* \u2014 the peer-to-peer mesh xrpld nodes form to gossip transactions, proposals and validations.\n- **Peer** *(per node)* \u2014 another server this node holds a protocol connection to.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#overlay)", + "type": "bargauge", + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 0 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "displayMode": "gradient", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(topk(20, {service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", __name__=~\".*_bytes_in\", __name__!~\"total_.*\"}), \"series\", \"$1\", \"__name__\", \"(.*)_bytes_in\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", + "unit": "decbytes", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1048576 + }, + { + "color": "red", + "value": 104857600 + } + ] + } + }, + "overrides": [] + }, + "id": 1, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 + }, { "title": "Ledger Data \u2014 Ledger", "description": "###### What this is:\n*Inbound bytes for ledger-data message categories, split into aggregate get/share plus the transaction-set, transaction-node, and account-state-node sub-types the node receives from peers.*\n\n###### How it's computed:\n*Per-category inbound byte rate per node.*\n\n###### Reading it:\n*Normally low and flat once synced. Account-state-node traffic dominates during state sync; transaction-set-candidate traffic dominates during consensus catch-up.*\n\n###### Healthy range:\n*workload-dependent; low and steady on a synced node.*\n\n###### Watch for:\n*Sustained high account-state or tx-node inbound bytes on a node that should be caught up (repeated re-sync, missing history), or a single peer driving all traffic.*\n\n###### Keywords:\n- **Ledger data** *(network event)* \u2014 the bulk transfer of ledger contents between peers during sync.\n- **Overlay** *(network-wide)* \u2014 the peer-to-peer mesh xrpld nodes form to gossip transactions, proposals and validations.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledger-data)", @@ -17,7 +78,7 @@ "h": 8, "w": 12, "x": 0, - "y": 0 + "y": 4 }, "options": { "tooltip": { @@ -49,12 +110,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] }, - "id": 1 + "id": 2 }, { "title": "Ledger Data \u2014 Transaction", @@ -64,7 +128,7 @@ "h": 8, "w": 12, "x": 12, - "y": 0 + "y": 4 }, "options": { "tooltip": { @@ -108,12 +172,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] }, - "id": 2 + "id": 3 }, { "title": "Ledger Data \u2014 Account State", @@ -123,7 +190,7 @@ "h": 8, "w": 12, "x": 0, - "y": 8 + "y": 12 }, "options": { "tooltip": { @@ -155,12 +222,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] }, - "id": 3 + "id": 4 }, { "title": "Ledger Traffic \u2014 Ledger", @@ -170,7 +240,7 @@ "h": 8, "w": 12, "x": 12, - "y": 8 + "y": 12 }, "options": { "tooltip": { @@ -202,12 +272,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] }, - "id": 4 + "id": 5 }, { "title": "Ledger Traffic \u2014 Transaction", @@ -217,7 +290,7 @@ "h": 8, "w": 12, "x": 0, - "y": 16 + "y": 20 }, "options": { "tooltip": { @@ -261,12 +334,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] }, - "id": 5 + "id": 6 }, { "title": "Ledger Traffic \u2014 Account State", @@ -276,7 +352,7 @@ "h": 8, "w": 12, "x": 12, - "y": 16 + "y": 20 }, "options": { "tooltip": { @@ -308,12 +384,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] }, - "id": 6 + "id": 7 }, { "title": "GetObject \u2014 Ledger", @@ -323,7 +402,7 @@ "h": 8, "w": 12, "x": 0, - "y": 24 + "y": 28 }, "options": { "tooltip": { @@ -355,12 +434,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] }, - "id": 7 + "id": 8 }, { "title": "GetObject \u2014 Transaction", @@ -370,7 +452,7 @@ "h": 8, "w": 12, "x": 12, - "y": 24 + "y": 28 }, "options": { "tooltip": { @@ -414,12 +496,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] }, - "id": 8 + "id": 9 }, { "title": "GetObject \u2014 Account State", @@ -429,7 +514,7 @@ "h": 8, "w": 12, "x": 0, - "y": 32 + "y": 36 }, "options": { "tooltip": { @@ -461,12 +546,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] }, - "id": 9 + "id": 10 }, { "title": "GetObject Messages \u2014 Ledger", @@ -476,7 +564,7 @@ "h": 8, "w": 12, "x": 12, - "y": 32 + "y": 36 }, "options": { "tooltip": { @@ -502,12 +590,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] }, - "id": 10 + "id": 11 }, { "title": "GetObject Messages \u2014 Transaction", @@ -517,7 +608,7 @@ "h": 8, "w": 12, "x": 0, - "y": 40 + "y": 44 }, "options": { "tooltip": { @@ -549,12 +640,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] }, - "id": 11 + "id": 12 }, { "title": "GetObject Messages \u2014 Account State", @@ -564,7 +658,7 @@ "h": 8, "w": 12, "x": 12, - "y": 40 + "y": 44 }, "options": { "tooltip": { @@ -590,12 +684,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] }, - "id": 12 + "id": 13 }, { "title": "GetObject Messages \u2014 Specials", @@ -605,7 +702,7 @@ "h": 8, "w": 12, "x": 0, - "y": 48 + "y": 52 }, "options": { "tooltip": { @@ -643,12 +740,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] }, - "id": 13 + "id": 14 }, { "title": "GetObject \u2014 Specials", @@ -658,7 +758,7 @@ "h": 8, "w": 12, "x": 12, - "y": 48 + "y": 52 }, "options": { "tooltip": { @@ -720,65 +820,10 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 - } - }, - "overrides": [] - }, - "id": 14 - }, - { - "title": "Overlay Traffic Heatmap (All Categories, Bytes In)", - "description": "###### What this is:\n*All overlay traffic categories ranked by inbound bytes, giving an at-a-glance view of which message types consume the most receive bandwidth.*\n\n###### How it's computed:\n*Top categories by latest inbound byte value across all traffic categories. Each bar is labelled with its traffic category followed by the node identity; the shared `_bytes_in` suffix is dropped from the category name because the panel already reports inbound bytes.*\n\n###### Reading it:\n*The longest bars are the biggest bandwidth consumers; on a synced node transactions, proposals, and validations usually lead.*\n\n###### Healthy range:\n*workload-dependent.*\n\n###### Watch for:\n*A single ledger-data or fetch category dominating (ongoing sync) or an unexpected category topping the list.*\n\n###### Keywords:\n- **Overlay** *(network-wide)* \u2014 the peer-to-peer mesh xrpld nodes form to gossip transactions, proposals and validations.\n- **Peer** *(per node)* \u2014 another server this node holds a protocol connection to.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#overlay)", - "type": "bargauge", - "gridPos": { - "h": 18, - "w": 24, - "x": 0, - "y": 56 - }, - "options": { - "tooltip": { - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - }, - "displayMode": "gradient", - "orientation": "horizontal", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "expr": "label_replace(topk(20, {service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", __name__=~\".*_bytes_in\", __name__!~\"total_.*\"}), \"series\", \"$1\", \"__name__\", \"(.*)_bytes_in\")" - } - ], - "fieldConfig": { - "defaults": { - "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "decbytes", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 1048576 - }, - { - "color": "red", - "value": 104857600 - } - ] + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] @@ -793,7 +838,7 @@ "h": 1, "w": 24, "x": 0, - "y": 74 + "y": 60 }, "id": 16, "panels": [] @@ -806,7 +851,7 @@ "h": 8, "w": 12, "x": 0, - "y": 75 + "y": 61 }, "options": { "tooltip": { @@ -833,7 +878,10 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" }, "mappings": [ { @@ -928,7 +976,7 @@ "h": 8, "w": 12, "x": 12, - "y": 75 + "y": 61 }, "options": { "tooltip": { @@ -962,7 +1010,10 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] @@ -970,14 +1021,14 @@ "id": 18 }, { - "title": "Time Spent Per State", + "title": "Time Spent Per State [$xrpl_network_type]", "description": "###### What this is:\n*Operating mode as a colour-coded timeline. Each band's width is the time spent in that state, so short-lived states show as thin slivers instead of vanishing.*\n\n###### How it's computed:\n*server_info{metric=\"server_state\"} (gauge), the raw OperatingMode 0-4. Uses server_state rather than state_value because state_value folds 5 and 6 onto FULL, which would split one Full band into three colours.*\n\n###### Reading it:\n*One green band across the window = healthy. Red/orange/yellow bands show when and for how long the node was degraded.*\n\n###### Healthy range:\n*Continuously green (Full), brief orange/yellow/blue only after a restart.*\n\n###### Watch for:\n*Repeated thin bands = the node is oscillating. This is sampled every 10s, so a state shorter than one sample can still be missed.*\n\n###### Keywords:\n- **Operating mode / server state** *(per node)* \u2014 the node's sync level: Disconnected, Connected, Syncing, Tracking, Full.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[app/misc/NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`NetworkOPsImp::setMode`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#operating-mode-server-state)", "type": "state-timeline", "gridPos": { - "h": 6, - "w": 24, + "h": 8, + "w": 12, "x": 0, - "y": 83 + "y": 69 }, "options": { "mergeValues": true, @@ -1080,7 +1131,10 @@ }, "overrides": [] }, - "id": 27 + "id": 19, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 }, { "title": "Ledger Close Rate", @@ -1089,8 +1143,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 89 + "x": 12, + "y": 69 }, "options": { "tooltip": { @@ -1124,7 +1178,10 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [ @@ -1146,7 +1203,7 @@ } ] }, - "id": 19 + "id": 20 }, { "title": "Job Queue Wait p95 By Type", @@ -1155,8 +1212,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 89 + "x": 0, + "y": 77 }, "options": { "tooltip": { @@ -1218,12 +1275,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] }, - "id": 20 + "id": 21 }, { "title": "NuDB Read Latency", @@ -1232,8 +1292,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 97 + "x": 12, + "y": 77 }, "options": { "tooltip": { @@ -1260,12 +1320,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] }, - "id": 21 + "id": 22 }, { "title": "I/O Scheduler Latency p95", @@ -1274,8 +1337,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 97 + "x": 0, + "y": 85 }, "options": { "tooltip": { @@ -1302,12 +1365,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] }, - "id": 22 + "id": 23 }, { "title": "NuDB Read Found Ratio", @@ -1316,8 +1382,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 105 + "x": 12, + "y": 85 }, "options": { "tooltip": { @@ -1344,12 +1410,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] }, - "id": 23 + "id": 24 }, { "title": "NuDB Read Pressure", @@ -1358,8 +1427,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 105 + "x": 0, + "y": 93 }, "options": { "tooltip": { @@ -1400,7 +1469,10 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [ @@ -1438,7 +1510,7 @@ } ] }, - "id": 24 + "id": 25 }, { "title": "Job Queue Depth", @@ -1447,8 +1519,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 113 + "x": 12, + "y": 93 }, "options": { "tooltip": { @@ -1475,12 +1547,16 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 - } + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" + }, + "decimals": 0 }, "overrides": [] }, - "id": 25 + "id": 26 }, { "title": "Load Factor & Peers", @@ -1489,8 +1565,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 113 + "x": 0, + "y": 101 }, "options": { "tooltip": { @@ -1531,12 +1607,16 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 - } + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" + }, + "decimals": 0 }, "overrides": [] }, - "id": 26 + "id": 27 }, { "title": "Job Queue Saturation", @@ -1545,11 +1625,11 @@ "h": 1, "w": 24, "x": 0, - "y": 121 + "y": 109 }, "collapsed": false, "panels": [], - "id": 38 + "id": 28 }, { "title": "Job Queue Backlog and Deferred by Type", @@ -1559,7 +1639,7 @@ "h": 8, "w": 12, "x": 0, - "y": 122 + "y": 110 }, "options": { "tooltip": { @@ -1593,12 +1673,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] }, - "id": 28 + "id": 29 }, { "title": "LedgerReq Wait by Handler", @@ -1608,7 +1691,7 @@ "h": 8, "w": 12, "x": 12, - "y": 122 + "y": 110 }, "options": { "tooltip": { @@ -1635,12 +1718,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] }, - "id": 29 + "id": 30 }, { "title": "Sync Bottleneck Discrimination", @@ -1649,11 +1735,11 @@ "h": 1, "w": 24, "x": 0, - "y": 130 + "y": 118 }, "collapsed": false, "panels": [], - "id": 30 + "id": 31 }, { "title": "NodeStore Read Latency (Bottleneck Discriminator)", @@ -1663,7 +1749,7 @@ "h": 8, "w": 12, "x": 0, - "y": 131 + "y": 119 }, "options": { "tooltip": { @@ -1720,12 +1806,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] }, - "id": 31 + "id": 32 }, { "title": "NuDB Writer Queue Depth", @@ -1735,7 +1824,7 @@ "h": 8, "w": 12, "x": 12, - "y": 131 + "y": 119 }, "options": { "tooltip": { @@ -1769,12 +1858,16 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 - } + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" + }, + "decimals": 0 }, "overrides": [] }, - "id": 32 + "id": 33 }, { "title": "NuDB Insert Time (Mean & Max)", @@ -1784,7 +1877,7 @@ "h": 8, "w": 12, "x": 0, - "y": 139 + "y": 127 }, "options": { "tooltip": { @@ -1829,12 +1922,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] }, - "id": 33 + "id": 34 }, { "title": "Acquire Deferrals vs Timeouts (All Lanes)", @@ -1844,7 +1940,7 @@ "h": 8, "w": 12, "x": 12, - "y": 139 + "y": 127 }, "options": { "tooltip": { @@ -1878,12 +1974,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] }, - "id": 34 + "id": 35 }, { "title": "Acquisition Progress (Completions, Give-Ups & Aborts)", @@ -1893,7 +1992,7 @@ "h": 8, "w": 12, "x": 0, - "y": 147 + "y": 135 }, "options": { "tooltip": { @@ -1934,12 +2033,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] }, - "id": 35 + "id": 36 }, { "title": "Discarded Acquire Work (Sweeps & Partial Aborts)", @@ -1949,7 +2051,7 @@ "h": 8, "w": 12, "x": 12, - "y": 147 + "y": 135 }, "options": { "tooltip": { @@ -1983,12 +2085,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] }, - "id": 36 + "id": 37 }, { "title": "Ledger Acquire Deferrals vs Timeouts (Ledger Lane Only)", @@ -1998,7 +2103,7 @@ "h": 8, "w": 12, "x": 0, - "y": 155 + "y": 143 }, "options": { "tooltip": { @@ -2032,12 +2137,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] }, - "id": 37 + "id": 38 } ], "schemaVersion": 39, diff --git a/docker/telemetry/grafana/dashboards/ledger-operations.json b/docker/telemetry/grafana/dashboards/ledger-operations.json index d3c6ae2342..00699deb6f 100644 --- a/docker/telemetry/grafana/dashboards/ledger-operations.json +++ b/docker/telemetry/grafana/dashboards/ledger-operations.json @@ -39,14 +39,27 @@ "links": [], "panels": [ { - "title": "Ledger Build Rate", + "title": "Ledger Build & Validate", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "collapsed": false, + "panels": [], + "id": 1 + }, + { + "title": "Ledger Build Rate [$xrpl_network_type]", "description": "###### What this is:\n*How many new ledgers this node finishes building per second.*\n\n###### How it's computed:\n*Per-second rate of completed ledger-build operations, averaged over 5 minutes and split by node.*\n\n###### Reading it:\n*A steady flat line; the value should track the network close cadence.*\n\n###### Healthy range:\n*About 0.2-0.3 ledgers/sec on mainnet (roughly one every 3-5s); workload-dependent on test networks.*\n\n###### Watch for:\n*A drop toward zero (node fell out of sync or stalled) or a value well above the network rate (rebuilding history).*\n\n###### Keywords:\n- **Ledger build** *(per node)* \u2014 constructing the new ledger by applying the agreed transaction set to the prior ledger.\n- **Ledger close interval** *(network-wide)* \u2014 the network's steady ledger rhythm \u2014 roughly one closed ledger every 3-5 seconds on Mainnet.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[BuildLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/BuildLedger.cpp)\n\n###### Function:\n`buildLedgerImpl`\n\n###### References:\n[Ledger build](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) \u00b7 [Ledger close interval](https://xrpl.org/docs/concepts/ledgers/ledger-close-times) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledger-build)", "type": "stat", "gridPos": { - "h": 8, + "h": 4, "w": 12, "x": 0, - "y": 0 + "y": 1 }, "options": { "tooltip": { @@ -70,7 +83,49 @@ "unit": "suffix: ledgers/s" }, "overrides": [] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 2 + }, + { + "title": "Ledger Validation Rate [$xrpl_network_type]", + "description": "###### What this is:\n*How often ledgers reach full validation (accepted by the trusted validator quorum) per second.*\n\n###### How it's computed:\n*Per-second rate of ledger-validation events over 5 minutes, per node.*\n\n###### Reading it:\n*Should closely match the build rate under normal, in-sync operation.*\n\n###### Healthy range:\n*About 0.2-0.3/sec on mainnet; workload-dependent elsewhere.*\n\n###### Watch for:\n*A validation rate that lags the build rate, signalling the node is building ahead of the network consensus it trusts.*\n\n###### Keywords:\n- **Ledger validation** *(network event)* \u2014 the second consensus stage where the node confirms a built ledger matches the trusted validator quorum and marks it final.\n- **Validation quorum** *(network-wide)* \u2014 the minimum number of agreeing trusted validations needed to declare a ledger fully validated.\n- **Ledger build** *(per node)* \u2014 constructing the new ledger by applying the agreed transaction set to the prior ledger.\n- **Consensus** *(network event)* \u2014 the protocol by which validators agree on the next ledger's transaction set and close time.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[LedgerMaster.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/LedgerMaster.cpp)\n\n###### Function:\n`LedgerMaster::checkAccept`\n\n###### References:\n[Ledger validation](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure#validation) \u00b7 [Validation quorum](https://xrpl.org/docs/concepts/consensus-protocol/negative-unl) \u00b7 [Ledger build](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) \u00b7 [Consensus](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledger-validation)", + "type": "stat", + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 1 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"ledger.validate\"}[$__rate_interval])), \"series\", \"Validations / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "suffix: ledgers/s" + }, + "overrides": [] + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 3 }, { "title": "Ledger Build Duration", @@ -79,8 +134,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 0 + "x": 0, + "y": 5 }, "options": { "tooltip": { @@ -107,45 +162,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } - }, - { - "title": "Ledger Validation Rate", - "description": "###### What this is:\n*How often ledgers reach full validation (accepted by the trusted validator quorum) per second.*\n\n###### How it's computed:\n*Per-second rate of ledger-validation events over 5 minutes, per node.*\n\n###### Reading it:\n*Should closely match the build rate under normal, in-sync operation.*\n\n###### Healthy range:\n*About 0.2-0.3/sec on mainnet; workload-dependent elsewhere.*\n\n###### Watch for:\n*A validation rate that lags the build rate, signalling the node is building ahead of the network consensus it trusts.*\n\n###### Keywords:\n- **Ledger validation** *(network event)* \u2014 the second consensus stage where the node confirms a built ledger matches the trusted validator quorum and marks it final.\n- **Validation quorum** *(network-wide)* \u2014 the minimum number of agreeing trusted validations needed to declare a ledger fully validated.\n- **Ledger build** *(per node)* \u2014 constructing the new ledger by applying the agreed transaction set to the prior ledger.\n- **Consensus** *(network event)* \u2014 the protocol by which validators agree on the next ledger's transaction set and close time.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[LedgerMaster.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/LedgerMaster.cpp)\n\n###### Function:\n`LedgerMaster::checkAccept`\n\n###### References:\n[Ledger validation](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure#validation) \u00b7 [Validation quorum](https://xrpl.org/docs/concepts/consensus-protocol/negative-unl) \u00b7 [Ledger build](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) \u00b7 [Consensus](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledger-validation)", - "type": "stat", - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 8 }, - "options": { - "tooltip": { - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"ledger.validate\"}[$__rate_interval])), \"series\", \"Validations / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" - } - ], - "fieldConfig": { - "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "unit": "suffix: ledgers/s" - }, - "overrides": [] - } + "id": 4 }, { "title": "Ledger Build Duration Heatmap", @@ -155,7 +180,7 @@ "h": 8, "w": 12, "x": 12, - "y": 8 + "y": 5 }, "options": { "tooltip": { @@ -183,7 +208,21 @@ "unit": "ms" }, "overrides": [] - } + }, + "id": 5 + }, + { + "title": "Transaction Apply", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 13 + }, + "collapsed": false, + "panels": [], + "id": 6 }, { "title": "Transaction Apply Duration", @@ -193,7 +232,7 @@ "h": 8, "w": 12, "x": 0, - "y": 16 + "y": 14 }, "options": { "tooltip": { @@ -220,11 +259,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 7 }, { "title": "Transaction Apply Rate", @@ -234,7 +277,7 @@ "h": 8, "w": 12, "x": 12, - "y": 16 + "y": 14 }, "options": { "tooltip": { @@ -261,21 +304,38 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 8 }, { - "title": "Ledger Store Rate", + "title": "Store & Close Timing", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 22 + }, + "collapsed": false, + "panels": [], + "id": 9 + }, + { + "title": "Ledger Store Rate [$xrpl_network_type]", "description": "###### What this is:\n*How often completed ledgers are written into ledger history per second.*\n\n###### How it's computed:\n*Per-second rate of ledger-store operations over 5 minutes, per node.*\n\n###### Reading it:\n*Should match the build rate during normal operation.*\n\n###### Healthy range:\n*About 0.2-0.3/sec on mainnet; can burst higher while backfilling history.*\n\n###### Watch for:\n*A store rate below the build rate (storage falling behind) or a stall at zero.*\n\n###### Keywords:\n- **Ledger store** *(per node)* \u2014 writing a completed ledger into the node's ledger history on disk.\n- **Ledger build** *(per node)* \u2014 constructing the new ledger by applying the agreed transaction set to the prior ledger.\n- **Back-fill / catch-up** *(per node)* \u2014 fetching missing historical ledgers from peers to fill gaps or reach the network tip.\n- **Consensus stall** *(per node)* \u2014 a health check reporting that consensus is not making forward progress.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[LedgerMaster.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/LedgerMaster.cpp)\n\n###### Function:\n`LedgerMaster::storeLedger`\n\n###### References:\n[Ledger build](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) \u00b7 [Consensus stall](https://xrpl.org/docs/concepts/consensus-protocol/consensus-principles-and-rules) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledger-store)", "type": "stat", "gridPos": { - "h": 8, + "h": 4, "w": 12, "x": 0, - "y": 24 + "y": 23 }, "options": { "tooltip": { @@ -299,7 +359,11 @@ "unit": "suffix: ledgers/s" }, "overrides": [] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 10 }, { "title": "Build vs Close Duration", @@ -308,8 +372,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 24 + "x": 0, + "y": 27 }, "options": { "tooltip": { @@ -343,11 +407,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 11 }, { "title": "Ledger Close Interval & Age", @@ -356,8 +424,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 32 + "x": 12, + "y": 27 }, "options": { "tooltip": { @@ -391,7 +459,10 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [ @@ -432,7 +503,8 @@ ] } ] - } + }, + "id": 12 } ], "schemaVersion": 39, diff --git a/docker/telemetry/grafana/dashboards/log-derived-insights.json b/docker/telemetry/grafana/dashboards/log-derived-insights.json index 8f7aedfbd5..2a10be92a0 100644 --- a/docker/telemetry/grafana/dashboards/log-derived-insights.json +++ b/docker/telemetry/grafana/dashboards/log-derived-insights.json @@ -220,7 +220,7 @@ "type": "text", "title": "Read This First — Debug Log Requirement", "gridPos": { - "h": 10, + "h": 4, "w": 24, "x": 0, "y": 0 @@ -244,24 +244,24 @@ "h": 1, "w": 24, "x": 0, - "y": 12 + "y": 4 }, "panels": [], "id": 2 }, { "type": "stat", - "title": "Nodes By Error Volume", + "title": "Nodes By Error Volume [$xrpl_network_type]", "description": "###### What this is:\n*Which nodes are logging the most errors.*\n\n###### How it's computed:\n*Count of ERR and FTL log lines per node over the dashboard window, ranked highest first.*\n\n###### Reading it:\n*The top entry is the node in the most trouble. Compare nodes rather than reading an absolute value.*\n\n###### Healthy range:\n*Zero, or a small flat count. ERR is not routine.*\n\n###### Watch for:\n*Any node pulling far ahead of its peers, which usually means a fault local to that node rather than a network condition.*\n\n###### Keywords:\n- **ERR / FTL** *(per line)* — the two most severe xrpld log levels; both survive the default Info threshold.\n\n###### Computation boundary:\n*Result: Per node — a count over the dashboard window, ranked worst-first.*\n*NOT recorded as a metric. Derived in the Grafana query by regex over raw log text.*\n\n###### Source:\n[Log.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/basics/Log.cpp)\n\n###### Function:\n`Logs::Sink::write`\n\n###### References:\n[Loki log queries](https://grafana.com/docs/loki/latest/query/log_queries/)", "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "gridPos": { - "h": 8, + "h": 4, "w": 12, "x": 0, - "y": 13 + "y": 5 }, "fieldConfig": { "defaults": { @@ -318,21 +318,24 @@ "legendFormat": "Errors [{{service_instance_id}}] [{{xrpl_network_type}}]" } ], - "id": 3 + "id": 3, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 }, { "type": "stat", - "title": "Nodes By Attack-Like Input", + "title": "Nodes By Attack-Like Input [$xrpl_network_type]", "description": "###### What this is:\n*Which nodes are receiving the most malformed or unwanted peer payloads.*\n\n###### How it's computed:\n*Count of Resource fee charges whose reason is useless data, unwanted data, or init drop, per node.*\n\n###### Reading it:\n*These three reasons indicate a peer sent something the node could not use, so a high count is the closest log-derived proxy for abusive input.*\n\n###### Healthy range:\n*Low and flat. Some useless data is normal on a busy overlay.*\n\n###### Watch for:\n*A sharp rise on one node, especially paired with a single dominant IP in the charged-peers table below.*\n\n###### Keywords:\n- **Useless data** *(per charge)* — payload the node could not use, e.g. an empty or malformed message.\n- **Init drop** *(per charge)* — a connection dropped during initialisation.\n\n###### Computation boundary:\n*Result: Per node — a count over the dashboard window, ranked worst-first.*\n*NOT recorded as a metric. Derived in the Grafana query by regex over raw log text.*\n\n###### Source:\n[ResourceManager.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/resource/detail/ResourceManager.cpp)\n\n###### Function:\n`Logic::charge`\n\n###### Note:\n*Requires debug logs on the Resource partition.*\n\n###### References:\n[Peer protocol](https://xrpl.org/docs/concepts/networks-and-servers/peer-protocol)", "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "gridPos": { - "h": 8, + "h": 4, "w": 12, "x": 12, - "y": 13 + "y": 5 }, "fieldConfig": { "defaults": { @@ -389,21 +392,24 @@ "legendFormat": "Charges [{{service_instance_id}}] [{{xrpl_network_type}}]" } ], - "id": 4 + "id": 4, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 }, { "type": "stat", - "title": "Nodes By Total Fee Charged", + "title": "Nodes By Total Fee Charged [$xrpl_network_type]", "description": "###### What this is:\n*Which nodes are absorbing the most peer-imposed load, weighted by fee amount.*\n\n###### How it's computed:\n*Sum of every resource fee amount parsed from the Resource partition, per node.*\n\n###### Reading it:\n*Weighted by cost rather than event count, so one heavy request at 2000 outranks eight moderate ones at 250.*\n\n###### Healthy range:\n*Proportional to peer count and request volume; compare nodes of similar role.*\n\n###### Watch for:\n*One node far above peers of the same role, which means it is carrying disproportionate peer load.*\n\n###### Keywords:\n- **Fee weight** *(per charge)* — the credit amount: 2000 heavy, 250 moderate, 150 useless data.\n\n###### Computation boundary:\n*Result: Per node — a count over the dashboard window, ranked worst-first.*\n*NOT recorded as a metric. Derived in the Grafana query by regex over raw log text.*\n\n###### Source:\n[ResourceManager.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/resource/detail/ResourceManager.cpp)\n\n###### Function:\n`Logic::charge`\n\n###### Note:\n*Requires debug logs on the Resource partition.*\n\n###### References:\n[Peer protocol](https://xrpl.org/docs/concepts/networks-and-servers/peer-protocol)", "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "gridPos": { - "h": 8, + "h": 4, "w": 12, "x": 0, - "y": 21 + "y": 9 }, "fieldConfig": { "defaults": { @@ -460,21 +466,24 @@ "legendFormat": "Fee Charged [{{service_instance_id}}] [{{xrpl_network_type}}]" } ], - "id": 5 + "id": 5, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 }, { "type": "stat", - "title": "Nodes By Manifest Rejection", + "title": "Nodes By Manifest Rejection [$xrpl_network_type]", "description": "###### What this is:\n*Which nodes are rejecting the most inbound manifests.*\n\n###### How it's computed:\n*Count of ManifestCache lines with a Stale, Invalid, Revoked, or UntrustedCapacity outcome, per node.*\n\n###### Reading it:\n*A manifest flood shows up here first. Stale dominates normally because peers re-gossip manifests the node already holds.*\n\n###### Healthy range:\n*Workload-dependent; nodes on the same network should sit close together.*\n\n###### Watch for:\n*A single node far ahead, or any Invalid at all, which means a signature failed verification.*\n\n###### Keywords:\n- **Rejection** *(per manifest)* — any non-accepted disposition.\n- **UntrustedCapacity** *(per manifest)* — rate-limit rejection for an unlisted key; rc5+ builds only.\n\n###### Computation boundary:\n*Result: Per node — a count over the dashboard window, ranked worst-first.*\n*NOT recorded as a metric. Derived in the Grafana query by regex over raw log text.*\n\n###### Source:\n[Manifest.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/server/Manifest.cpp)\n\n###### Function:\n`ManifestCache::applyManifest`\n\n###### Note:\n*Requires debug logs on the ManifestCache partition.*\n\n###### References:\n[Validator keys](https://xrpl.org/docs/concepts/consensus-protocol/validator-keys)", "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "gridPos": { - "h": 8, + "h": 4, "w": 12, "x": 12, - "y": 21 + "y": 9 }, "fieldConfig": { "defaults": { @@ -531,21 +540,24 @@ "legendFormat": "Rejections [{{service_instance_id}}] [{{xrpl_network_type}}]" } ], - "id": 6 + "id": 6, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 }, { "type": "stat", - "title": "Nodes By Consensus Problem", + "title": "Nodes By Consensus Problem [$xrpl_network_type]", "description": "###### What this is:\n*Which nodes are logging the most consensus warnings and errors.*\n\n###### How it's computed:\n*Count of LedgerConsensus lines at WRN severity or above, per node.*\n\n###### Reading it:\n*Consensus is a network-wide process, so a single node standing out points at that node rather than the network.*\n\n###### Healthy range:\n*Low. Some warnings occur during normal round churn.*\n\n###### Watch for:\n*One node far above its peers, or a step change after a deploy.*\n\n###### Keywords:\n- **Consensus round** *(per ledger)* — one Open to Establish to Accepted cycle.\n- **Dispute** *(per transaction)* — a transaction peers disagree about including.\n\n###### Computation boundary:\n*Result: Per node — a count over the dashboard window, ranked worst-first.*\n*NOT recorded as a metric. Derived in the Grafana query by regex over raw log text.*\n\n###### Source:\n[Consensus.h](https://github.com/XRPLF/rippled/blob/develop/include/xrpl/consensus/Consensus.h)\n\n###### Function:\n`Consensus::timerEntry / checkConsensus`\n\n###### References:\n[Consensus](https://xrpl.org/docs/concepts/consensus-protocol)", "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "gridPos": { - "h": 8, + "h": 4, "w": 12, "x": 0, - "y": 29 + "y": 13 }, "fieldConfig": { "defaults": { @@ -602,21 +614,24 @@ "legendFormat": "Consensus Events [{{service_instance_id}}] [{{xrpl_network_type}}]" } ], - "id": 7 + "id": 7, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 }, { "type": "stat", - "title": "Nodes By Job Latency Breach", + "title": "Nodes By Job Latency Breach [$xrpl_network_type]", "description": "###### What this is:\n*Which nodes are missing job latency targets most often.*\n\n###### How it's computed:\n*Count of LoadMonitor slow-job lines per node; the emitter only fires above a 500ms threshold.*\n\n###### Reading it:\n*A direct read on which node is most overloaded. Works at the default log level.*\n\n###### Healthy range:\n*Low. A busy node breaches occasionally.*\n\n###### Watch for:\n*A node far ahead of its peers, which usually means disk or CPU pressure local to it.*\n\n###### Keywords:\n- **Breach** *(per job)* — one job execution over the 500ms LoadMonitor threshold.\n\n###### Computation boundary:\n*Result: Per node — a count of BREACHES, never of total job executions.*\n*Derived in the Grafana query. Use the native `job_*` metrics for totals.*\n\n###### Source:\n[LoadMonitor.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/core/detail/LoadMonitor.cpp)\n\n###### Function:\n`LoadMonitor::addLoadSample`\n\n###### Note:\n*Counts breaches above 500ms only, so it is an exception count and not a latency measure.*\n\n###### References:\n[Job queue](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/core/detail/JobQueue.cpp)", "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "gridPos": { - "h": 8, + "h": 4, "w": 12, "x": 12, - "y": 29 + "y": 13 }, "fieldConfig": { "defaults": { @@ -673,21 +688,24 @@ "legendFormat": "Breaches [{{service_instance_id}}] [{{xrpl_network_type}}]" } ], - "id": 8 + "id": 8, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 }, { "type": "stat", - "title": "Nodes By Sync Instability", + "title": "Nodes By Sync Instability [$xrpl_network_type]", "description": "###### What this is:\n*Which nodes are spending the most time out of the full state.*\n\n###### How it's computed:\n*Count of STATE-> transitions into any non-full state, per node.*\n\n###### Reading it:\n*A stable node holds full and emits nothing here, so any non-zero value means it left full that many times.*\n\n###### Healthy range:\n*Zero on a synced node.*\n\n###### Watch for:\n*A repeating count, which means the node is flapping rather than having had one bad moment.*\n\n###### Keywords:\n- **Operating mode** *(per node)* — Disconnected, Connected, Syncing, Tracking, Full.\n- **Flapping** *(per node)* — repeated departures from full.\n\n###### Computation boundary:\n*Result: Per node — a count over the dashboard window, ranked worst-first.*\n*NOT recorded as a metric. Derived in the Grafana query by regex over raw log text.*\n\n###### Source:\n[NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`NetworkOPsImp::setMode`\n\n###### References:\n[Server states](https://xrpl.org/docs/concepts/networks-and-servers/rippled-server-states)", "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "gridPos": { - "h": 8, + "h": 4, "w": 12, "x": 0, - "y": 37 + "y": 17 }, "fieldConfig": { "defaults": { @@ -744,21 +762,24 @@ "legendFormat": "Transitions [{{service_instance_id}}] [{{xrpl_network_type}}]" } ], - "id": 9 + "id": 9, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 }, { "type": "stat", - "title": "Nodes By Duplicate Ledger Fetches", + "title": "Nodes By Duplicate Ledger Fetches [$xrpl_network_type]", "description": "###### What this is:\n*Which nodes wasted the most ledger-fetch bandwidth, counted as duplicate SHAMap nodes received.*\n\n###### How it's computed:\n*Sum of the `dupe` counter from `Ledger AS/TX node stats` log lines, per node, over the dashboard window. `dupe` is matched by its own regex because the emitter omits any counter that is zero.*\n\n###### Reading it:\n*Every duplicate is a ledger node the peer sent that this node already held: bytes crossed the network, the peer spent resource credit serving it, and it was discarded.*\n\n###### Healthy range:\n*Workload-dependent. Compare nodes of the same role rather than reading an absolute value; some duplication is unavoidable when fetching one ledger from several peers.*\n\n###### Watch for:\n*A node far ahead of its siblings, which points at that node's fetch logic or peer set rather than a network condition.*\n\n###### Keywords:\n- **Ledger node** *(per fetch)* — one SHAMap node fetched while acquiring a ledger.\n- **Duplicate** *(per fetch)* — a node already held locally; wasted bandwidth and peer credit.\n\n###### Computation boundary:\n*Result: Per node — a COUNT of duplicates, not a ratio.*\n*NOT recorded as a metric. Derived in the Grafana query via `unwrap` over raw log text.*\n\n###### Source:\n[InboundLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/InboundLedger.cpp)\n\n###### Function:\n`InboundLedger::onTimer`\n\n###### Note:\n*Requires debug logs on the InboundLedger partition. For the duplicate RATIO see \"Ledger Node Fetch Duplicate Ratio\" in the Ledger Acquisition Efficiency row — a stat panel cannot render a division, because Loki does not name binary-operation frames and Grafana falls back to \"Value #A\".*\n\n###### References:\n[Ledgers](https://xrpl.org/docs/concepts/ledgers)", "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "gridPos": { - "h": 8, + "h": 4, "w": 12, "x": 12, - "y": 37 + "y": 17 }, "fieldConfig": { "defaults": { @@ -815,7 +836,10 @@ "legendFormat": "Duplicates [{{service_instance_id}}] [{{xrpl_network_type}}]" } ], - "id": 10 + "id": 10, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 }, { "type": "row", @@ -825,7 +849,7 @@ "h": 1, "w": 24, "x": 0, - "y": 45 + "y": 21 }, "panels": [], "id": 11 @@ -839,10 +863,10 @@ "uid": "${DS_LOKI}" }, "gridPos": { - "h": 10, + "h": 8, "w": 12, "x": 0, - "y": 46 + "y": 22 }, "fieldConfig": { "defaults": { @@ -851,13 +875,14 @@ "min": 0, "custom": { "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, + "lineWidth": 1, + "fillOpacity": 0, "axisLabel": "Transitions / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -892,17 +917,17 @@ }, { "type": "state-timeline", - "title": "Node State Timeline", + "title": "Node State Timeline [$xrpl_network_type]", "description": "###### What this is:\n*The node's operating state over time, as a discrete timeline.*\n\n###### How it's computed:\n*The most recent `STATE->` transition in each interval, rendered as a state band.*\n\n###### Reading it:\n*Shows at a glance how long the node spent in each state and exactly when it left full.*\n\n###### Healthy range:\n*One unbroken full band across the window.*\n\n###### Watch for:\n*Any band that is not full, and repeated narrow bands, which indicate state flapping.*\n\n###### Keywords:\n- **Operating mode / server state** *(per node)* — the node's sync level: Disconnected, Connected, Syncing, Tracking, Full.\n\n###### Computation boundary:\n*Result: Per node — one band per server per state.*\n*NOT recorded as a metric anywhere. Derived in the Grafana query by regex over raw log text.*\n\n###### Source:\n[NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`NetworkOPsImp::setMode`\n\n###### Note:\n*Only transitions are logged, so a node that never changes state produces no data here. Read with the transition-rate panel beside it.*\n\n###### References:\n[Server states](https://xrpl.org/docs/concepts/networks-and-servers/rippled-server-states)", "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "gridPos": { - "h": 10, + "h": 8, "w": 12, "x": 12, - "y": 46 + "y": 22 }, "fieldConfig": { "defaults": { @@ -945,7 +970,10 @@ "expr": "sum by (state, service_instance_id) (count_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | severity =~ \"$severity\" | regexp `STATE->(?P\\w+)` | state != `` | label_format state=`{{if eq .state \"full\"}}Full{{else if eq .state \"syncing\"}}Syncing{{else if eq .state \"tracking\"}}Tracking{{else if eq .state \"connected\"}}Connected{{else if eq .state \"disconnected\"}}Disconnected{{else}}{{.state}}{{end}}` [$__auto]))" } ], - "id": 13 + "id": 13, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 }, { "type": "row", @@ -955,7 +983,7 @@ "h": 1, "w": 24, "x": 0, - "y": 56 + "y": 30 }, "panels": [], "id": 14 @@ -965,10 +993,10 @@ "title": "Log Line Rate By Severity", "description": "###### What this is:\n*Rate of log lines emitted by xrpld, split by severity.*\n\n###### How it's computed:\n*Per-second count of matching log lines grouped by the severity field parsed out of each line.*\n\n###### Reading it:\n*Use this to confirm the log pipeline is alive, and to see at a glance whether DBG lines are being collected at all.*\n\n###### Healthy range:\n*Workload-dependent. If the DBG series is absent, every panel in a [DBG] row on this dashboard will be empty.*\n\n###### Watch for:\n*A sudden collapse to only WRN and ERR, which means debug logging was turned off and the [DBG] rows have gone blind rather than quiet.*\n\n###### Keywords:\n- **Severity** *(per line)* — xrpld log level: DBG, NFO, WRN, ERR, FTL.\n- **Structured metadata** *(per line)* — Loki fields parsed from the line, filtered with `|` rather than in the stream selector.\n\n###### Computation boundary:\n*Result: Per node per severity — a count of log lines, not of events in the node.*\n*Derived in the Grafana query; the collector's filelog receiver parses severity, xrpld itself exports no such metric.*\n\n###### Source:\n[Log.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/basics/Log.cpp)\n\n###### Function:\n`Logs::Sink::write`\n\n###### References:\n[Loki structured metadata](https://grafana.com/docs/loki/latest/get-started/labels/structured-metadata/)", "gridPos": { - "h": 10, + "h": 8, "w": 12, "x": 0, - "y": 57 + "y": 31 }, "fieldConfig": { "defaults": { @@ -976,13 +1004,14 @@ "unit": "suffix: lines/s", "custom": { "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, + "lineWidth": 1, + "fillOpacity": 0, "axisLabel": "Log Lines / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -1024,10 +1053,10 @@ "title": "Log Line Rate By Partition (Top $topn)", "description": "###### What this is:\n*The busiest xrpld log partitions by line rate.*\n\n###### How it's computed:\n*Per-second count of log lines grouped by the partition field, limited to the top N series.*\n\n###### Reading it:\n*Shows which subsystem dominates log volume, which is the main cost driver for Loki ingest.*\n\n###### Healthy range:\n*Workload-dependent. Resource, JobQueue, and LedgerConsensus are normally the loudest.*\n\n###### Watch for:\n*A partition suddenly dominating, which usually means a subsystem entered a retry or error loop.*\n\n###### Keywords:\n- **Partition** *(per line)* — the xrpld subsystem that emitted the line, e.g. ManifestCache, Resource, LedgerConsensus.\n\n###### Computation boundary:\n*Result: Per node per partition — a count of log lines.*\n*Derived in the Grafana query; truncated to the top N by Loki's series limit.*\n\n###### Source:\n[Log.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/basics/Log.cpp)\n\n###### Function:\n`Logs::Sink::write`\n\n###### References:\n[Loki log queries](https://grafana.com/docs/loki/latest/query/log_queries/)", "gridPos": { - "h": 10, + "h": 8, "w": 12, "x": 12, - "y": 57 + "y": 31 }, "fieldConfig": { "defaults": { @@ -1035,13 +1064,14 @@ "unit": "suffix: lines/s", "custom": { "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, + "lineWidth": 1, + "fillOpacity": 0, "axisLabel": "Log Lines / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -1086,7 +1116,7 @@ "h": 1, "w": 24, "x": 0, - "y": 67 + "y": 39 }, "panels": [], "id": 17 @@ -1096,10 +1126,10 @@ "title": "Manifest Disposition Rate", "description": "###### What this is:\n*Rate of manifest apply outcomes: accepted, stale, revoked, invalid, or rate-limited.*\n\n###### How it's computed:\n*Per-second count of ManifestCache log lines, with the action parsed out of the `Manifest: ;Pk: ...` text.*\n\n###### Reading it:\n*AcceptedNew and AcceptedUpdate are useful work; Stale and UntrustedCapacity are rejections and normally dominate.*\n\n###### Healthy range:\n*Accepted rates are low and bursty. A high sustained Stale rate is normal — peers gossip manifests this node already holds.*\n\n###### Watch for:\n*Any Invalid, which means a signature failed verification, and a sustained UntrustedCapacity climb, which indicates a manifest flood from unlisted keys.*\n\n###### Keywords:\n- **Manifest** *(per validator)* — a signed record binding a validator's master key to its current signing key.\n- **Disposition** *(per manifest)* — the apply outcome: AcceptedNew, AcceptedUpdate, Stale, Revoked, Invalid, UntrustedCapacity.\n- **Stale** *(per manifest)* — sequence number not greater than the one already held; the common benign rejection.\n\n###### Computation boundary:\n*Result: Per node per action — counts log EVENTS, not distinct manifests.*\n*NOT recorded as a metric. `applyManifest` has no instrumentation; this is regex over `logMftAct` output in the Grafana query.*\n\n###### Source:\n[Manifest.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/server/Manifest.cpp)\n\n###### Function:\n`ManifestCache::applyManifest / logMftAct`\n\n###### References:\n[Validator keys](https://xrpl.org/docs/concepts/consensus-protocol/validator-keys)", "gridPos": { - "h": 10, + "h": 8, "w": 12, "x": 0, - "y": 68 + "y": 40 }, "fieldConfig": { "defaults": { @@ -1107,13 +1137,14 @@ "unit": "suffix: manifests/s", "custom": { "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, + "lineWidth": 1, + "fillOpacity": 0, "axisLabel": "Manifests / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -1155,23 +1186,24 @@ "title": "Manifest Accept Vs Reject Rate", "description": "###### What this is:\n*Manifest outcomes collapsed into two series: accepted versus rejected.*\n\n###### How it's computed:\n*Per-second counts of ManifestCache lines, with AcceptedNew and AcceptedUpdate summed as accepted and all other actions summed as rejected.*\n\n###### Reading it:\n*Gives the useful-work fraction of inbound manifest processing without per-action detail.*\n\n###### Healthy range:\n*Rejected normally far exceeds accepted; peers routinely re-gossip known manifests.*\n\n###### Watch for:\n*A rejected rate in the hundreds per second, which indicates a manifest flood consuming the JtManifest job queue.*\n\n###### Keywords:\n- **Accepted** *(per manifest)* — AcceptedNew or AcceptedUpdate; the manifest changed cache state.\n- **Rejected** *(per manifest)* — Stale, Invalid, Revoked, or UntrustedCapacity.\n\n###### Computation boundary:\n*Result: Per node — two summed series over the same log lines.*\n*Derived in the Grafana query. Accepted may legitimately be absent: measured 1,360 AcceptedNew per 7d, so short windows often contain none. Both series carry `or vector(0)` so a zero-accept window renders a flat zero line rather than disappearing.*\n\n###### Source:\n[Manifest.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/server/Manifest.cpp)\n\n###### Function:\n`ManifestCache::applyManifest / logMftAct`\n\n###### References:\n[Validator keys](https://xrpl.org/docs/concepts/consensus-protocol/validator-keys)", "gridPos": { - "h": 10, + "h": 8, "w": 12, "x": 12, - "y": 68 + "y": 40 }, "fieldConfig": { "defaults": { "unit": "suffix: manifests/s", "custom": { "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, + "lineWidth": 1, + "fillOpacity": 0, "axisLabel": "Manifests / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -1220,13 +1252,13 @@ }, { "type": "table", - "title": "Top $topn Manifest Producers By Master Key", + "title": "Top $topn Manifest Producers By Master Key [$xrpl_network_type]", "description": "###### What this is:\n*The master keys responsible for the most manifest log events in the selected window.*\n\n###### How it's computed:\n*Count of ManifestCache lines grouped by the base58 master key parsed from the log text, limited to the top N.\nCounts are per log EVENT, not per distinct manifest.*\n\n###### Reading it:\n*Identifies which validator keys generate the most manifest churn on this node.*\n\n###### Healthy range:\n*A small number of keys with modest repeat counts. Live observation shows roughly 19 to 24 events per key per hour.*\n\n###### Watch for:\n*A single unlisted key with a very high count, which is the signature of a targeted manifest flood.*\n\n###### Keywords:\n- **Master key** *(per validator)* — the long-lived base58 key a manifest is signed under.\n- **Log event** *(per line)* — one manifest apply attempt; the same key recurs as peers re-gossip.\n\n###### Computation boundary:\n*Result: Per master key per node — counts log EVENTS, NOT distinct manifests and NOT distinct keys.*\n*Derived in the Grafana query. Truncated to top N: Loki rejects queries returning over 2000 series, so a true distinct-key count is impossible here.*\n\n###### Source:\n[Manifest.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/server/Manifest.cpp)\n\n###### Function:\n`logMftAct`\n\n###### Note:\n*Truncated to the top N. Loki rejects a query returning over 2000 series, so a full distinct-key count is not possible here.*\n\n###### References:\n[Validator keys](https://xrpl.org/docs/concepts/consensus-protocol/validator-keys)", "gridPos": { - "h": 14, + "h": 12, "w": 24, "x": 0, - "y": 78 + "y": 48 }, "fieldConfig": { "defaults": { @@ -1353,7 +1385,10 @@ "datasource": { "type": "loki", "uid": "${DS_LOKI}" - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 }, { "type": "row", @@ -1363,7 +1398,7 @@ "h": 1, "w": 24, "x": 0, - "y": 92 + "y": 60 }, "panels": [], "id": 21 @@ -1373,10 +1408,10 @@ "title": "Fee Charge Rate By Reason", "description": "###### What this is:\n*Rate at which this node charges peers a resource fee, split by the reason for the charge.*\n\n###### How it's computed:\n*Per-second count of Resource log lines, with the reason parsed from the `for ($)` text.*\n\n###### Reading it:\n*Shows what kind of peer behaviour is costing this node the most. Moderate peer request normally dominates.*\n\n###### Healthy range:\n*Workload-dependent and roughly proportional to peer count and request volume.*\n\n###### Watch for:\n*A rising useless data rate, which means peers are sending malformed or unwanted payloads.*\n\n###### Keywords:\n- **Resource fee** *(per peer)* — an internal load credit charged against a peer, unrelated to XRP transaction fees.\n- **Reason** *(per charge)* — why the charge was applied, e.g. moderate peer request, useless data.\n\n###### Computation boundary:\n*Result: Per node per reason — a count of charge events.*\n*NOT recorded as a metric. Derived in the Grafana query from `Logic::charge` log output.*\n\n###### Source:\n[ResourceManager.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/resource/detail/ResourceManager.cpp)\n\n###### Function:\n`Logic::charge`\n\n###### References:\n[Peer protocol](https://xrpl.org/docs/concepts/networks-and-servers/peer-protocol)", "gridPos": { - "h": 10, + "h": 8, "w": 12, "x": 0, - "y": 93 + "y": 61 }, "fieldConfig": { "defaults": { @@ -1384,13 +1419,14 @@ "unit": "suffix: charges/s", "custom": { "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, + "lineWidth": 1, + "fillOpacity": 0, "axisLabel": "Charges / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -1432,10 +1468,10 @@ "title": "Fee-Weighted Charge Load", "description": "###### What this is:\n*Resource fee charges weighted by the fee amount, rather than counted equally.*\n\n###### How it's computed:\n*Per-second sum of the fee value parsed from each Resource charge line, using LogQL unwrap.*\n\n###### Reading it:\n*A single heavy peer request at 2000 outweighs eight moderate ones at 250, so this ranks real cost rather than event count.*\n\n###### Healthy range:\n*Workload-dependent; should track peer request volume smoothly.*\n\n###### Watch for:\n*Spikes that do not appear in the charge-count panel, which mean a shift toward expensive request types.*\n\n###### Keywords:\n- **Fee weight** *(per charge)* — the credit amount, e.g. 250 moderate, 2000 heavy; higher means costlier.\n\n###### Computation boundary:\n*Result: Per node per reason — a SUM of fee amounts, not a count of events.*\n*Derived in the Grafana query via LogQL `unwrap` over the parsed fee value.*\n\n###### Source:\n[ResourceManager.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/resource/detail/ResourceManager.cpp)\n\n###### Function:\n`Logic::charge`\n\n###### References:\n[Peer protocol](https://xrpl.org/docs/concepts/networks-and-servers/peer-protocol)", "gridPos": { - "h": 10, + "h": 8, "w": 12, "x": 12, - "y": 93 + "y": 61 }, "fieldConfig": { "defaults": { @@ -1443,13 +1479,14 @@ "unit": "suffix: fee/s", "custom": { "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, + "lineWidth": 1, + "fillOpacity": 0, "axisLabel": "Fee Units / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -1488,13 +1525,13 @@ }, { "type": "table", - "title": "Top $topn Charged Peers By IP And Public Key", + "title": "Top $topn Charged Peers By IP And Public Key [$xrpl_network_type]", "description": "###### What this is:\n*The peers accumulating the most resource fee charges, identified by IP address and node public key.*\n\n###### How it's computed:\n*Sum of fee amounts parsed from Resource charge lines, grouped by IP and public key, limited to the top N.*\n\n###### Reading it:\n*This is the direct answer to which peer is costing this node the most, and is the primary abuse-attribution view.*\n\n###### Healthy range:\n*Charges spread across many peers with no single dominant entry.*\n\n###### Watch for:\n*One IP far above the rest, especially paired with the useless data reason, which indicates an abusive or misbehaving peer.*\n\n###### Keywords:\n- **Node public key** *(per peer)* — the peer's base58 identity, stable across reconnects.\n- **IP address** *(per peer)* — source address; the optional `:port` suffix is stripped so one peer is one row.\n\n###### Computation boundary:\n*Result: Per peer per node — a SUM of fee amounts over the dashboard window.*\n*Derived in the Grafana query. Truncated to top N by Loki's 2000-series limit, so this is the head of the distribution, never a total.*\n\n###### Source:\n[ResourceManager.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/resource/detail/ResourceManager.cpp)\n\n###### Function:\n`Logic::charge`\n\n###### Note:\n*Truncated to the top N because of Loki's 2000-series query limit.*\n\n###### References:\n[Peer protocol](https://xrpl.org/docs/concepts/networks-and-servers/peer-protocol)", "gridPos": { - "h": 14, + "h": 12, "w": 24, "x": 0, - "y": 103 + "y": 69 }, "fieldConfig": { "defaults": { @@ -1633,7 +1670,10 @@ "datasource": { "type": "loki", "uid": "${DS_LOKI}" - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 }, { "type": "row", @@ -1643,7 +1683,7 @@ "h": 1, "w": 24, "x": 0, - "y": 117 + "y": 81 }, "panels": [], "id": 25 @@ -1653,10 +1693,10 @@ "title": "Ledger Node Fetch Duplicate Ratio", "description": "###### What this is:\n*The fraction of fetched ledger nodes that were duplicates this node already held.*\n\n###### How it's computed:\n*Sum of the dupe counter divided by the sum of good plus dupe, parsed from the `Ledger AS/TX node stats` log lines. Each field is matched independently because the emitter omits any counter that is zero.*\n\n###### Reading it:\n*This is wasted ledger-fetch bandwidth. A ratio of 0.5 means half of everything fetched was already present.*\n\n###### Healthy range:\n*Below roughly 0.3. Some duplication is unavoidable when fetching from several peers at once.*\n\n###### Watch for:\n*Sustained values above 0.8. Live observation has recorded good:142 dupe:891, an 86 percent duplicate rate, which wastes both bandwidth and peer resource credit.*\n\n###### Keywords:\n- **Ledger node** *(per fetch)* — one SHAMap node fetched while acquiring a ledger.\n- **Duplicate** *(per fetch)* — a node already held locally; wasted bandwidth and peer credit.\n\n###### Computation boundary:\n*Result: Per node — a ratio of two summed counters, dupe / (good + dupe).*\n*NOT recorded as a metric. Derived in the Grafana query via `unwrap` over the counters in the InboundLedger stats line.*\n\n###### Source:\n[InboundLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/InboundLedger.cpp)\n\n###### Function:\n`InboundLedger::onTimer / takeAsRootNode`\n\n###### References:\n[Ledgers](https://xrpl.org/docs/concepts/ledgers)", "gridPos": { - "h": 10, + "h": 8, "w": 12, "x": 0, - "y": 118 + "y": 82 }, "fieldConfig": { "defaults": { @@ -1665,13 +1705,14 @@ "max": 1, "custom": { "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, + "lineWidth": 1, + "fillOpacity": 0, "axisLabel": "Duplicate Ratio", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -1731,23 +1772,24 @@ "title": "Ledger Node Fetch Rate — Good Vs Duplicate Vs Timeout", "description": "###### What this is:\n*Absolute rate of useful ledger nodes fetched, duplicates received, and acquire timeouts.*\n\n###### How it's computed:\n*Per-second sums of the good, dupe, and timeouts counters parsed from InboundLedger acquire log lines.*\n\n###### Reading it:\n*Gives the absolute volumes behind the duplicate-ratio panel, so a high ratio at trivial volume can be told apart from a high ratio at scale.*\n\n###### Healthy range:\n*Good should exceed duplicate during active sync. Timeouts should stay near zero.*\n\n###### Watch for:\n*A rising timeout series, which means peers are not answering ledger requests and sync will stall.*\n\n###### Keywords:\n- **Good** *(per fetch)* — a useful, previously unheld ledger node.\n- **Timeout** *(per acquire)* — a ledger request a peer never answered.\n\n###### Computation boundary:\n*Result: Per node — SUMS of the parsed counters, giving absolute volumes behind the ratio panel.*\n*Derived in the Grafana query via `unwrap`.*\n\n###### Source:\n[InboundLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/InboundLedger.cpp)\n\n###### Function:\n`InboundLedger::onTimer`\n\n###### References:\n[Ledgers](https://xrpl.org/docs/concepts/ledgers)", "gridPos": { - "h": 10, + "h": 8, "w": 12, "x": 12, - "y": 118 + "y": 82 }, "fieldConfig": { "defaults": { "unit": "suffix: nodes/s", "custom": { "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, + "lineWidth": 1, + "fillOpacity": 0, "axisLabel": "Ledger Nodes / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -1811,7 +1853,7 @@ "h": 1, "w": 24, "x": 0, - "y": 128 + "y": 90 }, "panels": [], "id": 28 @@ -1821,10 +1863,10 @@ "title": "Peer Disconnect Rate By Reason", "description": "###### What this is:\n*Rate of peer connection endings, split by the reason recorded in the log.*\n\n###### How it's computed:\n*Per-second count of Peer log lines matching Timeout, Closed, or a refused connection attempt.*\n\n###### Reading it:\n*Distinguishes clean teardown from failure. Closed is a normal ending; Timeout and Connection refused are not.*\n\n###### Healthy range:\n*Closed dominant with a low, steady background of the others.*\n\n###### Watch for:\n*A Timeout rate approaching the Closed rate, which points at network trouble or unresponsive peers.*\n\n###### Keywords:\n- **Closed** *(per peer)* — a clean connection teardown; the normal ending.\n- **Timeout** *(per peer)* — the peer stopped responding.\n- **Connection refused** *(per attempt)* — an outbound attempt the remote rejected.\n\n###### Computation boundary:\n*Result: Per node per outcome — a count of peer lifecycle events.*\n*Derived in the Grafana query. Note `overlay_peer_disconnects` exists as a metric but carries no reason breakdown, which is what this panel adds.*\n\n###### Source:\n[PeerImp.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/PeerImp.cpp)\n\n###### Function:\n`PeerImp::close / onTimer`\n\n###### References:\n[Peer protocol](https://xrpl.org/docs/concepts/networks-and-servers/peer-protocol)", "gridPos": { - "h": 10, + "h": 8, "w": 12, "x": 0, - "y": 129 + "y": 91 }, "fieldConfig": { "defaults": { @@ -1832,13 +1874,14 @@ "unit": "suffix: events/s", "custom": { "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, + "lineWidth": 1, + "fillOpacity": 0, "axisLabel": "Peer Events / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -1880,23 +1923,24 @@ "title": "Peer Handshake And Accept Rate", "description": "###### What this is:\n*Rate of completed peer handshakes and accepted inbound connections.*\n\n###### How it's computed:\n*Per-second count of PeerFinder log lines matching handshake and accept events.*\n\n###### Reading it:\n*Read together with the disconnect panel: healthy churn has handshakes roughly balancing disconnects.*\n\n###### Healthy range:\n*Low and steady once the node has a full peer slate.*\n\n###### Watch for:\n*A high handshake rate alongside a high disconnect rate, which means peers connect and immediately drop.*\n\n###### Keywords:\n- **Handshake** *(per peer)* — protocol negotiation completed with a peer.\n- **Accept** *(per peer)* — an inbound connection admitted to a peer slot.\n\n###### Computation boundary:\n*Result: Per node — counts of PeerFinder events.*\n*Derived in the Grafana query.*\n\n###### Source:\n[Logic.h](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/peerfinder/detail/Logic.h)\n\n###### Function:\n`Logic::on_handshake / on_accept`\n\n###### References:\n[Peer protocol](https://xrpl.org/docs/concepts/networks-and-servers/peer-protocol)", "gridPos": { - "h": 10, + "h": 8, "w": 12, "x": 12, - "y": 129 + "y": 91 }, "fieldConfig": { "defaults": { "unit": "suffix: events/s", "custom": { "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, + "lineWidth": 1, + "fillOpacity": 0, "axisLabel": "Peer Events / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -1951,7 +1995,7 @@ "h": 1, "w": 24, "x": 0, - "y": 139 + "y": 99 }, "panels": [], "id": 31 @@ -1961,10 +2005,10 @@ "title": "Consensus Phase Transition Rate", "description": "###### What this is:\n*Rate of consensus phase transitions into Open, Establish, and Accepted.*\n\n###### How it's computed:\n*Per-second count of log lines announcing a ConsensusPhase transition, grouped by the target phase.*\n\n###### Reading it:\n*A healthy node cycles Open to Establish to Accepted once per ledger, so all three series should track together.*\n\n###### Healthy range:\n*Roughly one transition per phase per ledger interval, near 0.25 per second on a 4-second close.*\n\n###### Watch for:\n*Establish transitions outnumbering Accepted, which means rounds start but fail to converge.*\n\n###### Keywords:\n- **Consensus phase** *(per round)* — Open collects transactions, Establish converges on a set, Accepted applies it.\n- **Transition** *(per round)* — one phase change; a healthy node cycles all three once per ledger.\n\n###### Computation boundary:\n*Result: Per node per phase — a count of transitions.*\n*Derived in the Grafana query by regex over the phase-change log line.*\n\n###### Source:\n[Consensus.h](https://github.com/XRPLF/rippled/blob/develop/include/xrpl/consensus/Consensus.h)\n\n###### Function:\n`Consensus::phase transition logging`\n\n###### References:\n[Consensus](https://xrpl.org/docs/concepts/consensus-protocol)", "gridPos": { - "h": 10, + "h": 8, "w": 12, "x": 0, - "y": 140 + "y": 100 }, "fieldConfig": { "defaults": { @@ -1972,13 +2016,14 @@ "unit": "suffix: transitions/s", "custom": { "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, + "lineWidth": 1, + "fillOpacity": 0, "axisLabel": "Transitions / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -2020,10 +2065,10 @@ "title": "Consensus Operating Mode Rate", "description": "###### What this is:\n*Rate of consensus heartbeat log lines, split by the operating mode reported in each.*\n\n###### How it's computed:\n*Per-second count of ConsensusLogger heartbeat lines, with the mode parsed from the `mode: ` text.*\n\n###### Reading it:\n*This acts as a mode-occupancy proxy: the heartbeat fires about once per second, so the dominant series is the node's current mode.*\n\n###### Healthy range:\n*Almost entirely the full series on a synced node.*\n\n###### Watch for:\n*Any sustained syncing, observing, or tracking share, which means the node is not participating normally.*\n\n###### Keywords:\n- **Consensus mode** *(per node)* — the node's participation level: full proposing, observing, syncing, tracking.\n- **Heartbeat** *(per second)* — the ConsensusLogger timer line, emitted roughly once per second.\n\n###### Computation boundary:\n*Result: Per node per mode — a count of heartbeat lines, used as a mode-occupancy PROXY rather than a true duration.*\n*Derived in the Grafana query. For exact durations use the state_accounting metrics on the Node Health dashboard.*\n\n###### Source:\n[NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`ConsensusLogger heartbeat`\n\n###### References:\n[Consensus](https://xrpl.org/docs/concepts/consensus-protocol)", "gridPos": { - "h": 10, + "h": 8, "w": 12, "x": 12, - "y": 140 + "y": 100 }, "fieldConfig": { "defaults": { @@ -2031,13 +2076,14 @@ "unit": "suffix: beats/s", "custom": { "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, + "lineWidth": 1, + "fillOpacity": 0, "axisLabel": "Heartbeats / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -2079,10 +2125,10 @@ "title": "Validator List Quorum And Trusted Set Size", "description": "###### What this is:\n*The quorum threshold and trusted validator count this node computed, as recorded in the log.*\n\n###### How it's computed:\n*Quorum and total are parsed from the `Using quorum of N for new set of M trusted validators` line and plotted as values.*\n\n###### Reading it:\n*Confirms the node agrees with the network on how many validators it trusts and how many must agree.*\n\n###### Healthy range:\n*Stable, with quorum at roughly 80 percent of the trusted total.*\n\n###### Watch for:\n*A drop in the trusted total, which shrinks the quorum and weakens the node's safety margin.*\n\n###### Keywords:\n- **Quorum** *(per node)* — how many trusted validators must agree; normally about 80 percent of the trusted set.\n- **Trusted set** *(per node)* — validators from the UNL this node currently trusts.\n\n###### Computation boundary:\n*Result: Per node — the last VALUE parsed from the log line, not a rate or count.*\n*Derived in the Grafana query via `unwrap`. Overlaps the `unl_quorum` metric; prefer that metric for the value and this panel for churn events.*\n\n###### Source:\n[ValidatorList.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/detail/ValidatorList.cpp)\n\n###### Function:\n`ValidatorList::updateTrusted`\n\n###### References:\n[Unique Node List](https://xrpl.org/docs/concepts/consensus-protocol/unique-node-list)", "gridPos": { - "h": 10, - "w": 24, + "h": 8, + "w": 12, "x": 0, - "y": 150 + "y": 108 }, "fieldConfig": { "defaults": { @@ -2090,17 +2136,19 @@ "min": 0, "custom": { "drawStyle": "line", - "lineWidth": 2, + "lineWidth": 1, "fillOpacity": 0, "axisLabel": "Validators", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" - } + }, + "decimals": 0 }, "overrides": [] }, @@ -2151,7 +2199,7 @@ "h": 1, "w": 24, "x": 0, - "y": 160 + "y": 116 }, "panels": [], "id": 35 @@ -2161,10 +2209,10 @@ "title": "Slow Job Run Time (p99)", "description": "###### What this is:\n*The 99th percentile run time of jobs that breached their latency target.*\n\n###### How it's computed:\n*Quantile over the run millisecond value parsed from LoadMonitor job lines, grouped by job name.*\n\n###### Reading it:\n*Identifies which job types are the slowest when they do run long.*\n\n###### Healthy range:\n*Only a handful of job names should appear at all. Values in the tens of seconds indicate real stalls.*\n\n###### Watch for:\n*ProcessLData or InboundLedger dominating, which is the signature of the cold-read sync bottleneck.*\n\n###### Keywords:\n- **Job** *(per task)* — a unit of work on xrpld's job queue, e.g. ProcessLData, InboundLedger.\n- **Run time** *(per job)* — time spent executing, excluding queue wait.\n- **Breach** *(per job)* — an execution exceeding the 500ms LoadMonitor threshold; only these are logged.\n\n###### Computation boundary:\n*Result: Per node per job type — a p99 over BREACHES ONLY, never over all executions.*\n*Derived in the Grafana query via `unwrap`. For total job counts and latencies use the native `job_*` metrics on the Job Queue Analysis dashboard.*\n\n###### Source:\n[LoadMonitor.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/core/detail/LoadMonitor.cpp)\n\n###### Function:\n`LoadMonitor::addLoadSample`\n\n###### Note:\n*These lines are only emitted above a 500ms latency threshold, so this panel counts BREACHES, never total job executions. Use the Job Queue Analysis dashboard for totals.*\n\n###### References:\n[Job queue](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/core/detail/JobQueue.cpp)", "gridPos": { - "h": 10, + "h": 8, "w": 12, "x": 0, - "y": 161 + "y": 117 }, "fieldConfig": { "defaults": { @@ -2173,13 +2221,14 @@ "min": 0, "custom": { "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, + "lineWidth": 1, + "fillOpacity": 0, "axisLabel": "Run Time (ms)", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -2221,10 +2270,10 @@ "title": "Slow Job Breach Rate By Job Name (Top $topn)", "description": "###### What this is:\n*How often each job type breaches its latency target.*\n\n###### How it's computed:\n*Per-second count of LoadMonitor job lines grouped by job name, limited to the top N.*\n\n###### Reading it:\n*Read with the p99 panel: a job can breach rarely but severely, or often but mildly.*\n\n###### Healthy range:\n*Near zero. Any sustained non-zero rate means a job type is routinely missing its target.*\n\n###### Watch for:\n*A step change after a deploy, which points at a regression in that job's path.*\n\n###### Keywords:\n- **Breach** *(per job)* — an execution over the 500ms threshold.\n- **Job type** *(per task)* — the named job class, e.g. sweep, ProcessLData.\n\n###### Computation boundary:\n*Result: Per node per job type — a count of BREACHES, not of executions.*\n*Derived in the Grafana query; truncated to top N.*\n\n###### Source:\n[LoadMonitor.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/core/detail/LoadMonitor.cpp)\n\n###### Function:\n`LoadMonitor::addLoadSample`\n\n###### Note:\n*Counts BREACHES above the 500ms threshold, not total job executions.*\n\n###### References:\n[Job queue](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/core/detail/JobQueue.cpp)", "gridPos": { - "h": 10, + "h": 8, "w": 12, "x": 12, - "y": 161 + "y": 117 }, "fieldConfig": { "defaults": { @@ -2232,13 +2281,14 @@ "unit": "suffix: breaches/s", "custom": { "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, + "lineWidth": 1, + "fillOpacity": 0, "axisLabel": "Breaches / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -2280,10 +2330,10 @@ "title": "Slow Job Queue Wait Time (p99)", "description": "###### What this is:\n*The 99th percentile queue wait time of jobs that breached their latency target.*\n\n###### How it's computed:\n*Quantile over the wait millisecond value parsed from LoadMonitor job lines, grouped by job name.*\n\n###### Reading it:\n*Separates queueing delay from execution cost. High wait with low run means the job queue is saturated rather than the work being slow.*\n\n###### Healthy range:\n*Near zero. Live observation shows wait at 0ms for most breaches, meaning run time is the cause.*\n\n###### Watch for:\n*Wait time rising above run time, which shifts the diagnosis from slow work to a saturated job queue.*\n\n###### Keywords:\n- **Wait time** *(per job)* — time queued before execution began.\n- **Run versus wait** *(per job)* — high wait means a saturated queue; high run means slow work.\n\n###### Computation boundary:\n*Result: Per node per job type — a p99 over BREACHES ONLY.*\n*Derived in the Grafana query via `unwrap`.*\n\n###### Source:\n[LoadMonitor.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/core/detail/LoadMonitor.cpp)\n\n###### Function:\n`LoadMonitor::addLoadSample`\n\n###### Note:\n*Counts BREACHES above the 500ms threshold, not total job executions.*\n\n###### References:\n[Job queue](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/core/detail/JobQueue.cpp)", "gridPos": { - "h": 10, - "w": 24, + "h": 8, + "w": 12, "x": 0, - "y": 171 + "y": 125 }, "fieldConfig": { "defaults": { @@ -2292,13 +2342,14 @@ "min": 0, "custom": { "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, + "lineWidth": 1, + "fillOpacity": 0, "axisLabel": "Queue Wait Time (ms)", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -2343,7 +2394,7 @@ "h": 1, "w": 24, "x": 0, - "y": 181 + "y": 133 }, "panels": [], "id": 39 @@ -2353,10 +2404,10 @@ "title": "Warning And Error Rate By Partition (Top $topn)", "description": "###### What this is:\n*Rate of WRN, ERR, and FTL log lines by partition.*\n\n###### How it's computed:\n*Per-second count of log lines at warning severity or above, grouped by partition and limited to the top N.*\n\n###### Reading it:\n*This is the one row that works fully at default log level, so it is the first place to look on an unmodified node.*\n\n###### Healthy range:\n*Low and flat. LoadMonitor warnings are expected on a busy node.*\n\n###### Watch for:\n*Any ERR or FTL series appearing, and step changes in a partition that is normally quiet.*\n\n###### Keywords:\n- **Severity** *(per line)* — WRN, ERR, or FTL; the levels a default-configured node still writes.\n- **Partition** *(per line)* — the emitting xrpld subsystem.\n\n###### Computation boundary:\n*Result: Per node per partition per severity — a count of log lines.*\n*Derived in the Grafana query. This row is the only one that works fully at default log level.*\n\n###### Source:\n[Log.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/basics/Log.cpp)\n\n###### Function:\n`Logs::Sink::write`\n\n###### References:\n[Loki log queries](https://grafana.com/docs/loki/latest/query/log_queries/)", "gridPos": { - "h": 10, + "h": 8, "w": 12, "x": 0, - "y": 182 + "y": 134 }, "fieldConfig": { "defaults": { @@ -2364,13 +2415,14 @@ "unit": "suffix: lines/s", "custom": { "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, + "lineWidth": 1, + "fillOpacity": 0, "axisLabel": "Log Lines / Sec", "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -2431,10 +2483,10 @@ "title": "Recent Warnings And Errors", "description": "###### What this is:\n*The most recent log lines at warning severity or above.*\n\n###### How it's computed:\n*Raw log stream filtered to WRN, ERR, and FTL, newest first.*\n\n###### Reading it:\n*Use this to read the actual text behind a spike in the rate panel beside it.*\n\n###### Healthy range:\n*Sparse. LoadMonitor slow-job warnings are the common benign entry.*\n\n###### Watch for:\n*Repeating identical errors, which indicate a stuck retry loop rather than a transient fault.*\n\n###### Keywords:\n- **Log stream** *(per node)* — the raw lines behind the aggregate rates, newest first.\n\n###### Computation boundary:\n*Result: Raw log lines, no aggregation.*\n*Rendered directly from Loki; no computation applied.*\n\n###### Source:\n[Log.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/basics/Log.cpp)\n\n###### Function:\n`Logs::Sink::write`\n\n###### References:\n[Loki log queries](https://grafana.com/docs/loki/latest/query/log_queries/)", "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 182 + "h": 12, + "w": 24, + "x": 0, + "y": 142 }, "fieldConfig": { "defaults": {}, diff --git a/docker/telemetry/grafana/dashboards/network-traffic.json b/docker/telemetry/grafana/dashboards/network-traffic.json index a74a889a6c..8851b9f09f 100644 --- a/docker/telemetry/grafana/dashboards/network-traffic.json +++ b/docker/telemetry/grafana/dashboards/network-traffic.json @@ -39,6 +39,19 @@ "id": null, "links": [], "panels": [ + { + "title": "Peer Connectivity", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "collapsed": false, + "panels": [], + "id": 1 + }, { "title": "Active Peers", "description": "###### What this is:\n*Number of active inbound and outbound peer connections the node currently holds.*\n\n###### How it's computed:\n*Current value of the inbound and outbound active-peer counts per node.*\n\n###### Reading it:\n*Outbound is what the node dials out; inbound is what others open to it. Both should be stable.*\n\n###### Healthy range:\n*roughly 10-21 outbound and 0-85 inbound on mainnet, depending on config.*\n\n###### Watch for:\n*Outbound dropping toward zero (isolation) or inbound pinned at the limit with churn (connection pressure).*\n\n###### Keywords:\n- **Peer** *(per node)* \u2014 another server this node holds a protocol connection to.\n- **Overlay** *(network-wide)* \u2014 the peer-to-peer mesh xrpld nodes form to gossip transactions, proposals and validations.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[PeerfinderManager.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/peerfinder/detail/PeerfinderManager.cpp)\n\n###### Function:\n`Logic Stats ctor`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#peer)", @@ -47,7 +60,7 @@ "h": 8, "w": 12, "x": 0, - "y": 0 + "y": 1 }, "options": { "tooltip": { @@ -81,11 +94,16 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 - } + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" + }, + "decimals": 0 }, "overrides": [] - } + }, + "id": 2 }, { "title": "Peer Disconnects", @@ -95,7 +113,7 @@ "h": 8, "w": 12, "x": 12, - "y": 0 + "y": 1 }, "options": { "tooltip": { @@ -122,11 +140,16 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 - } + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" + }, + "decimals": 0 }, "overrides": [] - } + }, + "id": 3 }, { "title": "Total Network Bytes", @@ -136,7 +159,7 @@ "h": 8, "w": 12, "x": 0, - "y": 8 + "y": 9 }, "options": { "tooltip": { @@ -170,11 +193,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 4 }, { "title": "Total Network Messages", @@ -184,7 +211,7 @@ "h": 8, "w": 12, "x": 12, - "y": 8 + "y": 9 }, "options": { "tooltip": { @@ -218,11 +245,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 5 }, { "title": "Transaction Traffic", @@ -232,7 +263,7 @@ "h": 8, "w": 12, "x": 0, - "y": 16 + "y": 17 }, "options": { "tooltip": { @@ -273,11 +304,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 6 }, { "title": "Proposal Traffic", @@ -287,7 +322,7 @@ "h": 8, "w": 12, "x": 12, - "y": 16 + "y": 17 }, "options": { "tooltip": { @@ -335,11 +370,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 7 }, { "title": "Validation Traffic", @@ -349,7 +388,7 @@ "h": 8, "w": 12, "x": 0, - "y": 24 + "y": 25 }, "options": { "tooltip": { @@ -397,21 +436,38 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 8 }, { - "title": "Overlay Traffic by Category (Bytes In)", + "title": "Traffic by Category", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 33 + }, + "collapsed": false, + "panels": [], + "id": 9 + }, + { + "title": "Overlay Traffic by Category (Bytes In) [$xrpl_network_type]", "description": "###### What this is:\n*Top overlay traffic categories ranked by inbound bytes, excluding the all-traffic total.*\n\n###### How it's computed:\n*Top categories by inbound byte rate per node.*\n\n###### Reading it:\n*Shows which message types dominate receive bandwidth right now.*\n\n###### Healthy range:\n*workload-dependent; transactions, proposals, and validations typically lead on a synced node.*\n\n###### Watch for:\n*A fetch or ledger-data category topping the list (sync activity) or an unexpected category dominating.*\n\n###### Keywords:\n- **Overlay** *(per node)* \u2014 the peer-to-peer network layer over which nodes exchange transactions, proposals, and validations.\n- **Proposal** *(network event)* \u2014 a validator's advertised set of candidate transactions for the next ledger, revised each round.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`\n\n###### References:\n[Overlay](https://xrpl.org/docs/concepts/networks-and-servers/peer-protocol) \u00b7 [Proposal](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#overlay)", "type": "bargauge", "gridPos": { - "h": 8, + "h": 4, "w": 12, - "x": 12, - "y": 24 + "x": 0, + "y": 34 }, "options": { "tooltip": { @@ -699,7 +755,11 @@ ] } ] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 10 }, { "title": "Duplicate Traffic (Wasted Bandwidth)", @@ -709,7 +769,7 @@ "h": 8, "w": 12, "x": 0, - "y": 32 + "y": 38 }, "options": { "tooltip": { @@ -771,11 +831,28 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 11 + }, + { + "title": "Detail", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 46 + }, + "collapsed": false, + "panels": [], + "id": 12 }, { "title": "All Traffic Categories (Detail)", @@ -784,8 +861,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 32 + "x": 0, + "y": 47 }, "options": { "tooltip": { @@ -812,11 +889,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 13 } ], "schemaVersion": 39, diff --git a/docker/telemetry/grafana/dashboards/node-health.json b/docker/telemetry/grafana/dashboards/node-health.json index 518f7bd5ca..1a744439ae 100644 --- a/docker/telemetry/grafana/dashboards/node-health.json +++ b/docker/telemetry/grafana/dashboards/node-health.json @@ -84,11 +84,11 @@ }, "gridPos": { "h": 4, - "w": 24, + "w": 12, "x": 0, "y": 0 }, - "id": 129, + "id": 1, "options": { "colorMode": "background", "graphMode": "none", @@ -115,8 +115,11 @@ "expr": "label_replace(label_join(label_replace((server_info{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"server_state\"} == bool 4) * on(service_instance_id) (ledgermaster_validated_ledger_age{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"} < bool 30), \"series\", \"Node Health\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" } ], - "title": "Node Health", - "type": "stat" + "title": "Node Health [$xrpl_network_type]", + "type": "stat", + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 }, { "datasource": { @@ -148,12 +151,12 @@ } }, "gridPos": { - "h": 12, + "h": 4, "w": 12, - "x": 0, - "y": 4 + "x": 12, + "y": 0 }, - "id": 67, + "id": 2, "options": { "colorMode": "value", "graphMode": "area", @@ -185,8 +188,11 @@ "refId": "A" } ], - "title": "Validated Ledger Age", - "type": "stat" + "title": "Validated Ledger Age [$xrpl_network_type]", + "type": "stat", + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 }, { "datasource": { @@ -218,12 +224,12 @@ } }, "gridPos": { - "h": 12, + "h": 4, "w": 12, - "x": 12, + "x": 0, "y": 4 }, - "id": 68, + "id": 3, "options": { "colorMode": "value", "graphMode": "area", @@ -255,8 +261,11 @@ "refId": "A" } ], - "title": "Published Ledger Age", - "type": "stat" + "title": "Published Ledger Age [$xrpl_network_type]", + "type": "stat", + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 }, { "datasource": { @@ -284,12 +293,12 @@ } }, "gridPos": { - "h": 12, + "h": 4, "w": 12, - "x": 0, - "y": 16 + "x": 12, + "y": 4 }, - "id": 69, + "id": 4, "options": { "colorMode": "value", "graphMode": "area", @@ -321,8 +330,11 @@ "refId": "A" } ], - "title": "Ledger Fetch Rate", - "type": "stat" + "title": "Ledger Fetch Rate [$xrpl_network_type]", + "type": "stat", + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 }, { "datasource": { @@ -350,12 +362,12 @@ } }, "gridPos": { - "h": 12, + "h": 4, "w": 12, - "x": 12, - "y": 16 + "x": 0, + "y": 8 }, - "id": 70, + "id": 5, "options": { "colorMode": "value", "graphMode": "area", @@ -387,8 +399,11 @@ "refId": "A" } ], - "title": "Ledger History Mismatches", - "type": "stat" + "title": "Ledger History Mismatches [$xrpl_network_type]", + "type": "stat", + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 }, { "datasource": { @@ -420,12 +435,12 @@ } }, "gridPos": { - "h": 12, + "h": 4, "w": 12, - "x": 0, - "y": 28 + "x": 12, + "y": 8 }, - "id": 71, + "id": 6, "options": { "colorMode": "value", "graphMode": "area", @@ -457,8 +472,11 @@ "refId": "A" } ], - "title": "Ledger Publish Gap", - "type": "stat" + "title": "Ledger Publish Gap [$xrpl_network_type]", + "type": "stat", + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 }, { "datasource": { @@ -490,7 +508,7 @@ "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, - "pointSize": 3, + "pointSize": 5, "scaleDistribution": { "type": "linear" }, @@ -523,12 +541,12 @@ } }, "gridPos": { - "h": 12, + "h": 8, "w": 12, - "x": 12, - "y": 28 + "x": 0, + "y": 12 }, - "id": 72, + "id": 7, "options": { "annotations": { "clustering": -1, @@ -569,12 +587,87 @@ "h": 1, "w": 24, "x": 0, - "y": 40 + "y": 20 }, - "id": 121, + "id": 8, "title": "Operating Mode", "type": "row" }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*The share of recent wall-clock time the node spent in Full mode.*\n\n###### How it's computed:\n*Per-second rate of the Full-mode duration counter divided by the sum of the per-second rates of all five mode duration counters.*\n\n###### Reading it:\n*Higher is better; 1.0 means the node was fully synced for the entire window.*\n\n###### Healthy range:\n*At or above 0.99.*\n\n###### Watch for:\n*Values dropping below 0.9, meaning the node spent meaningful time outside Full mode.*\n\n###### Keywords:\n- **Operating mode / server state** *(per node)* \u2014 the node's sync level: Disconnected, Connected, Syncing, Tracking, Full (and Validating/Proposing).\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`NetworkOPsImp::Stats`\n\n###### References:\n[Operating mode / server state](https://xrpl.org/docs/references/http-websocket-apis/api-conventions/xrpld-server-states) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#operating-mode-server-state)", + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": 0 + }, + { + "color": "yellow", + "value": 0.9 + }, + { + "color": "green", + "value": 0.99 + } + ] + }, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 21 + }, + "id": 9, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + }, + "wideLayout": true + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(state_accounting_full_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) / (rate(state_accounting_disconnected_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) + rate(state_accounting_connected_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) + rate(state_accounting_syncing_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) + rate(state_accounting_tracking_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) + rate(state_accounting_full_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])), \"series\", \"% Time in Full\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "% Time in Full [$xrpl_network_type]", + "type": "stat", + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 + }, { "datasource": { "type": "prometheus", @@ -605,7 +698,7 @@ "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, - "pointSize": 3, + "pointSize": 5, "scaleDistribution": { "type": "linear" }, @@ -640,12 +733,12 @@ } }, "gridPos": { - "h": 12, + "h": 8, "w": 12, "x": 0, - "y": 41 + "y": 25 }, - "id": 74, + "id": 10, "options": { "annotations": { "clustering": -1, @@ -742,7 +835,7 @@ "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, - "pointSize": 3, + "pointSize": 5, "scaleDistribution": { "type": "linear" }, @@ -775,12 +868,12 @@ } }, "gridPos": { - "h": 12, + "h": 8, "w": 12, "x": 12, - "y": 41 + "y": 25 }, - "id": 75, + "id": 11, "options": { "annotations": { "clustering": -1, @@ -877,7 +970,7 @@ "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, - "pointSize": 3, + "pointSize": 5, "scaleDistribution": { "type": "linear" }, @@ -910,12 +1003,12 @@ } }, "gridPos": { - "h": 12, + "h": 8, "w": 12, "x": 0, - "y": 53 + "y": 33 }, - "id": 76, + "id": 12, "options": { "annotations": { "clustering": -1, @@ -982,78 +1075,6 @@ "title": "State Duration Rate (All States)", "type": "timeseries" }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "###### What this is:\n*The share of recent wall-clock time the node spent in Full mode.*\n\n###### How it's computed:\n*Per-second rate of the Full-mode duration counter divided by the sum of the per-second rates of all five mode duration counters.*\n\n###### Reading it:\n*Higher is better; 1.0 means the node was fully synced for the entire window.*\n\n###### Healthy range:\n*At or above 0.99.*\n\n###### Watch for:\n*Values dropping below 0.9, meaning the node spent meaningful time outside Full mode.*\n\n###### Keywords:\n- **Operating mode / server state** *(per node)* \u2014 the node's sync level: Disconnected, Connected, Syncing, Tracking, Full (and Validating/Proposing).\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`NetworkOPsImp::Stats`\n\n###### References:\n[Operating mode / server state](https://xrpl.org/docs/references/http-websocket-apis/api-conventions/xrpld-server-states) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#operating-mode-server-state)", - "fieldConfig": { - "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": 0 - }, - { - "color": "yellow", - "value": 0.9 - }, - { - "color": "green", - "value": 0.99 - } - ] - }, - "unit": "percentunit" - } - }, - "gridPos": { - "h": 12, - "w": 12, - "x": 12, - "y": 53 - }, - "id": 77, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "tooltip": { - "maxHeight": 600, - "mode": "single", - "sort": "none" - }, - "wideLayout": true - }, - "pluginVersion": "13.2.0-28926505616", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(rate(state_accounting_full_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) / (rate(state_accounting_disconnected_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) + rate(state_accounting_connected_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) + rate(state_accounting_syncing_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) + rate(state_accounting_tracking_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) + rate(state_accounting_full_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])), \"series\", \"% Time in Full\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", - "refId": "A" - } - ], - "title": "% Time in Full", - "type": "stat" - }, { "datasource": { "type": "prometheus", @@ -1141,12 +1162,12 @@ } }, "gridPos": { - "h": 24, - "w": 24, - "x": 0, - "y": 65 + "h": 8, + "w": 12, + "x": 12, + "y": 33 }, - "id": 78, + "id": 13, "options": { "alignValue": "left", "annotations": { @@ -1181,8 +1202,11 @@ "refId": "A" } ], - "title": "Operating Mode (State Timeline)", - "type": "state-timeline" + "title": "Operating Mode (State Timeline) [$xrpl_network_type]", + "type": "state-timeline", + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 }, { "collapsed": false, @@ -1190,9 +1214,9 @@ "h": 1, "w": 24, "x": 0, - "y": 89 + "y": 41 }, - "id": 122, + "id": 14, "title": "NodeStore I/O", "type": "row" }, @@ -1216,7 +1240,7 @@ "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 10, + "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, @@ -1225,7 +1249,7 @@ }, "insertNulls": false, "lineInterpolation": "linear", - "lineWidth": 2, + "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" @@ -1259,12 +1283,12 @@ } }, "gridPos": { - "h": 12, + "h": 8, "w": 12, "x": 0, - "y": 90 + "y": 42 }, - "id": 80, + "id": 15, "options": { "annotations": { "clustering": -1, @@ -1335,7 +1359,7 @@ "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 10, + "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, @@ -1344,7 +1368,7 @@ }, "insertNulls": false, "lineInterpolation": "linear", - "lineWidth": 2, + "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" @@ -1378,12 +1402,12 @@ } }, "gridPos": { - "h": 12, + "h": 8, "w": 12, "x": 12, - "y": 90 + "y": 42 }, - "id": 81, + "id": 16, "options": { "annotations": { "clustering": -1, @@ -1438,7 +1462,7 @@ "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 10, + "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, @@ -1447,7 +1471,7 @@ }, "insertNulls": false, "lineInterpolation": "linear", - "lineWidth": 2, + "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" @@ -1485,12 +1509,12 @@ } }, "gridPos": { - "h": 12, + "h": 8, "w": 12, "x": 0, - "y": 102 + "y": 50 }, - "id": 82, + "id": 17, "options": { "annotations": { "clustering": -1, @@ -1553,7 +1577,7 @@ "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 10, + "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, @@ -1562,7 +1586,7 @@ }, "insertNulls": false, "lineInterpolation": "linear", - "lineWidth": 2, + "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" @@ -1596,12 +1620,12 @@ } }, "gridPos": { - "h": 12, + "h": 8, "w": 12, "x": 12, - "y": 102 + "y": 50 }, - "id": 83, + "id": 18, "options": { "annotations": { "clustering": -1, @@ -1664,7 +1688,7 @@ "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 10, + "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, @@ -1673,7 +1697,7 @@ }, "insertNulls": false, "lineInterpolation": "linear", - "lineWidth": 2, + "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" @@ -1703,16 +1727,17 @@ } ] }, - "unit": "short" + "unit": "short", + "decimals": 0 } }, "gridPos": { - "h": 12, + "h": 8, "w": 12, "x": 0, - "y": 114 + "y": 58 }, - "id": 84, + "id": 19, "options": { "annotations": { "clustering": -1, @@ -1783,7 +1808,7 @@ "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 10, + "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, @@ -1792,7 +1817,7 @@ }, "insertNulls": false, "lineInterpolation": "linear", - "lineWidth": 2, + "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" @@ -1826,12 +1851,12 @@ } }, "gridPos": { - "h": 12, + "h": 8, "w": 12, "x": 12, - "y": 114 + "y": 58 }, - "id": 85, + "id": 20, "options": { "annotations": { "clustering": -1, @@ -1872,9 +1897,9 @@ "h": 1, "w": 24, "x": 0, - "y": 126 + "y": 66 }, - "id": 123, + "id": 21, "title": "Jobs", "type": "row" }, @@ -1908,7 +1933,7 @@ "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, - "pointSize": 3, + "pointSize": 5, "scaleDistribution": { "type": "linear" }, @@ -1937,16 +1962,17 @@ } ] }, - "unit": "short" + "unit": "short", + "decimals": 0 } }, "gridPos": { - "h": 12, + "h": 8, "w": 12, "x": 0, - "y": 127 + "y": 67 }, - "id": 87, + "id": 22, "options": { "annotations": { "clustering": -1, @@ -2011,7 +2037,7 @@ "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, - "pointSize": 3, + "pointSize": 5, "scaleDistribution": { "type": "linear" }, @@ -2044,12 +2070,12 @@ } }, "gridPos": { - "h": 12, + "h": 8, "w": 12, "x": 12, - "y": 127 + "y": 67 }, - "id": 88, + "id": 23, "options": { "annotations": { "clustering": -1, @@ -2194,7 +2220,7 @@ "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, - "pointSize": 3, + "pointSize": 5, "scaleDistribution": { "type": "linear" }, @@ -2227,12 +2253,12 @@ } }, "gridPos": { - "h": 12, + "h": 8, "w": 12, "x": 0, - "y": 139 + "y": 75 }, - "id": 89, + "id": 24, "options": { "annotations": { "clustering": -1, @@ -2377,7 +2403,7 @@ "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, - "pointSize": 3, + "pointSize": 5, "scaleDistribution": { "type": "linear" }, @@ -2410,12 +2436,12 @@ } }, "gridPos": { - "h": 16, - "w": 24, - "x": 0, - "y": 151 + "h": 8, + "w": 12, + "x": 12, + "y": 75 }, - "id": 90, + "id": 25, "options": { "annotations": { "clustering": -1, @@ -2480,7 +2506,7 @@ "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, - "pointSize": 3, + "pointSize": 5, "scaleDistribution": { "type": "linear" }, @@ -2513,12 +2539,12 @@ } }, "gridPos": { - "h": 16, - "w": 24, + "h": 8, + "w": 12, "x": 0, - "y": 167 + "y": 83 }, - "id": 91, + "id": 26, "options": { "annotations": { "clustering": -1, @@ -2559,9 +2585,9 @@ "h": 1, "w": 24, "x": 0, - "y": 183 + "y": 91 }, - "id": 124, + "id": 27, "panels": [ { "datasource": { @@ -3007,9 +3033,9 @@ "h": 1, "w": 24, "x": 0, - "y": 184 + "y": 92 }, - "id": 125, + "id": 28, "title": "Server Info", "type": "row" }, @@ -3145,12 +3171,12 @@ ] }, "gridPos": { - "h": 12, + "h": 4, "w": 12, "x": 0, - "y": 185 + "y": 93 }, - "id": 98, + "id": 29, "options": { "colorMode": "background", "graphMode": "none", @@ -3183,8 +3209,11 @@ "refId": "A" } ], - "title": "Server State", - "type": "stat" + "title": "Server State [$xrpl_network_type]", + "type": "stat", + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 }, { "datasource": { @@ -3210,12 +3239,12 @@ } }, "gridPos": { - "h": 12, + "h": 4, "w": 12, "x": 12, - "y": 185 + "y": 93 }, - "id": 99, + "id": 30, "options": { "colorMode": "value", "graphMode": "none", @@ -3245,8 +3274,11 @@ "refId": "A" } ], - "title": "Uptime", - "type": "stat" + "title": "Uptime [$xrpl_network_type]", + "type": "stat", + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 }, { "datasource": { @@ -3276,16 +3308,17 @@ } ] }, - "unit": "short" + "unit": "short", + "decimals": 0 } }, "gridPos": { - "h": 12, + "h": 4, "w": 12, "x": 0, - "y": 197 + "y": 97 }, - "id": 100, + "id": 31, "options": { "colorMode": "value", "graphMode": "none", @@ -3315,8 +3348,11 @@ "refId": "A" } ], - "title": "Peer Count", - "type": "stat" + "title": "Peer Count [$xrpl_network_type]", + "type": "stat", + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 }, { "datasource": { @@ -3340,12 +3376,12 @@ } }, "gridPos": { - "h": 12, + "h": 4, "w": 12, "x": 12, - "y": 197 + "y": 97 }, - "id": 103, + "id": 32, "options": { "colorMode": "none", "graphMode": "area", @@ -3377,8 +3413,11 @@ "refId": "A" } ], - "title": "Current Ledger Index", - "type": "stat" + "title": "Current Ledger Index [$xrpl_network_type]", + "type": "stat", + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 }, { "datasource": { @@ -3438,12 +3477,12 @@ ] }, "gridPos": { - "h": 7, + "h": 4, "w": 12, "x": 0, - "y": 209 + "y": 101 }, - "id": 58, + "id": 33, "options": { "colorMode": "background", "graphMode": "area", @@ -3475,8 +3514,11 @@ "refId": "A" } ], - "title": "Validated Ledger Seq \u2014 Convergence (Max \u2212 Min, per network)", - "type": "stat" + "title": "Validated Ledger Seq — Convergence (Max − Min, per network) [$xrpl_network_type]", + "type": "stat", + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 }, { "datasource": { @@ -3502,12 +3544,12 @@ } }, "gridPos": { - "h": 7, + "h": 4, "w": 12, "x": 12, - "y": 209 + "y": 101 }, - "id": 56, + "id": 34, "options": { "colorMode": "value", "graphMode": "none", @@ -3541,8 +3583,11 @@ "refId": "A" } ], - "title": "Validated Ledger Seq \u2014 Lag Behind Network Tip", - "type": "stat" + "title": "Validated Ledger Seq — Lag Behind Network Tip [$xrpl_network_type]", + "type": "stat", + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 }, { "datasource": { @@ -3564,12 +3609,12 @@ } }, "gridPos": { - "h": 12, + "h": 4, "w": 12, "x": 0, - "y": 216 + "y": 105 }, - "id": 102, + "id": 35, "options": { "colorMode": "none", "graphMode": "none", @@ -3602,8 +3647,11 @@ "refId": "A" } ], - "title": "Build Version", - "type": "stat" + "title": "Build Version [$xrpl_network_type]", + "type": "stat", + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 }, { "datasource": { @@ -3625,7 +3673,7 @@ "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 10, + "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, @@ -3634,7 +3682,7 @@ }, "insertNulls": false, "lineInterpolation": "linear", - "lineWidth": 2, + "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" @@ -3664,12 +3712,12 @@ } }, "gridPos": { - "h": 12, + "h": 8, "w": 12, - "x": 12, - "y": 216 + "x": 0, + "y": 109 }, - "id": 105, + "id": 36, "options": { "annotations": { "clustering": -1, @@ -3724,7 +3772,7 @@ "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 10, + "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, @@ -3733,7 +3781,7 @@ }, "insertNulls": false, "lineInterpolation": "linear", - "lineWidth": 2, + "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" @@ -3759,16 +3807,17 @@ } ] }, - "unit": "short" + "unit": "short", + "decimals": 0 } }, "gridPos": { - "h": 12, + "h": 8, "w": 12, - "x": 0, - "y": 228 + "x": 12, + "y": 109 }, - "id": 104, + "id": 37, "options": { "annotations": { "clustering": -1, @@ -3823,7 +3872,7 @@ "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 10, + "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, @@ -3832,7 +3881,7 @@ }, "insertNulls": false, "lineInterpolation": "linear", - "lineWidth": 2, + "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" @@ -3900,12 +3949,12 @@ ] }, "gridPos": { - "h": 12, + "h": 8, "w": 12, - "x": 12, - "y": 228 + "x": 0, + "y": 117 }, - "id": 128, + "id": 38, "options": { "annotations": { "clustering": -1, @@ -3968,7 +4017,7 @@ "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 10, + "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, @@ -3977,7 +4026,7 @@ }, "insertNulls": false, "lineInterpolation": "linear", - "lineWidth": 2, + "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" @@ -4007,12 +4056,12 @@ } }, "gridPos": { - "h": 12, + "h": 8, "w": 12, - "x": 0, - "y": 240 + "x": 12, + "y": 117 }, - "id": 106, + "id": 39, "options": { "annotations": { "clustering": -1, @@ -4067,7 +4116,7 @@ "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "bars", - "fillOpacity": 80, + "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, @@ -4076,7 +4125,7 @@ }, "insertNulls": false, "lineInterpolation": "linear", - "lineWidth": 0, + "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" @@ -4106,16 +4155,17 @@ } ] }, - "unit": "short" + "unit": "short", + "decimals": 0 } }, "gridPos": { - "h": 16, - "w": 24, + "h": 8, + "w": 12, "x": 0, - "y": 252 + "y": 125 }, - "id": 107, + "id": 40, "options": { "annotations": { "clustering": -1, @@ -4156,12 +4206,81 @@ "h": 1, "w": 24, "x": 0, - "y": 268 + "y": 133 }, - "id": 126, + "id": 41, "title": "Complete Ledgers & DB", "type": "row" }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*How many historical ledgers the node is back-filling per minute.*\n\n###### How it's computed:\n*Current value of the historical-fetch-per-minute gauge.*\n\n###### Reading it:\n*Near zero once history is complete; elevated while back-filling.*\n\n###### Healthy range:\n*Close to zero in steady state.*\n\n###### Watch for:\n*A sustained high rate means the node is still filling gaps in its stored history.*\n\n###### Keywords:\n- **Historical fetch rate** *(per node)* \u2014 how many historical ledgers the node is back-filling per minute.\n- **Back-fill / catch-up** *(per node)* \u2014 fetching missing historical ledgers from peers to fill gaps or reach the network tip.\n- **Complete ledger ranges** *(per node)* \u2014 the contiguous spans of ledgers the node holds locally; one unbroken range is ideal.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerDbMetricsGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#historical-fetch-rate)", + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "suffix: fetches/min" + } + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 134 + }, + "id": 42, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "wideLayout": true + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(db_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"historical_perminute\"}, \"series\", \"Fetches/min\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Historical Fetch Rate [$xrpl_network_type]", + "type": "stat", + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 + }, { "datasource": { "type": "prometheus", @@ -4198,11 +4317,11 @@ }, "gridPos": { "h": 12, - "w": 12, + "w": 24, "x": 0, - "y": 269 + "y": 138 }, - "id": 109, + "id": 43, "options": { "cellHeight": "sm", "showHeader": true, @@ -4222,8 +4341,11 @@ "refId": "A" } ], - "title": "Complete Ledger Ranges", - "type": "table" + "title": "Complete Ledger Ranges [$xrpl_network_type]", + "type": "table", + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 }, { "datasource": { @@ -4245,7 +4367,7 @@ "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 10, + "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, @@ -4254,7 +4376,7 @@ }, "insertNulls": false, "lineInterpolation": "linear", - "lineWidth": 2, + "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" @@ -4288,12 +4410,12 @@ } }, "gridPos": { - "h": 12, + "h": 8, "w": 12, - "x": 12, - "y": 269 + "x": 0, + "y": 150 }, - "id": 110, + "id": 44, "options": { "annotations": { "clustering": -1, @@ -4344,72 +4466,6 @@ "title": "Database Sizes", "type": "timeseries" }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "###### What this is:\n*How many historical ledgers the node is back-filling per minute.*\n\n###### How it's computed:\n*Current value of the historical-fetch-per-minute gauge.*\n\n###### Reading it:\n*Near zero once history is complete; elevated while back-filling.*\n\n###### Healthy range:\n*Close to zero in steady state.*\n\n###### Watch for:\n*A sustained high rate means the node is still filling gaps in its stored history.*\n\n###### Keywords:\n- **Historical fetch rate** *(per node)* \u2014 how many historical ledgers the node is back-filling per minute.\n- **Back-fill / catch-up** *(per node)* \u2014 fetching missing historical ledgers from peers to fill gaps or reach the network tip.\n- **Complete ledger ranges** *(per node)* \u2014 the contiguous spans of ledgers the node holds locally; one unbroken range is ideal.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerDbMetricsGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#historical-fetch-rate)", - "fieldConfig": { - "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "suffix: fetches/min" - } - }, - "gridPos": { - "h": 12, - "w": 12, - "x": 0, - "y": 281 - }, - "id": 111, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "tooltip": { - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - }, - "wideLayout": true - }, - "pluginVersion": "13.2.0-28926505616", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(db_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"historical_perminute\"}, \"series\", \"Fetches/min\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", - "refId": "A" - } - ], - "title": "Historical Fetch Rate", - "type": "stat" - }, { "datasource": { "type": "prometheus", @@ -4425,12 +4481,12 @@ "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "", + "axisLabel": "Disconnects", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 10, + "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, @@ -4439,7 +4495,7 @@ }, "insertNulls": false, "lineInterpolation": "linear", - "lineWidth": 2, + "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" @@ -4469,16 +4525,17 @@ } ] }, - "unit": "short" + "unit": "short", + "decimals": 0 } }, "gridPos": { - "h": 12, + "h": 8, "w": 12, "x": 12, - "y": 281 + "y": 150 }, - "id": 112, + "id": 45, "options": { "annotations": { "clustering": -1, @@ -4519,9 +4576,9 @@ "h": 1, "w": 24, "x": 0, - "y": 293 + "y": 158 }, - "id": 127, + "id": 46, "title": "Ledger Economy", "type": "row" }, @@ -4551,12 +4608,12 @@ } }, "gridPos": { - "h": 12, + "h": 4, "w": 12, "x": 0, - "y": 294 + "y": 159 }, - "id": 114, + "id": 47, "options": { "colorMode": "value", "graphMode": "area", @@ -4588,8 +4645,11 @@ "refId": "A" } ], - "title": "Base Fee (drops)", - "type": "stat" + "title": "Base Fee (drops) [$xrpl_network_type]", + "type": "stat", + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 }, { "datasource": { @@ -4617,12 +4677,12 @@ } }, "gridPos": { - "h": 12, + "h": 4, "w": 12, "x": 12, - "y": 294 + "y": 159 }, - "id": 115, + "id": 48, "options": { "colorMode": "value", "graphMode": "area", @@ -4654,8 +4714,11 @@ "refId": "A" } ], - "title": "Reserve Base (drops)", - "type": "stat" + "title": "Reserve Base (drops) [$xrpl_network_type]", + "type": "stat", + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 }, { "datasource": { @@ -4683,12 +4746,12 @@ } }, "gridPos": { - "h": 12, + "h": 4, "w": 12, "x": 0, - "y": 306 + "y": 163 }, - "id": 116, + "id": 49, "options": { "colorMode": "value", "graphMode": "area", @@ -4720,8 +4783,11 @@ "refId": "A" } ], - "title": "Reserve Inc (drops)", - "type": "stat" + "title": "Reserve Inc (drops) [$xrpl_network_type]", + "type": "stat", + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 }, { "datasource": { @@ -4743,7 +4809,7 @@ "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 10, + "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, @@ -4752,8 +4818,8 @@ }, "insertNulls": false, "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 3, + "lineWidth": 1, + "pointSize": 5, "scaleDistribution": { "type": "linear" }, @@ -4790,12 +4856,12 @@ } }, "gridPos": { - "h": 16, + "h": 8, "w": 12, - "x": 12, - "y": 306 + "x": 0, + "y": 167 }, - "id": 117, + "id": 50, "options": { "annotations": { "clustering": -1, @@ -4850,7 +4916,7 @@ "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 10, + "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, @@ -4859,8 +4925,8 @@ }, "insertNulls": false, "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 3, + "lineWidth": 1, + "pointSize": 5, "scaleDistribution": { "type": "linear" }, @@ -4893,12 +4959,12 @@ } }, "gridPos": { - "h": 16, - "w": 24, - "x": 0, - "y": 322 + "h": 8, + "w": 12, + "x": 12, + "y": 167 }, - "id": 118, + "id": 51, "options": { "annotations": { "clustering": -1, @@ -4963,7 +5029,7 @@ "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, - "pointSize": 3, + "pointSize": 5, "scaleDistribution": { "type": "linear" }, @@ -4996,12 +5062,12 @@ } }, "gridPos": { - "h": 16, + "h": 8, "w": 12, "x": 0, - "y": 338 + "y": 175 }, - "id": 119, + "id": 52, "options": { "annotations": { "clustering": -1, @@ -5066,7 +5132,7 @@ "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, - "pointSize": 3, + "pointSize": 5, "scaleDistribution": { "type": "linear" }, @@ -5099,12 +5165,12 @@ } }, "gridPos": { - "h": 16, + "h": 8, "w": 12, "x": 12, - "y": 338 + "y": 175 }, - "id": 120, + "id": 53, "options": { "annotations": { "clustering": -1, @@ -5146,10 +5212,11 @@ "h": 1, "w": 24, "x": 0, - "y": 354 + "y": 183 }, "collapsed": false, - "panels": [] + "panels": [], + "id": 54 }, { "title": "Job Queue Saturation (Running vs Limit)", @@ -5157,9 +5224,9 @@ "type": "timeseries", "gridPos": { "h": 8, - "w": 24, + "w": 12, "x": 0, - "y": 355 + "y": 184 }, "options": { "tooltip": { @@ -5230,22 +5297,24 @@ "custom": { "axisLabel": "Running / Limit", "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, + "lineWidth": 1, + "fillOpacity": 0, "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3, + "pointSize": 5, "thresholdsStyle": { "mode": "line" - } + }, + "gradientMode": "none" }, "color": { "mode": "palette-classic" } }, "overrides": [] - } + }, + "id": 55 } ], "schemaVersion": 39, diff --git a/docker/telemetry/grafana/dashboards/overlay-traffic-detail.json b/docker/telemetry/grafana/dashboards/overlay-traffic-detail.json index b21a017f81..316bdf5234 100644 --- a/docker/telemetry/grafana/dashboards/overlay-traffic-detail.json +++ b/docker/telemetry/grafana/dashboards/overlay-traffic-detail.json @@ -109,11 +109,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 1 }, { "title": "Overhead Traffic Breakdown (Bytes)", @@ -185,11 +189,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 2 }, { "title": "Validator List Traffic", @@ -247,7 +255,10 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [ @@ -268,7 +279,8 @@ ] } ] - } + }, + "id": 3 }, { "title": "Set Get/Share Traffic (Bytes)", @@ -326,11 +338,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 4 }, { "title": "Have/Requested Transactions (Messages)", @@ -388,11 +404,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 5 }, { "title": "Unknown / Unclassified Traffic", @@ -450,7 +470,10 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [ @@ -471,7 +494,8 @@ ] } ] - } + }, + "id": 6 }, { "title": "Proof Path Traffic", @@ -529,11 +553,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 7 }, { "title": "Replay Delta Traffic", @@ -591,11 +619,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 8 }, { "title": "GetObject Handler (TMGetObjectByHash)", @@ -607,7 +639,8 @@ "y": 32 }, "collapsed": false, - "panels": [] + "panels": [], + "id": 9 }, { "title": "GetObject Handler Latency Breakdown", @@ -615,7 +648,7 @@ "type": "timeseries", "gridPos": { "h": 8, - "w": 24, + "w": 12, "x": 0, "y": 33 }, @@ -655,17 +688,19 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3, + "pointSize": 5, "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 5 + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" }, "color": { "mode": "palette-classic" } }, "overrides": [] - } + }, + "id": 10 }, { "title": "GetObject Request Size Distribution", @@ -674,8 +709,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 41 + "x": 12, + "y": 33 }, "options": { "tooltip": { @@ -703,7 +738,8 @@ "unit": "short" }, "overrides": [] - } + }, + "id": 11 }, { "title": "GetObject Lookups by Result", @@ -712,7 +748,7 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, + "x": 0, "y": 41 }, "options": { @@ -739,11 +775,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 12 }, { "title": "GetObject Rejections", @@ -752,8 +792,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 49 + "x": 12, + "y": 41 }, "options": { "tooltip": { @@ -779,11 +819,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 13 }, { "title": "GetObject Charge Distribution", @@ -792,7 +836,7 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, + "x": 0, "y": 49 }, "options": { @@ -831,17 +875,19 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3, + "pointSize": 5, "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 5 + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" }, "color": { "mode": "palette-classic" } }, "overrides": [] - } + }, + "id": 14 } ], "schemaVersion": 39, diff --git a/docker/telemetry/grafana/dashboards/peer-network.json b/docker/telemetry/grafana/dashboards/peer-network.json index 44fde48888..b27d693f8a 100644 --- a/docker/telemetry/grafana/dashboards/peer-network.json +++ b/docker/telemetry/grafana/dashboards/peer-network.json @@ -39,6 +39,19 @@ "id": null, "links": [], "panels": [ + { + "title": "Proposal & Validation Receive", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "collapsed": false, + "panels": [], + "id": 1 + }, { "title": "Peer Proposal Receive Rate", "description": "###### What this is:\n*How many consensus proposals this node receives from peers per second.*\n\n###### How it's computed:\n*Per-second rate of received proposals over 5 minutes, per node.*\n\n###### Reading it:\n*A steady rate roughly proportional to the number of proposing validators.*\n\n###### Healthy range:\n*Workload-dependent; scales with validator count and connectivity.*\n\n###### Watch for:\n*A drop toward zero (isolation from the network) or a sudden flood far above baseline (proposal spam).*\n\n###### Keywords:\n- **Consensus** *(network event)* \u2014 the protocol by which validators agree on the next ledger's transaction set and close time.\n- **Proposal** *(network event)* \u2014 a validator's advertised set of candidate transactions for the next ledger, revised each round.\n- **Proposers** *(network event)* \u2014 the count of validators whose proposals this node heard in the last closed round.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[PeerImp.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/PeerImp.cpp)\n\n###### Function:\n`PeerImp::onMessage(TMProposeSet)`\n\n###### References:\n[Consensus](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#consensus)", @@ -47,7 +60,7 @@ "h": 8, "w": 12, "x": 0, - "y": 0 + "y": 1 }, "options": { "tooltip": { @@ -74,11 +87,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 2 }, { "title": "Peer Validation Receive Rate", @@ -88,7 +105,7 @@ "h": 8, "w": 12, "x": 12, - "y": 0 + "y": 1 }, "options": { "tooltip": { @@ -115,11 +132,28 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 3 + }, + { + "title": "Trust Split", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "collapsed": false, + "panels": [], + "id": 4 }, { "title": "Proposals Trusted vs Untrusted", @@ -129,7 +163,7 @@ "h": 8, "w": 12, "x": 0, - "y": 8 + "y": 10 }, "options": { "tooltip": { @@ -153,7 +187,8 @@ "unit": "short" }, "overrides": [] - } + }, + "id": 5 }, { "title": "Validations Trusted vs Untrusted", @@ -163,7 +198,7 @@ "h": 8, "w": 12, "x": 12, - "y": 8 + "y": 10 }, "options": { "tooltip": { @@ -187,7 +222,21 @@ "unit": "short" }, "overrides": [] - } + }, + "id": 6 + }, + { + "title": "Reduce-Relay", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 18 + }, + "collapsed": false, + "panels": [], + "id": 7 }, { "title": "Reduce-Relay Peer Selection", @@ -197,7 +246,7 @@ "h": 8, "w": 12, "x": 0, - "y": 16 + "y": 19 }, "options": { "tooltip": { @@ -238,11 +287,16 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 - } + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" + }, + "decimals": 0 }, "overrides": [] - } + }, + "id": 8 }, { "title": "Reduce-Relay Missing-Tx Frequency", @@ -252,7 +306,7 @@ "h": 8, "w": 12, "x": 12, - "y": 16 + "y": 19 }, "options": { "tooltip": { @@ -279,11 +333,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 9 } ], "schemaVersion": 39, diff --git a/docker/telemetry/grafana/dashboards/peer-quality.json b/docker/telemetry/grafana/dashboards/peer-quality.json index e17bb2dda4..ad4bc0c526 100644 --- a/docker/telemetry/grafana/dashboards/peer-quality.json +++ b/docker/telemetry/grafana/dashboards/peer-quality.json @@ -39,15 +39,28 @@ "id": null, "links": [], "panels": [ + { + "title": "Peer Latency", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "collapsed": false, + "panels": [], + "id": 1 + }, { "title": "P90 Peer Latency", "description": "###### What this is:\n*90th-percentile round-trip latency to connected peers, in milliseconds.*\n\n###### How it's computed:\n*Instantaneous gauge reading of the p90 peer latency.*\n\n###### Reading it:\n*Lower is better; green under 200ms, yellow to 500ms, red above.*\n\n###### Healthy range:\n*Under 200ms.*\n\n###### Watch for:\n*Rising latency, which points to network congestion or geographically distant, poorly performing peers.*\n\n###### Keywords:\n- **Peer** *(per node)* \u2014 another server this node holds a protocol connection to.\n- **Overlay** *(network-wide)* \u2014 the peer-to-peer mesh xrpld nodes form to gossip transactions, proposals and validations.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerPeerQualityGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#peer)", "type": "timeseries", "gridPos": { "h": 8, - "w": 24, + "w": 12, "x": 0, - "y": 0 + "y": 1 }, "options": { "tooltip": { @@ -72,12 +85,13 @@ "custom": { "axisLabel": "Latency (ms)", "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, + "lineWidth": 1, + "fillOpacity": 0, "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -100,17 +114,31 @@ } }, "overrides": [] - } + }, + "id": 2 }, { - "title": "Insane/Diverged Peers", + "title": "Peer Health", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "collapsed": false, + "panels": [], + "id": 3 + }, + { + "title": "Insane/Diverged Peers [$xrpl_network_type]", "description": "###### What this is:\n*Count of connected peers whose ledger state has diverged from the network.*\n\n###### How it's computed:\n*Instantaneous gauge reading of the diverged-peer count.*\n\n###### Reading it:\n*Zero is healthy; any count means those peers disagree on ledger state.*\n\n###### Healthy range:\n*0 diverged peers.*\n\n###### Watch for:\n*A persistent non-zero count, which can indicate peers on a fork or misbehaving peers.*\n\n###### Keywords:\n- **Insane / diverged peers** *(per node)* \u2014 connected peers whose ledger state disagrees with the network \u2014 possibly on a fork or misbehaving.\n- **Fork** *(network-wide)* \u2014 when nodes validate divergent ledger chains instead of a single agreed history.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerPeerQualityGauge`\n\n###### References:\n[Fork](https://xrpl.org/docs/concepts/consensus-protocol) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#insane-diverged-peers)", "type": "stat", "gridPos": { - "h": 8, - "w": 24, + "h": 4, + "w": 12, "x": 0, - "y": 8 + "y": 10 }, "options": { "tooltip": { @@ -148,20 +176,25 @@ } ] }, - "custom": {} + "custom": {}, + "decimals": 0 }, "overrides": [] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 4 }, { - "title": "Higher Version Peers %", + "title": "Higher Version Peers % [$xrpl_network_type]", "description": "###### What this is:\n*Percentage of connected peers running a newer rippled version than this node.*\n\n###### How it's computed:\n*Instantaneous gauge reading of the higher-version peer percentage.*\n\n###### Reading it:\n*A high percentage suggests this node is behind and should be upgraded.*\n\n###### Healthy range:\n*Under 30%.*\n\n###### Watch for:\n*A majority of peers on a newer version, a strong upgrade signal.*\n\n###### Keywords:\n- **Peer** *(per node)* \u2014 another server this node holds a protocol connection to.\n- **Overlay** *(network-wide)* \u2014 the peer-to-peer mesh xrpld nodes form to gossip transactions, proposals and validations.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerPeerQualityGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#peer)", "type": "stat", "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 16 + "h": 4, + "w": 12, + "x": 12, + "y": 10 }, "options": { "tooltip": { @@ -204,17 +237,21 @@ "custom": {} }, "overrides": [] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 5 }, { - "title": "Upgrade Recommended", + "title": "Upgrade Recommended [$xrpl_network_type]", "description": "###### What this is:\n*A flag indicating whether an upgrade is advised based on peer version analysis (Yes/No).*\n\n###### How it's computed:\n*Instantaneous gauge reading of the upgrade-recommended flag.*\n\n###### Reading it:\n*No is healthy; Yes means most peers run a newer version.*\n\n###### Healthy range:\n*No.*\n\n###### Watch for:\n*A Yes state, indicating the node risks falling out of step with the network.*\n\n###### Keywords:\n- **Peer** *(per node)* \u2014 another server this node holds a protocol connection to.\n- **Overlay** *(network-wide)* \u2014 the peer-to-peer mesh xrpld nodes form to gossip transactions, proposals and validations.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerPeerQualityGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#peer)", "type": "stat", "gridPos": { - "h": 8, - "w": 24, + "h": 4, + "w": 12, "x": 0, - "y": 24 + "y": 14 }, "options": { "tooltip": { @@ -271,64 +308,34 @@ "custom": {} }, "overrides": [] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 6 }, { - "title": "Resource Disconnects", - "description": "###### What this is:\n*Cumulative count of peers dropped for exceeding resource (load) limits.*\n\n###### How it's computed:\n*Instantaneous gauge reading of the resource-disconnect total over time.*\n\n###### Reading it:\n*A flat line is healthy; a rising line means peers are being dropped for overuse.*\n\n###### Healthy range:\n*Flat / near constant.*\n\n###### Watch for:\n*A steep climb, which flags aggressive or misbehaving peers being shed as backpressure.*\n\n###### Keywords:\n- **Resource disconnect** *(per node)* \u2014 a peer dropped for exceeding resource/load limits \u2014 the node shedding abusive or overactive peers.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerServerInfoGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#resource-disconnect)", - "type": "timeseries", + "title": "Disconnects & Connection Mix", + "type": "row", "gridPos": { - "h": 8, + "h": 1, "w": 24, "x": 0, - "y": 32 + "y": 18 }, - "options": { - "tooltip": { - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(server_info{metric=\"peer_disconnects_resources\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Disconnects\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" - } - ], - "fieldConfig": { - "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "unit": "short", - "custom": { - "axisLabel": "Disconnects", - "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, - "spanNulls": 1800000, - "insertNulls": false, - "showPoints": "auto", - "pointSize": 3 - }, - "color": { - "mode": "palette-classic" - } - }, - "overrides": [] - } + "collapsed": false, + "panels": [], + "id": 7 }, { - "title": "Inbound vs Outbound Peers", + "title": "Inbound vs Outbound Peers [$xrpl_network_type]", "description": "###### What this is:\n*Active inbound versus outbound peer connection counts.*\n\n###### How it's computed:\n*Instantaneous gauge readings of active inbound and outbound peer counts.*\n\n###### Reading it:\n*A balanced mix is healthy for connectivity and resilience.*\n\n###### Healthy range:\n*A mix of both; some outbound connections present.*\n\n###### Watch for:\n*All-inbound with no outbound, which usually means NAT or firewall issues block outbound peering.*\n\n###### Keywords:\n- **Peer** *(per node)* \u2014 another server this node holds a protocol connection to.\n- **Overlay** *(network-wide)* \u2014 the peer-to-peer mesh xrpld nodes form to gossip transactions, proposals and validations.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[PeerfinderManager.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/peerfinder/detail/PeerfinderManager.cpp)\n\n###### Function:\n`Logic Stats ctor`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#peer)", "type": "bargauge", "gridPos": { - "h": 8, - "w": 24, + "h": 4, + "w": 12, "x": 0, - "y": 40 + "y": 19 }, "options": { "orientation": "horizontal", @@ -367,7 +374,8 @@ } ] }, - "custom": {} + "custom": {}, + "decimals": 0 }, "overrides": [ { @@ -401,7 +409,61 @@ ] } ] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 8 + }, + { + "title": "Resource Disconnects", + "description": "###### What this is:\n*Cumulative count of peers dropped for exceeding resource (load) limits.*\n\n###### How it's computed:\n*Instantaneous gauge reading of the resource-disconnect total over time.*\n\n###### Reading it:\n*A flat line is healthy; a rising line means peers are being dropped for overuse.*\n\n###### Healthy range:\n*Flat / near constant.*\n\n###### Watch for:\n*A steep climb, which flags aggressive or misbehaving peers being shed as backpressure.*\n\n###### Keywords:\n- **Resource disconnect** *(per node)* \u2014 a peer dropped for exceeding resource/load limits \u2014 the node shedding abusive or overactive peers.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerServerInfoGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#resource-disconnect)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 23 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(server_info{metric=\"peer_disconnects_resources\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"Disconnects\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "short", + "custom": { + "axisLabel": "Disconnects", + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 0, + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 5, + "gradientMode": "none" + }, + "color": { + "mode": "palette-classic" + }, + "decimals": 0 + }, + "overrides": [] + }, + "id": 9 } ], "schemaVersion": 39, diff --git a/docker/telemetry/grafana/dashboards/rpc-pathfinding.json b/docker/telemetry/grafana/dashboards/rpc-pathfinding.json index 610a7965f0..a23656883e 100644 --- a/docker/telemetry/grafana/dashboards/rpc-pathfinding.json +++ b/docker/telemetry/grafana/dashboards/rpc-pathfinding.json @@ -40,11 +40,11 @@ "links": [], "panels": [ { - "title": "RPC Request Rate", + "title": "RPC Request Rate [$xrpl_network_type]", "description": "###### What this is:\n*How many RPC requests the server counts per second.*\n\n###### How it's computed:\n*Per-second rate of the RPC request counter over 5 minutes, per node.*\n\n###### Reading it:\n*A steady line proportional to client demand; cross-checks the trace-based RPC dashboard.*\n\n###### Healthy range:\n*Workload-dependent; tracks client activity.*\n\n###### Watch for:\n*A sudden surge above baseline, consistent with a client flooding the RPC endpoint.*\n\n###### Keywords:\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[ServerHandler.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/ServerHandler.cpp)\n\n###### Function:\n`ServerHandler ctor`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rpc-command-method)", "type": "stat", "gridPos": { - "h": 8, + "h": 4, "w": 12, "x": 0, "y": 0 @@ -71,7 +71,24 @@ "unit": "reqps" }, "overrides": [] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 1 + }, + { + "title": "RPC Response", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 4 + }, + "collapsed": false, + "panels": [], + "id": 2 }, { "title": "RPC Response Time", @@ -80,8 +97,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 0 + "x": 0, + "y": 5 }, "options": { "tooltip": { @@ -108,11 +125,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 3 }, { "title": "RPC Response Size", @@ -121,8 +142,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 8 + "x": 12, + "y": 5 }, "options": { "tooltip": { @@ -149,11 +170,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 4 }, { "title": "RPC Response Time Distribution", @@ -162,8 +187,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 8 + "x": 0, + "y": 13 }, "options": { "tooltip": { @@ -204,103 +229,38 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 5 }, { - "title": "Pathfinding Fast Duration", - "description": "###### What this is:\n*The 95th-percentile time of the fast pathfinding search.*\n\n###### How it's computed:\n*95th-percentile of fast pathfinding durations over 5 minutes, per node.*\n\n###### Reading it:\n*Lower is better; fast mode trades accuracy for speed.*\n\n###### Healthy range:\n*Typically a few to tens of milliseconds; workload-dependent.*\n\n###### Watch for:\n*A rising trend, indicating heavy pathfinding demand or an expensive order book.*\n\n###### Keywords:\n- **Pathfinding (fast / full)** *(per node)* \u2014 searching for payment paths through intermediaries; fast mode trades accuracy for speed, full is exhaustive.\n- **Order book** *(network-wide)* \u2014 the ledger's list of standing offers to trade a currency pair on the decentralized exchange.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[PathRequestManager.h](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/PathRequestManager.h)\n\n###### Function:\n`ctor`\n\n###### References:\n[Pathfinding (fast / full)](https://xrpl.org/docs/concepts/tokens/fungible-tokens/paths) \u00b7 [Order book](https://xrpl.org/docs/concepts/tokens/decentralized-exchange) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#pathfinding-fast-full)", - "type": "timeseries", + "title": "Pathfinding", + "type": "row", "gridPos": { - "h": 8, - "w": 12, + "h": 1, + "w": 24, "x": 0, - "y": 16 + "y": 21 }, - "options": { - "tooltip": { - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(pathfind_fast_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"P95 Fast Pathfind\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" - } - ], - "fieldConfig": { - "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "unit": "ms", - "custom": { - "axisLabel": "Duration (ms)", - "spanNulls": 1800000, - "insertNulls": false, - "showPoints": "auto", - "pointSize": 3 - } - }, - "overrides": [] - } + "collapsed": false, + "panels": [], + "id": 6 }, { - "title": "Pathfinding Full Duration", - "description": "###### What this is:\n*The 95th-percentile time of the full, exhaustive pathfinding search.*\n\n###### How it's computed:\n*95th-percentile of full pathfinding durations over 5 minutes, per node.*\n\n###### Reading it:\n*Lower is better; full mode is much more expensive than fast mode.*\n\n###### Healthy range:\n*Tens to hundreds of milliseconds; workload-dependent.*\n\n###### Watch for:\n*Sustained high durations, consistent with pathfinding-heavy clients straining the node.*\n\n###### Keywords:\n- **Pathfinding (fast / full)** *(per node)* \u2014 searching for payment paths through intermediaries; fast mode trades accuracy for speed, full is exhaustive.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[PathRequestManager.h](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/PathRequestManager.h)\n\n###### Function:\n`ctor`\n\n###### References:\n[Pathfinding (fast / full)](https://xrpl.org/docs/concepts/tokens/fungible-tokens/paths) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#pathfinding-fast-full)", - "type": "timeseries", - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 16 - }, - "options": { - "tooltip": { - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(pathfind_full_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"P95 Full Pathfind\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" - } - ], - "fieldConfig": { - "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "unit": "ms", - "custom": { - "axisLabel": "Duration (ms)", - "spanNulls": 1800000, - "insertNulls": false, - "showPoints": "auto", - "pointSize": 3 - } - }, - "overrides": [] - } - }, - { - "title": "Resource Warnings Rate", + "title": "Resource Warnings Rate [$xrpl_network_type]", "description": "###### What this is:\n*How often the resource manager warns a peer or client for excessive usage, per second.*\n\n###### How it's computed:\n*Per-second rate of resource-warning events over 5 minutes, per node.*\n\n###### Reading it:\n*Green near zero, yellow above 0.1/sec, red above 1/sec.*\n\n###### Healthy range:\n*Near zero in normal operation.*\n\n###### Watch for:\n*A rising rate, consistent with aggressive clients that may need throttling.*\n\n###### Keywords:\n- **Resource drops / warnings** *(per node)* \u2014 the resource manager warning (then dropping/blocking) a peer or client for excessive usage.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[Logic.h](https://github.com/XRPLF/rippled/blob/develop/include/xrpl/resource/detail/Logic.h)\n\n###### Function:\n`Logic::Stats ctor`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#resource-drops-warnings)", "type": "stat", "gridPos": { - "h": 8, + "h": 4, "w": 12, "x": 0, - "y": 24 + "y": 22 }, "options": { "tooltip": { @@ -340,17 +300,21 @@ } }, "overrides": [] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 7 }, { - "title": "Resource Drops Rate", + "title": "Resource Drops Rate [$xrpl_network_type]", "description": "###### What this is:\n*How often the resource manager drops or blocks a peer or client for abuse, per second.*\n\n###### How it's computed:\n*Per-second rate of resource-drop events over 5 minutes, per node.*\n\n###### Reading it:\n*Green near zero, yellow above 0.01/sec, red above 0.1/sec.*\n\n###### Healthy range:\n*Zero when no abusive consumers are present.*\n\n###### Watch for:\n*Non-zero values, meaning the node is actively rejecting abusive connections.*\n\n###### Keywords:\n- **Resource drops / warnings** *(per node)* \u2014 the resource manager warning (then dropping/blocking) a peer or client for excessive usage.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[Logic.h](https://github.com/XRPLF/rippled/blob/develop/include/xrpl/resource/detail/Logic.h)\n\n###### Function:\n`Logic::Stats ctor`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#resource-drops-warnings)", "type": "stat", "gridPos": { - "h": 8, + "h": 4, "w": 12, "x": 12, - "y": 24 + "y": 22 }, "options": { "tooltip": { @@ -390,7 +354,114 @@ } }, "overrides": [] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 8 + }, + { + "title": "Pathfinding Fast Duration", + "description": "###### What this is:\n*The 95th-percentile time of the fast pathfinding search.*\n\n###### How it's computed:\n*95th-percentile of fast pathfinding durations over 5 minutes, per node.*\n\n###### Reading it:\n*Lower is better; fast mode trades accuracy for speed.*\n\n###### Healthy range:\n*Typically a few to tens of milliseconds; workload-dependent.*\n\n###### Watch for:\n*A rising trend, indicating heavy pathfinding demand or an expensive order book.*\n\n###### Keywords:\n- **Pathfinding (fast / full)** *(per node)* \u2014 searching for payment paths through intermediaries; fast mode trades accuracy for speed, full is exhaustive.\n- **Order book** *(network-wide)* \u2014 the ledger's list of standing offers to trade a currency pair on the decentralized exchange.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[PathRequestManager.h](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/PathRequestManager.h)\n\n###### Function:\n`ctor`\n\n###### References:\n[Pathfinding (fast / full)](https://xrpl.org/docs/concepts/tokens/fungible-tokens/paths) \u00b7 [Order book](https://xrpl.org/docs/concepts/tokens/decentralized-exchange) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#pathfinding-fast-full)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 26 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(pathfind_fast_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"P95 Fast Pathfind\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ms", + "custom": { + "axisLabel": "Duration (ms)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" + } + }, + "overrides": [] + }, + "id": 9 + }, + { + "title": "Pathfinding Full Duration", + "description": "###### What this is:\n*The 95th-percentile time of the full, exhaustive pathfinding search.*\n\n###### How it's computed:\n*95th-percentile of full pathfinding durations over 5 minutes, per node.*\n\n###### Reading it:\n*Lower is better; full mode is much more expensive than fast mode.*\n\n###### Healthy range:\n*Tens to hundreds of milliseconds; workload-dependent.*\n\n###### Watch for:\n*Sustained high durations, consistent with pathfinding-heavy clients straining the node.*\n\n###### Keywords:\n- **Pathfinding (fast / full)** *(per node)* \u2014 searching for payment paths through intermediaries; fast mode trades accuracy for speed, full is exhaustive.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[PathRequestManager.h](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/PathRequestManager.h)\n\n###### Function:\n`ctor`\n\n###### References:\n[Pathfinding (fast / full)](https://xrpl.org/docs/concepts/tokens/fungible-tokens/paths) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#pathfinding-fast-full)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 26 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(pathfind_full_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"P95 Full Pathfind\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ms", + "custom": { + "axisLabel": "Duration (ms)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" + } + }, + "overrides": [] + }, + "id": 10 + }, + { + "title": "gRPC (Spans)", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 34 + }, + "collapsed": false, + "panels": [], + "id": 11 }, { "title": "gRPC Request Rate by Method (Spans)", @@ -400,7 +471,7 @@ "h": 8, "w": 12, "x": 0, - "y": 32 + "y": 35 }, "options": { "tooltip": { @@ -427,11 +498,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 12 }, { "title": "gRPC Latency P95 by Method (Spans)", @@ -441,7 +516,7 @@ "h": 8, "w": 12, "x": 12, - "y": 32 + "y": 35 }, "options": { "tooltip": { @@ -468,11 +543,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 13 }, { "title": "gRPC Error Rate by Status (Spans)", @@ -482,7 +561,7 @@ "h": 8, "w": 12, "x": 0, - "y": 40 + "y": 43 }, "options": { "tooltip": { @@ -509,11 +588,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 14 }, { "title": "Pathfinding Compute Duration (Spans)", @@ -523,7 +606,7 @@ "h": 8, "w": 12, "x": 12, - "y": 40 + "y": 43 }, "options": { "tooltip": { @@ -550,11 +633,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 15 }, { "title": "Pathfinding Request & Discovery Rate (Spans)", @@ -562,9 +649,9 @@ "type": "timeseries", "gridPos": { "h": 8, - "w": 24, + "w": 12, "x": 0, - "y": 48 + "y": 51 }, "options": { "tooltip": { @@ -598,11 +685,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 16 } ], "schemaVersion": 39, diff --git a/docker/telemetry/grafana/dashboards/rpc-performance.json b/docker/telemetry/grafana/dashboards/rpc-performance.json index a378b29e40..3fed35f9dc 100644 --- a/docker/telemetry/grafana/dashboards/rpc-performance.json +++ b/docker/telemetry/grafana/dashboards/rpc-performance.json @@ -39,9 +39,9 @@ "links": [], "panels": [ { - "id": 11, + "id": 1, "type": "row", - "title": "Aggregate RPC (all commands)", + "title": "Aggregate RPC (All Commands)", "collapsed": false, "gridPos": { "h": 1, @@ -52,98 +52,14 @@ "panels": [] }, { - "title": "RPC Request Rate by Command", - "description": "###### What this is:\n*How many times each RPC command runs per second.*\n\n###### How it's computed:\n*Per-second rate of command executions over 5 minutes, grouped by command name and node.*\n\n###### Reading it:\n*Compare bars/lines across commands to see which endpoints drive load.*\n\n###### Healthy range:\n*Workload-dependent; dominated by whichever commands clients call most.*\n\n###### Watch for:\n*A single command spiking far above its norm, consistent with a client hammering one endpoint.*\n\n###### Keywords:\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[RPCHandler.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/RPCHandler.cpp)\n\n###### Function:\n`callMethod`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rpc-command-method)", - "type": "timeseries", - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 1 - }, - "options": { - "tooltip": { - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(sum by (command, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", command=~\"$command\", span_name=~\"rpc.command.*\"}[$__rate_interval])), \"series\", \"$1\", \"command\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" - } - ], - "fieldConfig": { - "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "unit": "reqps", - "custom": { - "axisLabel": "Requests / Sec", - "spanNulls": 1800000, - "insertNulls": false, - "showPoints": "auto", - "pointSize": 3 - } - }, - "overrides": [] - }, - "id": 1 - }, - { - "title": "RPC Latency P95 by Command", - "description": "###### What this is:\n*The 95th-percentile response time for each RPC command.*\n\n###### How it's computed:\n*95th-percentile of per-command execution durations over 5 minutes, grouped by command and node.*\n\n###### Reading it:\n*Lower is better; watch the slowest commands.*\n\n###### Healthy range:\n*Fast lookups a few ms; heavy queries tens to hundreds of ms. Workload-dependent.*\n\n###### Watch for:\n*A command whose latency climbs steadily, pointing to an expensive or degrading query path.*\n\n###### Keywords:\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[RPCHandler.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/RPCHandler.cpp)\n\n###### Function:\n`callMethod`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rpc-command-method)", - "type": "timeseries", - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 1 - }, - "options": { - "tooltip": { - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, command, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", command=~\"$command\", span_name=~\"rpc.command.*\"}[5m]))), \"series\", \"P95 $1\", \"command\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" - } - ], - "fieldConfig": { - "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "unit": "ms", - "custom": { - "axisLabel": "Latency (ms)", - "spanNulls": 1800000, - "insertNulls": false, - "showPoints": "auto", - "pointSize": 3 - } - }, - "overrides": [] - }, - "id": 2 - }, - { - "title": "RPC Error Rate", + "title": "RPC Error Rate [$xrpl_network_type]", "description": "###### What this is:\n*The percentage of each command's calls that finished with an error.*\n\n###### How it's computed:\n*Error calls divided by total calls per command over 5 minutes, expressed as a percent.*\n\n###### Reading it:\n*Green under 1%, yellow 1-5%, red above 5%.*\n\n###### Healthy range:\n*Below 1% per command in normal operation.*\n\n###### Watch for:\n*A command sitting red, indicating malformed input, overload, or a broken handler.*\n\n###### Keywords:\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[RPCHandler.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/RPCHandler.cpp)\n\n###### Function:\n`callMethod`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rpc-command-method)", "type": "bargauge", "gridPos": { - "h": 8, + "h": 4, "w": 12, "x": 0, - "y": 9 + "y": 1 }, "options": { "tooltip": { @@ -184,7 +100,176 @@ }, "overrides": [] }, - "id": 3 + "id": 2, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 + }, + { + "title": "Top Commands by Volume [$xrpl_network_type]", + "description": "###### What this is:\n*The ten most-called RPC commands over the recent window.*\n\n###### How it's computed:\n*Total invocation counts per command over the last 5 minutes, ranked and capped at ten.*\n\n###### Reading it:\n*The top bars are the hottest endpoints driving node load.*\n\n###### Healthy range:\n*Workload-dependent; typically led by lightweight status queries.*\n\n###### Watch for:\n*An unexpected command dominating the ranking, consistent with automated abuse.*\n\n###### Keywords:\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[RPCHandler.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/RPCHandler.cpp)\n\n###### Function:\n`callMethod`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rpc-command-method)", + "type": "bargauge", + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 1 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(topk(10, sum by (command, service_instance_id, xrpl_branch, xrpl_node_role) (increase(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", command=~\"$command\", span_name=~\"rpc.command.*\"}[$__rate_interval]))), \"series\", \"$1\", \"command\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "short" + }, + "overrides": [] + }, + "id": 3, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 + }, + { + "title": "WebSocket Message Rate [$xrpl_network_type]", + "description": "###### What this is:\n*How many RPC messages arrive over WebSocket connections per second.*\n\n###### How it's computed:\n*Per-second rate of processed WebSocket RPC messages over 5 minutes, per node.*\n\n###### Reading it:\n*Non-zero only when clients use WebSocket; zero is normal for HTTP-only nodes.*\n\n###### Healthy range:\n*Workload-dependent; zero if no WebSocket clients.*\n\n###### Watch for:\n*A sudden surge from a single source, consistent with a chatty or abusive subscription client.*\n\n###### Keywords:\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n- **WebSocket** *(per node)* \u2014 a persistent-connection API transport used by subscription clients; zero on HTTP-only nodes.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[ServerHandler.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/ServerHandler.cpp)\n\n###### Function:\n`ServerHandler::processSession`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [WebSocket](https://xrpl.org/docs/references/http-websocket-apis/api-conventions) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rpc-command-method)", + "type": "stat", + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 5 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", command=~\"$command\", span_name=\"rpc.ws_message\"}[$__rate_interval])), \"series\", \"WS Messages / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "suffix: messages/s" + }, + "overrides": [] + }, + "id": 4, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 + }, + { + "title": "RPC Request Rate by Command", + "description": "###### What this is:\n*How many times each RPC command runs per second.*\n\n###### How it's computed:\n*Per-second rate of command executions over 5 minutes, grouped by command name and node.*\n\n###### Reading it:\n*Compare bars/lines across commands to see which endpoints drive load.*\n\n###### Healthy range:\n*Workload-dependent; dominated by whichever commands clients call most.*\n\n###### Watch for:\n*A single command spiking far above its norm, consistent with a client hammering one endpoint.*\n\n###### Keywords:\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[RPCHandler.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/RPCHandler.cpp)\n\n###### Function:\n`callMethod`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rpc-command-method)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (command, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", command=~\"$command\", span_name=~\"rpc.command.*\"}[$__rate_interval])), \"series\", \"$1\", \"command\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "reqps", + "custom": { + "axisLabel": "Requests / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" + } + }, + "overrides": [] + }, + "id": 5 + }, + { + "title": "RPC Latency P95 by Command", + "description": "###### What this is:\n*The 95th-percentile response time for each RPC command.*\n\n###### How it's computed:\n*95th-percentile of per-command execution durations over 5 minutes, grouped by command and node.*\n\n###### Reading it:\n*Lower is better; watch the slowest commands.*\n\n###### Healthy range:\n*Fast lookups a few ms; heavy queries tens to hundreds of ms. Workload-dependent.*\n\n###### Watch for:\n*A command whose latency climbs steadily, pointing to an expensive or degrading query path.*\n\n###### Keywords:\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[RPCHandler.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/RPCHandler.cpp)\n\n###### Function:\n`callMethod`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rpc-command-method)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, command, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", command=~\"$command\", span_name=~\"rpc.command.*\"}[5m]))), \"series\", \"P95 $1\", \"command\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ms", + "custom": { + "axisLabel": "Latency (ms)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" + } + }, + "overrides": [] + }, + "id": 6 }, { "title": "RPC Latency Heatmap", @@ -193,8 +278,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 9 + "x": 0, + "y": 17 }, "options": { "tooltip": { @@ -223,7 +308,7 @@ "unit": "ms" } }, - "id": 4 + "id": 7 }, { "title": "Overall RPC Throughput", @@ -232,7 +317,7 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, + "x": 12, "y": 17 }, "options": { @@ -267,12 +352,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] }, - "id": 5 + "id": 8 }, { "title": "RPC Success vs Error", @@ -281,8 +369,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 17 + "x": 0, + "y": 25 }, "options": { "tooltip": { @@ -316,82 +404,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] }, - "id": 6 - }, - { - "title": "Top Commands by Volume", - "description": "###### What this is:\n*The ten most-called RPC commands over the recent window.*\n\n###### How it's computed:\n*Total invocation counts per command over the last 5 minutes, ranked and capped at ten.*\n\n###### Reading it:\n*The top bars are the hottest endpoints driving node load.*\n\n###### Healthy range:\n*Workload-dependent; typically led by lightweight status queries.*\n\n###### Watch for:\n*An unexpected command dominating the ranking, consistent with automated abuse.*\n\n###### Keywords:\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[RPCHandler.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/RPCHandler.cpp)\n\n###### Function:\n`callMethod`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rpc-command-method)", - "type": "bargauge", - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 25 - }, - "options": { - "tooltip": { - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(topk(10, sum by (command, service_instance_id, xrpl_branch, xrpl_node_role) (increase(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", command=~\"$command\", span_name=~\"rpc.command.*\"}[$__rate_interval]))), \"series\", \"$1\", \"command\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" - } - ], - "fieldConfig": { - "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "unit": "short" - }, - "overrides": [] - }, - "id": 7 - }, - { - "title": "WebSocket Message Rate", - "description": "###### What this is:\n*How many RPC messages arrive over WebSocket connections per second.*\n\n###### How it's computed:\n*Per-second rate of processed WebSocket RPC messages over 5 minutes, per node.*\n\n###### Reading it:\n*Non-zero only when clients use WebSocket; zero is normal for HTTP-only nodes.*\n\n###### Healthy range:\n*Workload-dependent; zero if no WebSocket clients.*\n\n###### Watch for:\n*A sudden surge from a single source, consistent with a chatty or abusive subscription client.*\n\n###### Keywords:\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n- **WebSocket** *(per node)* \u2014 a persistent-connection API transport used by subscription clients; zero on HTTP-only nodes.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[ServerHandler.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/ServerHandler.cpp)\n\n###### Function:\n`ServerHandler::processSession`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [WebSocket](https://xrpl.org/docs/references/http-websocket-apis/api-conventions) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rpc-command-method)", - "type": "stat", - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 25 - }, - "options": { - "tooltip": { - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", command=~\"$command\", span_name=\"rpc.ws_message\"}[$__rate_interval])), \"series\", \"WS Messages / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" - } - ], - "fieldConfig": { - "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "unit": "suffix: messages/s" - }, - "overrides": [] - }, - "id": 8 + "id": 9 }, { "title": "RPC Resource Cost by Command", @@ -399,7 +420,7 @@ "type": "timeseries", "gridPos": { "h": 8, - "w": 12, + "w": 24, "x": 0, "y": 33 }, @@ -433,12 +454,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] }, - "id": 9 + "id": 10 }, { "title": "Batch vs Single RPC Requests", @@ -447,8 +471,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 33 + "x": 0, + "y": 41 }, "options": { "tooltip": { @@ -482,12 +506,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] }, - "id": 10 + "id": 11 }, { "id": 12, @@ -498,366 +525,19 @@ "h": 1, "w": 24, "x": 0, - "y": 41 + "y": 49 }, "panels": [] }, { - "title": "RPC Call Rate (All Methods)", - "description": "###### What this is:\n*Overall rate of RPC method calls that started, finished, and errored, across all methods.*\n\n###### How it's computed:\n*Per-second rate of each counter over a 5-minute window, summed per node.*\n\n###### Reading it:\n*Started should closely track finished; errored should be a small fraction.*\n\n###### Healthy range:\n*Workload-dependent; started \u2248 finished, errored near zero.*\n\n###### Watch for:\n*A growing gap between started and finished (calls hanging), or an errored line that rises with load.*\n\n###### Keywords:\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordRpcStarted / recordRpcFinished / recordRpcErrored`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rpc-command-method)", - "type": "timeseries", - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 42 - }, - "options": { - "tooltip": { - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(rpc_method_started_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", method=~\"$method\"}[$__rate_interval])), \"series\", \"Started/s\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(rpc_method_finished_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", method=~\"$method\"}[$__rate_interval])), \"series\", \"Finished/s\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(rpc_method_errored_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", method=~\"$method\"}[$__rate_interval])), \"series\", \"Errored/s\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" - } - ], - "fieldConfig": { - "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "unit": "suffix: calls/s", - "custom": { - "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, - "axisLabel": "Calls / Sec", - "spanNulls": 1800000, - "insertNulls": false, - "showPoints": "auto", - "pointSize": 3 - }, - "color": { - "mode": "palette-classic" - } - }, - "overrides": [] - }, - "id": 13 - }, - { - "title": "Per-Method Call Rate (Top 10)", - "description": "###### What this is:\n*The ten busiest RPC methods by call rate.*\n\n###### How it's computed:\n*Per-second start rate over 5 minutes, per method, showing the top ten.*\n\n###### Reading it:\n*Identifies which methods dominate load; the mix shifts with client behaviour.*\n\n###### Healthy range:\n*Workload-dependent.*\n\n###### Watch for:\n*A single method suddenly dominating, which can signal a runaway client or abusive query pattern.*\n\n###### Keywords:\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordRpcStarted`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rpc-command-method)", - "type": "timeseries", - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 50 - }, - "options": { - "tooltip": { - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "right", - "calcs": ["mean", "max"] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(topk(10, rate(rpc_method_started_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", method=~\"$method\"}[$__rate_interval])), \"series\", \"$1\", \"method\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" - } - ], - "fieldConfig": { - "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "unit": "suffix: calls/s", - "custom": { - "drawStyle": "line", - "lineWidth": 1, - "fillOpacity": 5, - "axisLabel": "Calls / Sec", - "spanNulls": 1800000, - "insertNulls": false, - "showPoints": "auto", - "pointSize": 3 - }, - "color": { - "mode": "palette-classic" - } - }, - "overrides": [] - }, - "id": 14 - }, - { - "title": "Per-Method Error Rate (Top 10)", - "description": "###### What this is:\n*The ten RPC methods producing the most errors.*\n\n###### How it's computed:\n*Per-second error rate over 5 minutes, per method, top ten.*\n\n###### Reading it:\n*Normally near zero; persistent errors point to a specific failing method.*\n\n###### Healthy range:\n*Near zero for well-behaved traffic.*\n\n###### Watch for:\n*Sustained errors concentrated on one method \u2014 a broken client, a bad input, or probing.*\n\n###### Keywords:\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordRpcErrored`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rpc-command-method)", - "type": "timeseries", - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 58 - }, - "options": { - "tooltip": { - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "right", - "calcs": ["mean", "max"] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(topk(10, rate(rpc_method_errored_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", method=~\"$method\"}[$__rate_interval])), \"series\", \"$1\", \"method\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" - } - ], - "fieldConfig": { - "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "unit": "suffix: errors/s", - "custom": { - "drawStyle": "line", - "lineWidth": 1, - "fillOpacity": 5, - "axisLabel": "Errors / Sec", - "spanNulls": 1800000, - "insertNulls": false, - "showPoints": "auto", - "pointSize": 3 - }, - "color": { - "mode": "palette-classic" - } - }, - "overrides": [] - }, - "id": 15 - }, - { - "title": "RPC Latency - All Methods", - "description": "###### What this is:\n*Aggregate RPC handler latency across all methods (p75 and p99).*\n\n###### How it's computed:\n*Percentiles of the method-duration histogram over a 5-minute window.*\n\n###### Reading it:\n*p75 reflects typical responsiveness; p99 captures the slow tail.*\n\n###### Healthy range:\n*Sub-millisecond to low-millisecond for light commands; heavier commands run longer.*\n\n###### Watch for:\n*A rising p99 while p75 stays flat \u2014 a subset of calls degrading, often from expensive queries.*\n\n###### Keywords:\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordRpcFinished`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rpc-command-method)", - "type": "timeseries", - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 66 - }, - "options": { - "tooltip": { - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(histogram_quantile(0.75, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(rpc_method_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", method=~\"$method\"}[5m]))), \"series\", \"p75\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(histogram_quantile(0.99, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(rpc_method_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", method=~\"$method\"}[5m]))), \"series\", \"p99\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" - } - ], - "fieldConfig": { - "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "unit": "\u00b5s", - "custom": { - "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 5, - "axisLabel": "Duration (\u03bcs)", - "spanNulls": 1800000, - "insertNulls": false, - "showPoints": "auto", - "pointSize": 3 - }, - "color": { - "mode": "palette-classic" - } - }, - "overrides": [] - }, - "id": 16 - }, - { - "title": "Per-Method Latency (p99, Top 10 Slowest)", - "description": "###### What this is:\n*The ten slowest RPC methods by tail latency.*\n\n###### How it's computed:\n*p99 of each method's duration histogram over 5 minutes, top ten.*\n\n###### Reading it:\n*Surfaces which specific methods are expensive.*\n\n###### Healthy range:\n*Method-dependent; ledger/account queries are heavier than status calls.*\n\n###### Watch for:\n*A method whose p99 climbs over time, or an unexpectedly cheap method appearing here.*\n\n###### Keywords:\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordRpcFinished`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rpc-command-method)", - "type": "timeseries", - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 74 - }, - "options": { - "tooltip": { - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "right", - "calcs": ["mean", "max"] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(topk(10, histogram_quantile(0.99, sum by (le, method, service_instance_id, xrpl_branch, xrpl_node_role) (rate(rpc_method_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", method=~\"$method\"}[5m])))), \"series\", \"$1\", \"method\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" - } - ], - "fieldConfig": { - "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "unit": "\u00b5s", - "custom": { - "drawStyle": "line", - "lineWidth": 1, - "fillOpacity": 5, - "axisLabel": "Duration (\u03bcs)", - "spanNulls": 1800000, - "insertNulls": false, - "showPoints": "auto", - "pointSize": 3 - }, - "color": { - "mode": "palette-classic" - } - }, - "overrides": [] - }, - "id": 17 - }, - { - "title": "RPC Error Ratio by Method", - "description": "###### What this is:\n*The methods with the highest error rates, for spotting failure hotspots.*\n\n###### How it's computed:\n*Per-second error rate over 5 minutes, per method, top ten.*\n\n###### Reading it:\n*Highlights where failures concentrate.*\n\n###### Healthy range:\n*Near zero.*\n\n###### Watch for:\n*One method with a persistently high error rate \u2014 malformed requests or targeted probing.*\n\n###### Keywords:\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordRpcErrored`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rpc-command-method)", - "type": "timeseries", - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 82 - }, - "options": { - "tooltip": { - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "right", - "calcs": ["mean", "max"] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(topk(10, rate(rpc_method_errored_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", method=~\"$method\"}[$__rate_interval]) / (rate(rpc_method_started_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", method=~\"$method\"}[$__rate_interval]) > 0)), \"series\", \"$1\", \"method\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" - } - ], - "fieldConfig": { - "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "unit": "percentunit", - "min": 0, - "max": 1, - "custom": { - "drawStyle": "line", - "lineWidth": 1, - "fillOpacity": 5, - "axisLabel": "Error Ratio", - "spanNulls": 1800000, - "insertNulls": false, - "showPoints": "auto", - "pointSize": 3 - }, - "color": { - "mode": "palette-classic" - }, - "thresholds": { - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 0.05 - }, - { - "color": "red", - "value": 0.25 - } - ] - } - }, - "overrides": [] - }, - "id": 18 - }, - { - "title": "Current RPC Latency (p99 Gauge)", + "title": "Current RPC Latency (p99 Gauge) [$xrpl_network_type]", "description": "###### What this is:\n*Current tail latency (p99) of RPC handling across all methods, as a live gauge.*\n\n###### How it's computed:\n*p99 of the method-duration histogram over the recent window.*\n\n###### Reading it:\n*A single at-a-glance number for current RPC responsiveness.*\n\n###### Healthy range:\n*Low-millisecond under normal load.*\n\n###### Watch for:\n*Sustained elevation, indicating the node is under query pressure.*\n\n###### Keywords:\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordRpcFinished`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rpc-command-method)", "type": "gauge", "gridPos": { - "h": 8, + "h": 4, "w": 12, "x": 0, - "y": 90 + "y": 50 }, "options": { "reduceOptions": { @@ -904,6 +584,362 @@ }, "overrides": [] }, + "id": 13, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 + }, + { + "title": "RPC Call Rate (All Methods)", + "description": "###### What this is:\n*Overall rate of RPC method calls that started, finished, and errored, across all methods.*\n\n###### How it's computed:\n*Per-second rate of each counter over a 5-minute window, summed per node.*\n\n###### Reading it:\n*Started should closely track finished; errored should be a small fraction.*\n\n###### Healthy range:\n*Workload-dependent; started \u2248 finished, errored near zero.*\n\n###### Watch for:\n*A growing gap between started and finished (calls hanging), or an errored line that rises with load.*\n\n###### Keywords:\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordRpcStarted / recordRpcFinished / recordRpcErrored`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rpc-command-method)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 54 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(rpc_method_started_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", method=~\"$method\"}[$__rate_interval])), \"series\", \"Started/s\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(rpc_method_finished_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", method=~\"$method\"}[$__rate_interval])), \"series\", \"Finished/s\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(rpc_method_errored_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", method=~\"$method\"}[$__rate_interval])), \"series\", \"Errored/s\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "suffix: calls/s", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 0, + "axisLabel": "Calls / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 5, + "gradientMode": "none" + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "id": 14 + }, + { + "title": "Per-Method Call Rate (Top 10)", + "description": "###### What this is:\n*The ten busiest RPC methods by call rate.*\n\n###### How it's computed:\n*Per-second start rate over 5 minutes, per method, showing the top ten.*\n\n###### Reading it:\n*Identifies which methods dominate load; the mix shifts with client behaviour.*\n\n###### Healthy range:\n*Workload-dependent.*\n\n###### Watch for:\n*A single method suddenly dominating, which can signal a runaway client or abusive query pattern.*\n\n###### Keywords:\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordRpcStarted`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rpc-command-method)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 62 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": ["mean", "max"] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(topk(10, rate(rpc_method_started_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", method=~\"$method\"}[$__rate_interval])), \"series\", \"$1\", \"method\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "suffix: calls/s", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 0, + "axisLabel": "Calls / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 5, + "gradientMode": "none" + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "id": 15 + }, + { + "title": "Per-Method Error Rate (Top 10)", + "description": "###### What this is:\n*The ten RPC methods producing the most errors.*\n\n###### How it's computed:\n*Per-second error rate over 5 minutes, per method, top ten.*\n\n###### Reading it:\n*Normally near zero; persistent errors point to a specific failing method.*\n\n###### Healthy range:\n*Near zero for well-behaved traffic.*\n\n###### Watch for:\n*Sustained errors concentrated on one method \u2014 a broken client, a bad input, or probing.*\n\n###### Keywords:\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordRpcErrored`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rpc-command-method)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 70 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": ["mean", "max"] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(topk(10, rate(rpc_method_errored_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", method=~\"$method\"}[$__rate_interval])), \"series\", \"$1\", \"method\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "suffix: errors/s", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 0, + "axisLabel": "Errors / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 5, + "gradientMode": "none" + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "id": 16 + }, + { + "title": "RPC Latency - All Methods", + "description": "###### What this is:\n*Aggregate RPC handler latency across all methods (p75 and p99).*\n\n###### How it's computed:\n*Percentiles of the method-duration histogram over a 5-minute window.*\n\n###### Reading it:\n*p75 reflects typical responsiveness; p99 captures the slow tail.*\n\n###### Healthy range:\n*Sub-millisecond to low-millisecond for light commands; heavier commands run longer.*\n\n###### Watch for:\n*A rising p99 while p75 stays flat \u2014 a subset of calls degrading, often from expensive queries.*\n\n###### Keywords:\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordRpcFinished`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rpc-command-method)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 78 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.75, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(rpc_method_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", method=~\"$method\"}[5m]))), \"series\", \"p75\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.99, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(rpc_method_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", method=~\"$method\"}[5m]))), \"series\", \"p99\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "\u00b5s", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 0, + "axisLabel": "Duration (\u03bcs)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 5, + "gradientMode": "none" + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "id": 17 + }, + { + "title": "Per-Method Latency (p99, Top 10 Slowest)", + "description": "###### What this is:\n*The ten slowest RPC methods by tail latency.*\n\n###### How it's computed:\n*p99 of each method's duration histogram over 5 minutes, top ten.*\n\n###### Reading it:\n*Surfaces which specific methods are expensive.*\n\n###### Healthy range:\n*Method-dependent; ledger/account queries are heavier than status calls.*\n\n###### Watch for:\n*A method whose p99 climbs over time, or an unexpectedly cheap method appearing here.*\n\n###### Keywords:\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordRpcFinished`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rpc-command-method)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 86 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": ["mean", "max"] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(topk(10, histogram_quantile(0.99, sum by (le, method, service_instance_id, xrpl_branch, xrpl_node_role) (rate(rpc_method_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", method=~\"$method\"}[5m])))), \"series\", \"$1\", \"method\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "\u00b5s", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 0, + "axisLabel": "Duration (\u03bcs)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 5, + "gradientMode": "none" + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "id": 18 + }, + { + "title": "RPC Error Ratio by Method", + "description": "###### What this is:\n*The methods with the highest error rates, for spotting failure hotspots.*\n\n###### How it's computed:\n*Per-second error rate over 5 minutes, per method, top ten.*\n\n###### Reading it:\n*Highlights where failures concentrate.*\n\n###### Healthy range:\n*Near zero.*\n\n###### Watch for:\n*One method with a persistently high error rate \u2014 malformed requests or targeted probing.*\n\n###### Keywords:\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordRpcErrored`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rpc-command-method)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 94 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": ["mean", "max"] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(topk(10, rate(rpc_method_errored_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", method=~\"$method\"}[$__rate_interval]) / (rate(rpc_method_started_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", method=~\"$method\"}[$__rate_interval]) > 0)), \"series\", \"$1\", \"method\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "percentunit", + "min": 0, + "max": 1, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 0, + "axisLabel": "Error Ratio", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 5, + "gradientMode": "none" + }, + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.05 + }, + { + "color": "red", + "value": 0.25 + } + ] + } + }, + "overrides": [] + }, "id": 19 } ], diff --git a/docker/telemetry/grafana/dashboards/transaction-overview.json b/docker/telemetry/grafana/dashboards/transaction-overview.json index f71b2ce73e..bb697f3336 100644 --- a/docker/telemetry/grafana/dashboards/transaction-overview.json +++ b/docker/telemetry/grafana/dashboards/transaction-overview.json @@ -39,12 +39,12 @@ "links": [], "panels": [ { - "title": "Transaction Apply Failed Rate", + "title": "Transaction Apply Failed Rate [$xrpl_network_type]", "description": "###### What this is:\n*Transactions that fail during the apply (transactor) stage, per second \u2014 i.e. the transactor ran but the result was not tesSUCCESS.*\n\n###### How it's computed:\n*Per-second rate of tx.transactor spans at the apply stage whose ter_result is not tesSUCCESS, over the dashboard rate interval, per node.*\n\n###### Reading it:\n*A steady low background is normal (tefPAST_SEQ, tecUNFUNDED, etc.); the transactor legitimately rejects many transactions.*\n\n###### Healthy range:\n*A low, stable rate; workload-dependent.*\n\n###### Watch for:\n*A sharp rise above baseline, which points to a submission pattern that is systematically failing at apply.*\n\n###### Keywords:\n- **Transactor** *(per node)* \u2014 the rippled component that executes one transaction's type-specific logic against the ledger.\n- **Transaction result codes** *(network-wide)* \u2014 the outcome code a transaction returns \u2014 tesSUCCESS, or a tec/tef/tem/ter/tel class code on failure.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[Transactor.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/tx/detail/Transactor.cpp)\n\n###### Function:\n`Transactor::operator()`\n\n###### References:\n[Transactor](https://xrpl.org/docs/references/protocol/transactions/transaction-results) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#transactor)", "type": "stat", "gridPos": { "h": 4, - "w": 6, + "w": 12, "x": 0, "y": 0 }, @@ -87,7 +87,10 @@ }, "overrides": [] }, - "id": 1 + "id": 1, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 }, { "title": "Transaction Processing Latency by Type", @@ -129,13 +132,29 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] }, "id": 2 }, + { + "title": "Throughput & Results", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 12 + }, + "collapsed": false, + "panels": [], + "id": 3 + }, { "title": "Transaction Rate by Type", "description": "###### What this is:\n*How many transactions of each type enter processing per second.*\n\n###### How it's computed:\n*Per-second processing rate over 5 minutes, grouped by transaction type and node.*\n\n###### Reading it:\n*Shows the transaction mix; payments usually lead.*\n\n###### Healthy range:\n*Workload-dependent; mix reflects network activity.*\n\n###### Watch for:\n*A single type spiking far above baseline, consistent with a spam campaign of that type.*\n\n###### Keywords:\n- **Transaction type** *(network-wide)* \u2014 the kind of transaction (Payment, OfferCreate, TrustSet, AMM*, NFToken*, etc.), used as a breakdown dimension.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`NetworkOPsImp::processTransaction`\n\n###### References:\n[Transaction type](https://xrpl.org/docs/references/protocol/transactions/types) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#transaction-type)", @@ -144,7 +163,7 @@ "h": 8, "w": 24, "x": 0, - "y": 12 + "y": 13 }, "options": { "tooltip": { @@ -176,12 +195,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] }, - "id": 3 + "id": 4 }, { "title": "Transaction Results by Type", @@ -191,7 +213,7 @@ "h": 8, "w": 24, "x": 0, - "y": 20 + "y": 21 }, "options": { "tooltip": { @@ -222,13 +244,16 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" }, "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}" }, "overrides": [] }, - "id": 4 + "id": 5 }, { "title": "Transaction Receive vs Suppressed", @@ -238,7 +263,7 @@ "h": 8, "w": 12, "x": 0, - "y": 28 + "y": 29 }, "options": { "tooltip": { @@ -265,12 +290,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] }, - "id": 5 + "id": 6 }, { "title": "Transaction Processing Rate", @@ -280,7 +308,7 @@ "h": 8, "w": 12, "x": 12, - "y": 28 + "y": 29 }, "options": { "tooltip": { @@ -314,12 +342,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] }, - "id": 6 + "id": 7 }, { "title": "Transaction Path Distribution", @@ -329,7 +360,7 @@ "h": 8, "w": 12, "x": 0, - "y": 36 + "y": 37 }, "fieldConfig": { "defaults": { @@ -354,7 +385,7 @@ "expr": "label_replace(label_join(label_replace(sum by (local, service_instance_id, xrpl_branch, xrpl_node_role) (increase(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", local=~\"$tx_origin\", span_name=\"tx.process\", tx_type=~\"$tx_type\"}[$__rate_interval])), \"series\", \"Local $1\", \"local\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" } ], - "id": 7 + "id": 8 }, { "title": "Transaction Processing Duration Heatmap", @@ -364,7 +395,7 @@ "h": 8, "w": 12, "x": 12, - "y": 36 + "y": 37 }, "options": { "tooltip": { @@ -393,7 +424,7 @@ "unit": "ms" } }, - "id": 8 + "id": 9 }, { "title": "Transactor Duration by Type (p95)", @@ -403,7 +434,7 @@ "h": 8, "w": 24, "x": 0, - "y": 44 + "y": 45 }, "options": { "tooltip": { @@ -435,7 +466,10 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] @@ -443,14 +477,14 @@ "id": 10 }, { - "title": "TxQ Accept: Applied Ratio per Node (State Timeline)", + "title": "TxQ Accept: Applied Ratio per Node (State Timeline) [$xrpl_network_type]", "description": "###### What this is:\n*The applied fraction of TxQ accepts per node over time: the share of queued transactions that were included in a ledger versus removed on failure.*\n\n###### How it's computed:\n*Per node, applied accepts divided by applied-plus-failed accepts over the window.*\n\n###### Reading it:\n*Green (>=90% applied) is a healthy drain; yellow is degraded; red means accepts are mostly failing.*\n\n###### Healthy range:\n*At or near 100% applied when the queue is draining healthily; workload-dependent.*\n\n###### Watch for:\n*A node dropping into yellow or red, which signals queue pressure, under-bidding, or fee escalation on that node.*\n\n###### Keywords:\n- **Transaction queue (TxQ)** *(per node)* \u2014 holds transactions that meet local cost but not the open-ledger cost, to include in a later ledger.\n- **Queue accept (drain)** *(per node)* \u2014 moving queued transactions into a newly closed ledger; the applied share shows drain health.\n- **Fee escalation** *(per node)* \u2014 the exponential rise in the open-ledger cost once the ledger exceeds its soft transaction target.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[TxQ.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/detail/TxQ.cpp)\n\n###### Function:\n`TxQ::accept`\n\n###### References:\n[Transaction queue (TxQ)](https://xrpl.org/docs/concepts/transactions/transaction-queue) \u00b7 [Fee escalation](https://xrpl.org/docs/concepts/transactions/transaction-cost#open-ledger-cost) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#transaction-queue-txq)", "type": "state-timeline", "gridPos": { "h": 8, - "w": 24, + "w": 12, "x": 0, - "y": 52 + "y": 53 }, "options": { "mergeValues": true, @@ -512,7 +546,73 @@ }, "overrides": [] }, - "id": 11 + "id": 11, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 + }, + { + "title": "Tx Apply Pipeline Rate by Stage", + "description": "**What:** Throughput of each apply-pipeline stage (preflight, preclaim, apply), showing where transactions drop out.\n**How it's computed:** Per-second rate per stage over a 5-minute window, per node.\n**Reading it:** A decline from earlier to later stages shows where transactions are filtered.\n**Healthy range:** Workload-dependent; later stages sit at or below earlier ones.\n**Watch for:** A large early-stage drop means many transactions fail basic checks, consistent with malformed floods.\n**Source:** src/libxrpl/tx/Transactor.cpp:Transactor::operator()", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 61 + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc", + "maxHeight": 600 + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": ["mean", "max"] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "sum by (stage, service_instance_id) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", span_name=~\"tx.preflight|tx.preclaim|tx.transactor\", stage=~\"$stage\"}[$__rate_interval]))", + "interval": "15s", + "legendFormat": "{{stage}} [{{service_instance_id}}]" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "axisLabel": "Spans / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" + } + }, + "overrides": [] + }, + "id": 12 + }, + { + "title": "Apply Pipeline", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 69 + }, + "collapsed": false, + "panels": [], + "id": 13 }, { "title": "Tx Apply Pipeline Latency by Stage (p95)", @@ -522,7 +622,7 @@ "h": 8, "w": 24, "x": 0, - "y": 60 + "y": 70 }, "options": { "tooltip": { @@ -554,12 +654,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] }, - "id": 12 + "id": 14 }, { "title": "Tx Apply Pipeline Failure Rate by Stage", @@ -569,7 +672,7 @@ "h": 8, "w": 24, "x": 0, - "y": 68 + "y": 78 }, "options": { "tooltip": { @@ -601,12 +704,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] }, - "id": 13 + "id": 15 }, { "title": "Tx Apply Pipeline Latency by Type and Stage (p95)", @@ -616,7 +722,7 @@ "h": 8, "w": 24, "x": 0, - "y": 76 + "y": 86 }, "options": { "tooltip": { @@ -647,13 +753,16 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" }, "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}" }, "overrides": [] }, - "id": 14 + "id": 16 }, { "title": "Transaction Apply Duration per Ledger", @@ -663,7 +772,7 @@ "h": 8, "w": 12, "x": 0, - "y": 84 + "y": 94 }, "options": { "tooltip": { @@ -690,12 +799,28 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] }, - "id": 15 + "id": 17 + }, + { + "title": "Transaction Queue", + "type": "row", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 102 + }, + "collapsed": false, + "panels": [], + "id": 18 }, { "title": "TxQ Enqueue Rate by Transaction Type", @@ -704,8 +829,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 84 + "x": 0, + "y": 103 }, "options": { "tooltip": { @@ -732,12 +857,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] }, - "id": 16 + "id": 19 }, { "title": "Queue Accept (Drain) Duration per Ledger", @@ -745,9 +873,9 @@ "type": "timeseries", "gridPos": { "h": 8, - "w": 24, - "x": 0, - "y": 92 + "w": 12, + "x": 12, + "y": 103 }, "options": { "tooltip": { @@ -774,12 +902,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] }, - "id": 19 + "id": 20 }, { "title": "Queue Cleanup Rate (Expired Entries)", @@ -789,7 +920,7 @@ "h": 8, "w": 12, "x": 0, - "y": 100 + "y": 111 }, "options": { "tooltip": { @@ -816,12 +947,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] }, - "id": 20 + "id": 21 } ], "schemaVersion": 39, diff --git a/docker/telemetry/grafana/dashboards/validator-health.json b/docker/telemetry/grafana/dashboards/validator-health.json index 4e32e7e606..870eb6c4d9 100644 --- a/docker/telemetry/grafana/dashboards/validator-health.json +++ b/docker/telemetry/grafana/dashboards/validator-health.json @@ -49,15 +49,16 @@ "y": 0 }, "collapsed": false, - "panels": [] + "panels": [], + "id": 1 }, { - "title": "Agreement % (1h)", + "title": "Agreement % (1h) [$xrpl_network_type]", "description": "###### What this is:\n*Share of ledgers over the last hour where this validator agreed with the network consensus.*\n\n###### How it's computed:\n*Instantaneous gauge reading of the 1-hour agreement percentage.*\n\n###### Reading it:\n*Higher is better; green at 95%+, yellow from 80%, red below.*\n\n###### Healthy range:\n*95-100%.*\n\n###### Watch for:\n*Values below 80%, meaning the validator frequently disagrees with consensus.*\n\n###### Keywords:\n- **Consensus** *(network event)* \u2014 the protocol by which validators agree on the next ledger's transaction set and close time.\n- **Validation agreement** *(per node)* \u2014 the share of ledgers where this validator's validation matched network consensus versus was missed.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerValidationAgreementGauge`\n\n###### References:\n[Consensus](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Validation agreement](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure#validation) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#consensus)", "type": "stat", "gridPos": { - "h": 8, - "w": 24, + "h": 4, + "w": 12, "x": 0, "y": 1 }, @@ -102,17 +103,21 @@ "custom": {} }, "overrides": [] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 2 }, { - "title": "Agreement % (24h)", + "title": "Agreement % (24h) [$xrpl_network_type]", "description": "###### What this is:\n*Share of ledgers over the last 24 hours where this validator agreed with consensus.*\n\n###### How it's computed:\n*Instantaneous gauge reading of the 24-hour agreement percentage.*\n\n###### Reading it:\n*Higher is better; a smoother, longer-term view than the 1h stat.*\n\n###### Healthy range:\n*95-100%.*\n\n###### Watch for:\n*A sustained dip below 90%, which can indicate configuration drift or a network partition.*\n\n###### Keywords:\n- **Validation agreement** *(per node)* \u2014 the share of ledgers where this validator's validation matched network consensus versus was missed.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerValidationAgreementGauge`\n\n###### References:\n[Validation agreement](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure#validation) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#validation-agreement)", "type": "stat", "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 9 + "h": 4, + "w": 12, + "x": 12, + "y": 1 }, "options": { "tooltip": { @@ -155,17 +160,21 @@ "custom": {} }, "overrides": [] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 3 }, { - "title": "Agreements vs Missed (1h)", + "title": "Agreements vs Missed (1h) [$xrpl_network_type]", "description": "###### What this is:\n*Counts of agreed versus missed validations over the last hour.*\n\n###### How it's computed:\n*Instantaneous gauge readings of the 1-hour agreed and missed counts.*\n\n###### Reading it:\n*Agreements should dominate; missed should be small.*\n\n###### Healthy range:\n*Missed near 0.*\n\n###### Watch for:\n*A high missed count, meaning the validator is skipping consensus rounds.*\n\n###### Keywords:\n- **Validation agreement** *(per node)* \u2014 the share of ledgers where this validator's validation matched network consensus versus was missed.\n- **Consensus round** *(network event)* \u2014 one propose-and-revise iteration of consensus; several may run before validators converge on a ledger.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerValidationAgreementGauge`\n\n###### References:\n[Validation agreement](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure#validation) \u00b7 [Consensus round](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#validation-agreement)", "type": "bargauge", "gridPos": { - "h": 8, - "w": 24, + "h": 4, + "w": 12, "x": 0, - "y": 17 + "y": 5 }, "options": { "orientation": "horizontal", @@ -223,17 +232,21 @@ ] } ] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 4 }, { - "title": "Agreements vs Missed (24h)", + "title": "Agreements vs Missed (24h) [$xrpl_network_type]", "description": "###### What this is:\n*Counts of agreed versus missed validations over the last 24 hours.*\n\n###### How it's computed:\n*Instantaneous gauge readings of the 24-hour agreed and missed counts.*\n\n###### Reading it:\n*Agreements should dominate over the full day.*\n\n###### Healthy range:\n*Missed a small fraction of agreements.*\n\n###### Watch for:\n*A growing missed share, signalling longer-term reliability problems.*\n\n###### Keywords:\n- **Validation agreement** *(per node)* \u2014 the share of ledgers where this validator's validation matched network consensus versus was missed.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerValidationAgreementGauge`\n\n###### References:\n[Validation agreement](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure#validation) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#validation-agreement)", "type": "bargauge", "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 25 + "h": 4, + "w": 12, + "x": 12, + "y": 5 }, "options": { "orientation": "horizontal", @@ -291,7 +304,11 @@ ] } ] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 5 }, { "title": "Validation Rates", @@ -300,20 +317,21 @@ "h": 1, "w": 24, "x": 0, - "y": 33 + "y": 9 }, "collapsed": false, - "panels": [] + "panels": [], + "id": 6 }, { - "title": "Validation Rate", + "title": "Validation Rate [$xrpl_network_type]", "description": "###### What this is:\n*Validations this node sends per minute.*\n\n###### How it's computed:\n*Per-second rate of the sent-validations counter over 5 minutes, scaled to per minute.*\n\n###### Reading it:\n*Should track the ledger close cadence; roughly one validation per closed ledger.*\n\n###### Healthy range:\n*About 12-20 per minute (one per closed ledger, ~3-5s close).*\n\n###### Watch for:\n*A drop toward zero, meaning the validator has stopped participating.*\n\n###### Keywords:\n- **Validations checked vs sent** *(per node)* \u2014 validations this node received from peers and checked, versus validations it issued itself.\n- **Ledger close interval** *(network-wide)* \u2014 the network's steady ledger rhythm \u2014 roughly one closed ledger every 3-5 seconds on Mainnet.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`incrementValidationsSent (caller RCLConsensus.cpp)`\n\n###### References:\n[Validations checked vs sent](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure#validation) \u00b7 [Ledger close interval](https://xrpl.org/docs/concepts/ledgers/ledger-close-times) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#validations-checked-vs-sent)", "type": "stat", "gridPos": { - "h": 8, - "w": 24, + "h": 4, + "w": 12, "x": 0, - "y": 34 + "y": 10 }, "options": { "tooltip": { @@ -354,17 +372,21 @@ "custom": {} }, "overrides": [] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 7 }, { - "title": "Validations Checked Rate", + "title": "Validations Checked Rate [$xrpl_network_type]", "description": "###### What this is:\n*Validations received from peers and checked per minute.*\n\n###### How it's computed:\n*Per-second rate of the checked-validations counter over 5 minutes, scaled to per minute.*\n\n###### Reading it:\n*Reflects how much validation traffic the network is delivering to this node.*\n\n###### Healthy range:\n*Workload-dependent; scales with trusted validator count.*\n\n###### Watch for:\n*A sudden collapse, which suggests peer connectivity loss or network isolation.*\n\n###### Keywords:\n- **Validations checked vs sent** *(per node)* \u2014 validations this node received from peers and checked, versus validations it issued itself.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`incrementValidationsChecked (caller NetworkOPs.cpp)`\n\n###### References:\n[Validations checked vs sent](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure#validation) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#validations-checked-vs-sent)", "type": "stat", "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 42 + "h": 4, + "w": 12, + "x": 12, + "y": 10 }, "options": { "tooltip": { @@ -389,17 +411,21 @@ "custom": {} }, "overrides": [] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 8 }, { - "title": "Amendment Blocked", + "title": "Amendment Blocked [$xrpl_network_type]", "description": "###### What this is:\n*Whether the node is amendment-blocked (OK/BLOCKED).*\n\n###### How it's computed:\n*Instantaneous gauge reading of the amendment-blocked flag.*\n\n###### Reading it:\n*OK is healthy; BLOCKED means an enabled amendment is unsupported by this build.*\n\n###### Healthy range:\n*OK.*\n\n###### Watch for:\n*A BLOCKED state, which halts validation and requires a software upgrade.*\n\n###### Keywords:\n- **Amendment blocked** *(per node)* \u2014 the node has halted because the network enabled an amendment its software version does not support.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerValidatorHealthGauge`\n\n###### References:\n[Amendment blocked](https://xrpl.org/docs/concepts/networks-and-servers/amendments) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#amendment-blocked)", "type": "stat", "gridPos": { - "h": 8, - "w": 24, + "h": 4, + "w": 12, "x": 0, - "y": 50 + "y": 14 }, "options": { "tooltip": { @@ -456,17 +482,21 @@ "custom": {} }, "overrides": [] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 9 }, { - "title": "UNL Expiry (days)", + "title": "UNL Expiry (days) [$xrpl_network_type]", "description": "###### What this is:\n*Days remaining until the current UNL (trusted validator list) expires.*\n\n###### How it's computed:\n*Instantaneous gauge reading of days-to-expiry.*\n\n###### Reading it:\n*Higher is safer; green at 30+, yellow under 7, red at expiry.*\n\n###### Healthy range:\n*30+ days.*\n\n###### Watch for:\n*Fewer than 7 days, after which the node loses its trusted validator set if not renewed.*\n\n###### Keywords:\n- **UNL (Unique Node List)** *(per node)* \u2014 the list of validators a node trusts not to collude; the basis for its consensus and quorum.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerValidatorHealthGauge`\n\n###### References:\n[UNL (Unique Node List)](https://xrpl.org/docs/concepts/consensus-protocol/unl) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#unl-unique-node-list)", "type": "stat", "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 58 + "h": 4, + "w": 12, + "x": 12, + "y": 14 }, "options": { "tooltip": { @@ -507,17 +537,21 @@ "custom": {} }, "overrides": [] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 10 }, { - "title": "UNL Blocked", + "title": "UNL Blocked [$xrpl_network_type]", "description": "###### What this is:\n*Whether the node's UNL is blocked (OK/BLOCKED).*\n\n###### How it's computed:\n*Instantaneous gauge reading of the UNL-blocked flag.*\n\n###### Reading it:\n*OK is healthy; BLOCKED means validator trust cannot be established.*\n\n###### Healthy range:\n*OK.*\n\n###### Watch for:\n*A BLOCKED state, which can stop the node participating in consensus.*\n\n###### Keywords:\n- **UNL (Unique Node List)** *(per node)* \u2014 the list of validators a node trusts not to collude; the basis for its consensus and quorum.\n- **UNL blocked** *(per node)* \u2014 the node cannot establish a usable trusted validator list, so it cannot safely validate.\n- **Consensus** *(network event)* \u2014 the protocol by which validators agree on the next ledger's transaction set and close time.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerValidatorHealthGauge`\n\n###### References:\n[UNL (Unique Node List)](https://xrpl.org/docs/concepts/consensus-protocol/unl) \u00b7 [UNL blocked](https://xrpl.org/docs/concepts/consensus-protocol) \u00b7 [Consensus](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#unl-unique-node-list)", "type": "stat", "gridPos": { - "h": 8, - "w": 24, + "h": 4, + "w": 12, "x": 0, - "y": 66 + "y": 18 }, "options": { "tooltip": { @@ -574,7 +608,11 @@ "custom": {} }, "overrides": [] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 11 }, { "title": "Agreement/Missed Counters (Rate)", @@ -582,9 +620,9 @@ "type": "timeseries", "gridPos": { "h": 8, - "w": 24, + "w": 12, "x": 0, - "y": 74 + "y": 22 }, "options": { "tooltip": { @@ -616,12 +654,13 @@ "custom": { "axisLabel": "Validations / Min", "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, + "lineWidth": 1, + "fillOpacity": 0, "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "gradientMode": "none" }, "color": { "mode": "palette-classic" @@ -644,7 +683,8 @@ ] } ] - } + }, + "id": 12 }, { "title": "Server State & Consensus", @@ -653,20 +693,21 @@ "h": 1, "w": 24, "x": 0, - "y": 82 + "y": 30 }, "collapsed": false, - "panels": [] + "panels": [], + "id": 13 }, { - "title": "Validation Quorum", + "title": "Validation Quorum [$xrpl_network_type]", "description": "###### What this is:\n*Minimum number of trusted validations required to declare a ledger fully validated.*\n\n###### How it's computed:\n*Instantaneous gauge reading of the current quorum requirement.*\n\n###### Reading it:\n*Tracks the quorum derived from the active validator list; changes when the list changes.*\n\n###### Healthy range:\n*Stable at the network-appropriate value.*\n\n###### Watch for:\n*An unexpected drop, which can weaken consensus safety guarantees.*\n\n###### Keywords:\n- **Validation quorum** *(network-wide)* \u2014 the minimum number of agreeing trusted validations needed to declare a ledger fully validated.\n- **Validator list** *(network-wide)* \u2014 signed lists of recommended validators (UNLs) that peers distribute to each other.\n- **Consensus** *(network event)* \u2014 the protocol by which validators agree on the next ledger's transaction set and close time.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerValidatorHealthGauge`\n\n###### References:\n[Validation quorum](https://xrpl.org/docs/concepts/consensus-protocol/negative-unl) \u00b7 [Validator list](https://xrpl.org/docs/concepts/consensus-protocol/unl) \u00b7 [Consensus](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#validation-quorum)", "type": "stat", "gridPos": { - "h": 8, - "w": 24, + "h": 4, + "w": 12, "x": 0, - "y": 83 + "y": 31 }, "options": { "tooltip": { @@ -688,67 +729,25 @@ "defaults": { "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", "unit": "none", - "custom": {} + "custom": {}, + "decimals": 0 }, "overrides": [] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 14 }, { - "title": "State Value Timeline", - "description": "###### What this is:\n*Numeric encoding of the server operating state (disconnected, connected, syncing, tracking, full, validating, proposing) over time.*\n\n###### How it's computed:\n*Instantaneous gauge reading of the encoded state value.*\n\n###### Reading it:\n*A flat line at the full-operation state is healthy; steps show transitions.*\n\n###### Healthy range:\n*Steady at the highest (full) state.*\n\n###### Watch for:\n*Frequent transitions, useful for correlating state flapping with other metrics.*\n\n###### Keywords:\n- **Operating mode / server state** *(per node)* \u2014 the node's sync level: Disconnected, Connected, Syncing, Tracking, Full (and Validating/Proposing).\n- **Consensus mode** *(per node)* \u2014 the node's role/health in the current round: Proposing, Observing, Wrong Ledger, or Switched Ledger.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerStateTrackingGauge`\n\n###### References:\n[Operating mode / server state](https://xrpl.org/docs/references/http-websocket-apis/api-conventions/xrpld-server-states) \u00b7 [Consensus mode](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#operating-mode-server-state)", - "type": "timeseries", - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 91 - }, - "options": { - "tooltip": { - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(state_tracking{metric=\"state_value\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"State\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" - } - ], - "fieldConfig": { - "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "unit": "short", - "custom": { - "axisLabel": "State", - "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10, - "spanNulls": 1800000, - "insertNulls": false, - "showPoints": "auto", - "pointSize": 3 - }, - "color": { - "mode": "palette-classic" - } - }, - "overrides": [] - } - }, - { - "title": "Time in Current State", + "title": "Time in Current State [$xrpl_network_type]", "description": "###### What this is:\n*How long the server has held its current operating state, in seconds.*\n\n###### How it's computed:\n*Current value of the time-in-state gauge.*\n\n###### Reading it:\n*Not yet wired in the code; the value currently always reads 0.*\n\n###### Healthy range:\n*Not applicable; the value is always 0 today.*\n\n###### Watch for:\n*n/a until implemented.*\n\n###### Keywords:\n- **Operating mode / server state** *(per node)* \u2014 the node's sync level: Disconnected, Connected, Syncing, Tracking, Full (and Validating/Proposing).\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerStateTrackingGauge`\n\n###### References:\n[Operating mode / server state](https://xrpl.org/docs/references/http-websocket-apis/api-conventions/xrpld-server-states) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#operating-mode-server-state)", "type": "stat", "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 99 + "h": 4, + "w": 12, + "x": 12, + "y": 31 }, "options": { "tooltip": { @@ -773,17 +772,21 @@ "custom": {} }, "overrides": [] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 15 }, { - "title": "State Changes Rate", + "title": "State Changes Rate [$xrpl_network_type]", "description": "###### What this is:\n*Rate of server operating-state changes per hour.*\n\n###### How it's computed:\n*Per-hour rate of the state-change counter, averaged over a 1-hour window.*\n\n###### Reading it:\n*Near zero is healthy; each increment is one state transition.*\n\n###### Healthy range:\n*Near 0 changes per hour.*\n\n###### Watch for:\n*Frequent transitions, which point to network instability or configuration problems.*\n\n###### Keywords:\n- **Operating mode / server state** *(per node)* \u2014 the node's sync level: Disconnected, Connected, Syncing, Tracking, Full (and Validating/Proposing).\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`incrementStateChanges (caller NetworkOPs.cpp)`\n\n###### References:\n[Operating mode / server state](https://xrpl.org/docs/references/http-websocket-apis/api-conventions/xrpld-server-states) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#operating-mode-server-state)", "type": "stat", "gridPos": { - "h": 8, - "w": 24, + "h": 4, + "w": 12, "x": 0, - "y": 107 + "y": 35 }, "options": { "tooltip": { @@ -824,17 +827,21 @@ "custom": {} }, "overrides": [] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 16 }, { - "title": "Ledgers Closed Rate", + "title": "Ledgers Closed Rate [$xrpl_network_type]", "description": "###### What this is:\n*Ledgers closed per minute by this node.*\n\n###### How it's computed:\n*Per-second rate of the ledgers-closed counter over 5 minutes, scaled to per minute.*\n\n###### Reading it:\n*Should match the network's steady close cadence.*\n\n###### Healthy range:\n*About 12-20 per minute (one per closed ledger, ~3-5s close).*\n\n###### Watch for:\n*Deviation from the expected cadence, which indicates consensus timing trouble or the node falling behind.*\n\n###### Keywords:\n- **Ledgers closed rate** *(per node)* \u2014 how many ledgers this node closed per minute; should match the network close cadence.\n- **Ledger close interval** *(network-wide)* \u2014 the network's steady ledger rhythm \u2014 roughly one closed ledger every 3-5 seconds on Mainnet.\n- **Consensus** *(network event)* \u2014 the protocol by which validators agree on the next ledger's transaction set and close time.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`incrementLedgersClosed (caller RCLConsensus.cpp)`\n\n###### References:\n[Ledgers closed rate](https://xrpl.org/docs/concepts/ledgers/ledger-close-times) \u00b7 [Consensus](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledgers-closed-rate)", "type": "stat", "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 115 + "h": 4, + "w": 12, + "x": 12, + "y": 35 }, "options": { "tooltip": { @@ -875,17 +882,21 @@ "custom": {} }, "overrides": [] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 17 }, { - "title": "Agreement % (7d)", + "title": "Agreement % (7d) [$xrpl_network_type]", "description": "###### What this is:\n*Share of ledgers over the trailing 7 days where this validator agreed with consensus.*\n\n###### How it's computed:\n*Instantaneous gauge reading of the 7-day agreement percentage.*\n\n###### Reading it:\n*The long-term reliability window; higher is better.*\n\n###### Healthy range:\n*95-100%.*\n\n###### Watch for:\n*A gradual decline, which reflects chronic rather than transient disagreement.*\n\n###### Keywords:\n- **Validation agreement** *(per node)* \u2014 the share of ledgers where this validator's validation matched network consensus versus was missed.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerValidationAgreementGauge`\n\n###### References:\n[Validation agreement](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure#validation) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#validation-agreement)", "type": "stat", "gridPos": { - "h": 8, - "w": 24, + "h": 4, + "w": 12, "x": 0, - "y": 123 + "y": 39 }, "options": { "tooltip": { @@ -928,7 +939,60 @@ "custom": {} }, "overrides": [] - } + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 18 + }, + { + "title": "State Value Timeline", + "description": "###### What this is:\n*Numeric encoding of the server operating state (disconnected, connected, syncing, tracking, full, validating, proposing) over time.*\n\n###### How it's computed:\n*Instantaneous gauge reading of the encoded state value.*\n\n###### Reading it:\n*A flat line at the full-operation state is healthy; steps show transitions.*\n\n###### Healthy range:\n*Steady at the highest (full) state.*\n\n###### Watch for:\n*Frequent transitions, useful for correlating state flapping with other metrics.*\n\n###### Keywords:\n- **Operating mode / server state** *(per node)* \u2014 the node's sync level: Disconnected, Connected, Syncing, Tracking, Full (and Validating/Proposing).\n- **Consensus mode** *(per node)* \u2014 the node's role/health in the current round: Proposing, Observing, Wrong Ledger, or Switched Ledger.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerStateTrackingGauge`\n\n###### References:\n[Operating mode / server state](https://xrpl.org/docs/references/http-websocket-apis/api-conventions/xrpld-server-states) \u00b7 [Consensus mode](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#operating-mode-server-state)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 43 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(state_tracking{metric=\"state_value\",service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}, \"series\", \"State\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "short", + "custom": { + "axisLabel": "State", + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 0, + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 5, + "gradientMode": "none" + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "id": 19 }, { "title": "Agreements vs Missed (7d)", @@ -936,9 +1000,9 @@ "type": "timeseries", "gridPos": { "h": 8, - "w": 24, - "x": 0, - "y": 131 + "w": 12, + "x": 12, + "y": 43 }, "options": { "tooltip": { @@ -972,11 +1036,15 @@ "spanNulls": 1800000, "insertNulls": false, "showPoints": "auto", - "pointSize": 3 + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" } }, "overrides": [] - } + }, + "id": 20 } ], "schemaVersion": 39,