diff --git a/docker/telemetry/grafana/dashboards/consensus-health.json b/docker/telemetry/grafana/dashboards/consensus-health.json index 140153313e..e423e3ec07 100644 --- a/docker/telemetry/grafana/dashboards/consensus-health.json +++ b/docker/telemetry/grafana/dashboards/consensus-health.json @@ -513,6 +513,133 @@ "panels": [], "id": 12 }, + { + "title": "Close-Time Resolution Change (per Round) [$xrpl_network_type]", + "description": "###### What this is:\n*Whether the close-time rounding granularity moved Coarser, Finer, or stayed Steady versus the previous round.*\n\n###### How it's computed:\n*Rounds are counted over the window and grouped by direction of change.*\n\n###### Reading it:\n*Mostly Steady is healthy; Coarser means widening bins (more disagreement), Finer means tightening.*\n\n###### Healthy range:\n*Predominantly Steady.*\n\n###### Watch for:\n*Frequent Coarser shifts indicate the network is repeatedly failing to agree on close time.*\n\n###### Keywords:\n- **Close-time resolution** *(network event)* \u2014 the granularity (in seconds) that close times are rounded to; widens when validators disagree.\n- **Consensus round** *(network event)* \u2014 one propose-and-revise iteration of consensus; several may run before validators converge on a ledger.\n- **Close time** *(network event)* \u2014 the timestamp validators agree to stamp on a ledger, rounded to a shared resolution.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as trace spans and stored in Tempo; the value is computed directly from those raw spans by the panel's TraceQL query (no Prometheus metric involved).*\n\n###### Source:\n[RCLConsensus.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/RCLConsensus.cpp)\n\n###### Function:\n`RCLConsensus::Adaptor::doAccept`\n\n###### References:\n[Close-time resolution](https://xrpl.org/docs/concepts/ledgers/ledger-close-times) \u00b7 [Consensus round](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#close-time-resolution)", + "type": "bargauge", + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 55 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "fixed", + "fixedColor": "blue" + } + }, + "overrides": [] + }, + "options": { + "orientation": "horizontal", + "displayMode": "gradient", + "showUnfilled": true, + "reduceOptions": { + "calcs": ["sum"], + "fields": "", + "values": false + }, + "legend": { + "showLegend": false + }, + "tooltip": {} + }, + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "${DS_TEMPO}" + }, + "queryType": "traceql", + "query": "{name=\"consensus.accept.apply\" && resource.service.instance.id=~\"$node\" && resource.service.name=~\"$service_name\" && resource.deployment.environment=~\"$deployment_environment\" && resource.xrpl.network.type=~\"$xrpl_network_type\" && span.resolution_direction=~\"$resolution_direction\"} | count_over_time() by (span.resolution_direction)", + "legendFormat": "{{span.resolution_direction}}", + "refId": "A" + } + ], + "id": 13, + "transformations": [ + { + "id": "renameByRegex", + "options": { + "regex": "increased", + "renamePattern": "Coarser (more disagreement)" + } + }, + { + "id": "renameByRegex", + "options": { + "regex": "decreased", + "renamePattern": "Finer (better agreement)" + } + }, + { + "id": "renameByRegex", + "options": { + "regex": "unchanged", + "renamePattern": "Steady" + } + } + ], + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 + }, + { + "title": "Close-Time Proposal Spread (Distinct Positions per Round) [$xrpl_network_type]", + "description": "###### What this is:\n*How spread out validators' proposed close times were, as the number of distinct positions per round.*\n\n###### How it's computed:\n*Rounds are counted over the window and grouped by their distinct-position count; each bar is one count value.*\n\n###### Reading it:\n*Weight on '1 distinct position' means everyone agreed; weight on 2 or more means the proposals split.*\n\n###### Healthy range:\n*Most rounds at a single distinct position.*\n\n###### Watch for:\n*A shift toward two or more distinct positions signals growing clock drift or latency across validators.*\n\n###### Keywords:\n- **Distinct positions** *(network event)* \u2014 the number of different close-time values validators proposed in a round; one means full agreement.\n- **Consensus round** *(network event)* \u2014 one propose-and-revise iteration of consensus; several may run before validators converge on a ledger.\n- **Proposal** *(network event)* \u2014 a validator's advertised set of candidate transactions for the next ledger, revised each round.\n- **Clock drift** *(network event)* \u2014 differences between validators' clocks, which spread their proposed close times.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as trace spans and stored in Tempo; the value is computed directly from those raw spans by the panel's TraceQL query (no Prometheus metric involved).*\n\n###### Source:\n[RCLConsensus.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/RCLConsensus.cpp)\n\n###### Function:\n`RCLConsensus::Adaptor::doAccept`\n\n###### References:\n[Distinct positions](https://xrpl.org/docs/concepts/ledgers/ledger-close-times) \u00b7 [Consensus round](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#distinct-positions)", + "type": "bargauge", + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 55 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "fixed", + "fixedColor": "blue" + } + }, + "overrides": [] + }, + "options": { + "orientation": "horizontal", + "displayMode": "gradient", + "showUnfilled": true, + "reduceOptions": { + "calcs": ["sum"], + "fields": "", + "values": false + }, + "legend": { + "showLegend": false + }, + "tooltip": {} + }, + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "${DS_TEMPO}" + }, + "queryType": "traceql", + "query": "{name=\"consensus.accept.apply\" && resource.service.instance.id=~\"$node\" && resource.service.name=~\"$service_name\" && resource.deployment.environment=~\"$deployment_environment\" && resource.xrpl.network.type=~\"$xrpl_network_type\"} | count_over_time() by (span.close_time_vote_bins)", + "legendFormat": "{{span.close_time_vote_bins}} Distinct Position(s)", + "refId": "A" + } + ], + "id": 14, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 + }, { "title": "Position Update Duration", "description": "###### What this is:\n*Time this node spends each round tallying disputes and updating its consensus position, at the 95th percentile.*\n\n###### How it's computed:\n*Per-round position-update durations are aggregated to their 95th percentile over a 5-minute window.*\n\n###### Reading it:\n*Lower is better; short, flat durations are normal.*\n\n###### Healthy range:\n*A few milliseconds under normal load.*\n\n###### Watch for:\n*Sustained high durations point to heavy dispute resolution or slow convergence on close time.*\n\n###### Keywords:\n- **Consensus round** *(network event)* \u2014 one propose-and-revise iteration of consensus; several may run before validators converge on a ledger.\n- **Position update** *(per node)* \u2014 the per-round work of tallying disputes and revising the node's own consensus position.\n- **Close time** *(network event)* \u2014 the timestamp validators agree to stamp on a ledger, rounded to a shared resolution.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[Consensus.h](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/consensus/Consensus.h)\n\n###### Function:\n`Consensus::updateOurPositions`\n\n###### References:\n[Consensus round](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Close time](https://xrpl.org/docs/concepts/ledgers/ledger-close-times) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#consensus-round)", @@ -521,7 +648,7 @@ "h": 8, "w": 12, "x": 0, - "y": 55 + "y": 59 }, "options": { "tooltip": { @@ -556,7 +683,7 @@ }, "overrides": [] }, - "id": 13 + "id": 15 }, { "title": "Ledger Close Duration", @@ -566,7 +693,7 @@ "h": 8, "w": 12, "x": 12, - "y": 55 + "y": 59 }, "options": { "tooltip": { @@ -601,7 +728,7 @@ }, "overrides": [] }, - "id": 14 + "id": 16 }, { "title": "Ledger Apply Duration (doAccept)", @@ -611,7 +738,7 @@ "h": 8, "w": 12, "x": 0, - "y": 63 + "y": 67 }, "targets": [ { @@ -639,7 +766,7 @@ }, "overrides": [] }, - "id": 15, + "id": 17, "options": { "tooltip": { "mode": "multi", @@ -656,7 +783,7 @@ "h": 8, "w": 12, "x": 12, - "y": 63 + "y": 67 }, "options": { "tooltip": { @@ -685,7 +812,7 @@ "unit": "ms" } }, - "id": 16 + "id": 18 }, { "title": "Close-Time Agreement Rate (Agreed vs Disagreed)", @@ -695,7 +822,7 @@ "h": 8, "w": 12, "x": 0, - "y": 71 + "y": 75 }, "targets": [ { @@ -723,7 +850,7 @@ }, "overrides": [] }, - "id": 17, + "id": 19, "options": { "tooltip": { "mode": "multi", @@ -740,7 +867,7 @@ "h": 8, "w": 12, "x": 12, - "y": 71 + "y": 75 }, "fieldConfig": { "defaults": { @@ -826,134 +953,7 @@ "refId": "B" } ], - "id": 18 - }, - { - "title": "Close-Time Resolution Change (per Round) [$xrpl_network_type]", - "description": "###### What this is:\n*Whether the close-time rounding granularity moved Coarser, Finer, or stayed Steady versus the previous round.*\n\n###### How it's computed:\n*Rounds are counted over the window and grouped by direction of change.*\n\n###### Reading it:\n*Mostly Steady is healthy; Coarser means widening bins (more disagreement), Finer means tightening.*\n\n###### Healthy range:\n*Predominantly Steady.*\n\n###### Watch for:\n*Frequent Coarser shifts indicate the network is repeatedly failing to agree on close time.*\n\n###### Keywords:\n- **Close-time resolution** *(network event)* \u2014 the granularity (in seconds) that close times are rounded to; widens when validators disagree.\n- **Consensus round** *(network event)* \u2014 one propose-and-revise iteration of consensus; several may run before validators converge on a ledger.\n- **Close time** *(network event)* \u2014 the timestamp validators agree to stamp on a ledger, rounded to a shared resolution.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as trace spans and stored in Tempo; the value is computed directly from those raw spans by the panel's TraceQL query (no Prometheus metric involved).*\n\n###### Source:\n[RCLConsensus.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/RCLConsensus.cpp)\n\n###### Function:\n`RCLConsensus::Adaptor::doAccept`\n\n###### References:\n[Close-time resolution](https://xrpl.org/docs/concepts/ledgers/ledger-close-times) \u00b7 [Consensus round](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#close-time-resolution)", - "type": "bargauge", - "gridPos": { - "h": 4, - "w": 12, - "x": 0, - "y": 79 - }, - "fieldConfig": { - "defaults": { - "unit": "short", - "decimals": 0, - "color": { - "mode": "fixed", - "fixedColor": "blue" - } - }, - "overrides": [] - }, - "options": { - "orientation": "horizontal", - "displayMode": "gradient", - "showUnfilled": true, - "reduceOptions": { - "calcs": ["sum"], - "fields": "", - "values": false - }, - "legend": { - "showLegend": false - }, - "tooltip": {} - }, - "targets": [ - { - "datasource": { - "type": "tempo", - "uid": "${DS_TEMPO}" - }, - "queryType": "traceql", - "query": "{name=\"consensus.accept.apply\" && resource.service.instance.id=~\"$node\" && resource.service.name=~\"$service_name\" && resource.deployment.environment=~\"$deployment_environment\" && resource.xrpl.network.type=~\"$xrpl_network_type\" && span.resolution_direction=~\"$resolution_direction\"} | count_over_time() by (span.resolution_direction)", - "legendFormat": "{{span.resolution_direction}}", - "refId": "A" - } - ], - "id": 19, - "transformations": [ - { - "id": "renameByRegex", - "options": { - "regex": "increased", - "renamePattern": "Coarser (more disagreement)" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "decreased", - "renamePattern": "Finer (better agreement)" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "unchanged", - "renamePattern": "Steady" - } - } - ], - "repeat": "xrpl_network_type", - "repeatDirection": "h", - "maxPerRow": 2 - }, - { - "title": "Close-Time Proposal Spread (Distinct Positions per Round) [$xrpl_network_type]", - "description": "###### What this is:\n*How spread out validators' proposed close times were, as the number of distinct positions per round.*\n\n###### How it's computed:\n*Rounds are counted over the window and grouped by their distinct-position count; each bar is one count value.*\n\n###### Reading it:\n*Weight on '1 distinct position' means everyone agreed; weight on 2 or more means the proposals split.*\n\n###### Healthy range:\n*Most rounds at a single distinct position.*\n\n###### Watch for:\n*A shift toward two or more distinct positions signals growing clock drift or latency across validators.*\n\n###### Keywords:\n- **Distinct positions** *(network event)* \u2014 the number of different close-time values validators proposed in a round; one means full agreement.\n- **Consensus round** *(network event)* \u2014 one propose-and-revise iteration of consensus; several may run before validators converge on a ledger.\n- **Proposal** *(network event)* \u2014 a validator's advertised set of candidate transactions for the next ledger, revised each round.\n- **Clock drift** *(network event)* \u2014 differences between validators' clocks, which spread their proposed close times.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as trace spans and stored in Tempo; the value is computed directly from those raw spans by the panel's TraceQL query (no Prometheus metric involved).*\n\n###### Source:\n[RCLConsensus.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/RCLConsensus.cpp)\n\n###### Function:\n`RCLConsensus::Adaptor::doAccept`\n\n###### References:\n[Distinct positions](https://xrpl.org/docs/concepts/ledgers/ledger-close-times) \u00b7 [Consensus round](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#distinct-positions)", - "type": "bargauge", - "gridPos": { - "h": 4, - "w": 12, - "x": 12, - "y": 79 - }, - "fieldConfig": { - "defaults": { - "unit": "short", - "decimals": 0, - "color": { - "mode": "fixed", - "fixedColor": "blue" - } - }, - "overrides": [] - }, - "options": { - "orientation": "horizontal", - "displayMode": "gradient", - "showUnfilled": true, - "reduceOptions": { - "calcs": ["sum"], - "fields": "", - "values": false - }, - "legend": { - "showLegend": false - }, - "tooltip": {} - }, - "targets": [ - { - "datasource": { - "type": "tempo", - "uid": "${DS_TEMPO}" - }, - "queryType": "traceql", - "query": "{name=\"consensus.accept.apply\" && resource.service.instance.id=~\"$node\" && resource.service.name=~\"$service_name\" && resource.deployment.environment=~\"$deployment_environment\" && resource.xrpl.network.type=~\"$xrpl_network_type\"} | count_over_time() by (span.close_time_vote_bins)", - "legendFormat": "{{span.close_time_vote_bins}} Distinct Position(s)", - "refId": "A" - } - ], - "id": 20, - "repeat": "xrpl_network_type", - "repeatDirection": "h", - "maxPerRow": 2 + "id": 20 }, { "title": "Outcomes & Failures", diff --git a/docker/telemetry/grafana/dashboards/ledger-data-sync.json b/docker/telemetry/grafana/dashboards/ledger-data-sync.json index dce81b4dae..e45ecef233 100644 --- a/docker/telemetry/grafana/dashboards/ledger-data-sync.json +++ b/docker/telemetry/grafana/dashboards/ledger-data-sync.json @@ -9,6 +9,67 @@ "id": null, "links": [], "panels": [ + { + "title": "Overlay Traffic Heatmap (All Categories, Bytes In) [$xrpl_network_type]", + "description": "###### What this is:\n*All overlay traffic categories ranked by inbound bytes, giving an at-a-glance view of which message types consume the most receive bandwidth.*\n\n###### How it's computed:\n*Top categories by latest inbound byte value across all traffic categories. Each bar is labelled with its traffic category followed by the node identity; the shared `_bytes_in` suffix is dropped from the category name because the panel already reports inbound bytes.*\n\n###### Reading it:\n*The longest bars are the biggest bandwidth consumers; on a synced node transactions, proposals, and validations usually lead.*\n\n###### Healthy range:\n*workload-dependent.*\n\n###### Watch for:\n*A single ledger-data or fetch category dominating (ongoing sync) or an unexpected category topping the list.*\n\n###### Keywords:\n- **Overlay** *(network-wide)* \u2014 the peer-to-peer mesh xrpld nodes form to gossip transactions, proposals and validations.\n- **Peer** *(per node)* \u2014 another server this node holds a protocol connection to.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#overlay)", + "type": "bargauge", + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 0 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "displayMode": "gradient", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus" + }, + "expr": "label_replace(topk(20, {service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", __name__=~\".*_bytes_in\", __name__!~\"total_.*\"}), \"series\", \"$1\", \"__name__\", \"(.*)_bytes_in\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", + "unit": "decbytes", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1048576 + }, + { + "color": "red", + "value": 104857600 + } + ] + } + }, + "overrides": [] + }, + "id": 1, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 + }, { "title": "Ledger Data \u2014 Ledger", "description": "###### What this is:\n*Inbound bytes for ledger-data message categories, split into aggregate get/share plus the transaction-set, transaction-node, and account-state-node sub-types the node receives from peers.*\n\n###### How it's computed:\n*Per-category inbound byte rate per node.*\n\n###### Reading it:\n*Normally low and flat once synced. Account-state-node traffic dominates during state sync; transaction-set-candidate traffic dominates during consensus catch-up.*\n\n###### Healthy range:\n*workload-dependent; low and steady on a synced node.*\n\n###### Watch for:\n*Sustained high account-state or tx-node inbound bytes on a node that should be caught up (repeated re-sync, missing history), or a single peer driving all traffic.*\n\n###### Keywords:\n- **Ledger data** *(network event)* \u2014 the bulk transfer of ledger contents between peers during sync.\n- **Overlay** *(network-wide)* \u2014 the peer-to-peer mesh xrpld nodes form to gossip transactions, proposals and validations.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledger-data)", @@ -17,7 +78,7 @@ "h": 8, "w": 12, "x": 0, - "y": 0 + "y": 4 }, "options": { "tooltip": { @@ -57,7 +118,7 @@ }, "overrides": [] }, - "id": 1 + "id": 2 }, { "title": "Ledger Data \u2014 Transaction", @@ -67,7 +128,7 @@ "h": 8, "w": 12, "x": 12, - "y": 0 + "y": 4 }, "options": { "tooltip": { @@ -119,7 +180,7 @@ }, "overrides": [] }, - "id": 2 + "id": 3 }, { "title": "Ledger Data \u2014 Account State", @@ -129,7 +190,7 @@ "h": 8, "w": 12, "x": 0, - "y": 8 + "y": 12 }, "options": { "tooltip": { @@ -169,7 +230,7 @@ }, "overrides": [] }, - "id": 3 + "id": 4 }, { "title": "Ledger Traffic \u2014 Ledger", @@ -179,7 +240,7 @@ "h": 8, "w": 12, "x": 12, - "y": 8 + "y": 12 }, "options": { "tooltip": { @@ -219,7 +280,7 @@ }, "overrides": [] }, - "id": 4 + "id": 5 }, { "title": "Ledger Traffic \u2014 Transaction", @@ -229,7 +290,7 @@ "h": 8, "w": 12, "x": 0, - "y": 16 + "y": 20 }, "options": { "tooltip": { @@ -281,7 +342,7 @@ }, "overrides": [] }, - "id": 5 + "id": 6 }, { "title": "Ledger Traffic \u2014 Account State", @@ -291,7 +352,7 @@ "h": 8, "w": 12, "x": 12, - "y": 16 + "y": 20 }, "options": { "tooltip": { @@ -331,7 +392,7 @@ }, "overrides": [] }, - "id": 6 + "id": 7 }, { "title": "GetObject \u2014 Ledger", @@ -341,7 +402,7 @@ "h": 8, "w": 12, "x": 0, - "y": 24 + "y": 28 }, "options": { "tooltip": { @@ -381,7 +442,7 @@ }, "overrides": [] }, - "id": 7 + "id": 8 }, { "title": "GetObject \u2014 Transaction", @@ -391,7 +452,7 @@ "h": 8, "w": 12, "x": 12, - "y": 24 + "y": 28 }, "options": { "tooltip": { @@ -443,7 +504,7 @@ }, "overrides": [] }, - "id": 8 + "id": 9 }, { "title": "GetObject \u2014 Account State", @@ -453,7 +514,7 @@ "h": 8, "w": 12, "x": 0, - "y": 32 + "y": 36 }, "options": { "tooltip": { @@ -493,7 +554,7 @@ }, "overrides": [] }, - "id": 9 + "id": 10 }, { "title": "GetObject Messages \u2014 Ledger", @@ -503,7 +564,7 @@ "h": 8, "w": 12, "x": 12, - "y": 32 + "y": 36 }, "options": { "tooltip": { @@ -537,7 +598,7 @@ }, "overrides": [] }, - "id": 10 + "id": 11 }, { "title": "GetObject Messages \u2014 Transaction", @@ -547,7 +608,7 @@ "h": 8, "w": 12, "x": 0, - "y": 40 + "y": 44 }, "options": { "tooltip": { @@ -587,7 +648,7 @@ }, "overrides": [] }, - "id": 11 + "id": 12 }, { "title": "GetObject Messages \u2014 Account State", @@ -597,7 +658,7 @@ "h": 8, "w": 12, "x": 12, - "y": 40 + "y": 44 }, "options": { "tooltip": { @@ -631,7 +692,7 @@ }, "overrides": [] }, - "id": 12 + "id": 13 }, { "title": "GetObject Messages \u2014 Specials", @@ -641,7 +702,7 @@ "h": 8, "w": 12, "x": 0, - "y": 48 + "y": 52 }, "options": { "tooltip": { @@ -687,7 +748,7 @@ }, "overrides": [] }, - "id": 13 + "id": 14 }, { "title": "GetObject \u2014 Specials", @@ -697,7 +758,7 @@ "h": 8, "w": 12, "x": 12, - "y": 48 + "y": 52 }, "options": { "tooltip": { @@ -767,68 +828,7 @@ }, "overrides": [] }, - "id": 14 - }, - { - "title": "Overlay Traffic Heatmap (All Categories, Bytes In) [$xrpl_network_type]", - "description": "###### What this is:\n*All overlay traffic categories ranked by inbound bytes, giving an at-a-glance view of which message types consume the most receive bandwidth.*\n\n###### How it's computed:\n*Top categories by latest inbound byte value across all traffic categories. Each bar is labelled with its traffic category followed by the node identity; the shared `_bytes_in` suffix is dropped from the category name because the panel already reports inbound bytes.*\n\n###### Reading it:\n*The longest bars are the biggest bandwidth consumers; on a synced node transactions, proposals, and validations usually lead.*\n\n###### Healthy range:\n*workload-dependent.*\n\n###### Watch for:\n*A single ledger-data or fetch category dominating (ongoing sync) or an unexpected category topping the list.*\n\n###### Keywords:\n- **Overlay** *(network-wide)* \u2014 the peer-to-peer mesh xrpld nodes form to gossip transactions, proposals and validations.\n- **Peer** *(per node)* \u2014 another server this node holds a protocol connection to.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl ctor (TrafficGauges)`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#overlay)", - "type": "bargauge", - "gridPos": { - "h": 4, - "w": 12, - "x": 0, - "y": 56 - }, - "options": { - "tooltip": { - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - }, - "displayMode": "gradient", - "orientation": "horizontal", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus" - }, - "expr": "label_replace(topk(20, {service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", __name__=~\".*_bytes_in\", __name__!~\"total_.*\"}), \"series\", \"$1\", \"__name__\", \"(.*)_bytes_in\")" - } - ], - "fieldConfig": { - "defaults": { - "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", - "unit": "decbytes", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 1048576 - }, - { - "color": "red", - "value": 104857600 - } - ] - } - }, - "overrides": [] - }, - "id": 15, - "repeat": "xrpl_network_type", - "repeatDirection": "h", - "maxPerRow": 2 + "id": 15 }, { "title": "Sync Diagnostics", diff --git a/docker/telemetry/grafana/dashboards/ledger-operations.json b/docker/telemetry/grafana/dashboards/ledger-operations.json index df1b0a5e31..00699deb6f 100644 --- a/docker/telemetry/grafana/dashboards/ledger-operations.json +++ b/docker/telemetry/grafana/dashboards/ledger-operations.json @@ -89,6 +89,44 @@ "maxPerRow": 2, "id": 2 }, + { + "title": "Ledger Validation Rate [$xrpl_network_type]", + "description": "###### What this is:\n*How often ledgers reach full validation (accepted by the trusted validator quorum) per second.*\n\n###### How it's computed:\n*Per-second rate of ledger-validation events over 5 minutes, per node.*\n\n###### Reading it:\n*Should closely match the build rate under normal, in-sync operation.*\n\n###### Healthy range:\n*About 0.2-0.3/sec on mainnet; workload-dependent elsewhere.*\n\n###### Watch for:\n*A validation rate that lags the build rate, signalling the node is building ahead of the network consensus it trusts.*\n\n###### Keywords:\n- **Ledger validation** *(network event)* \u2014 the second consensus stage where the node confirms a built ledger matches the trusted validator quorum and marks it final.\n- **Validation quorum** *(network-wide)* \u2014 the minimum number of agreeing trusted validations needed to declare a ledger fully validated.\n- **Ledger build** *(per node)* \u2014 constructing the new ledger by applying the agreed transaction set to the prior ledger.\n- **Consensus** *(network event)* \u2014 the protocol by which validators agree on the next ledger's transaction set and close time.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[LedgerMaster.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/LedgerMaster.cpp)\n\n###### Function:\n`LedgerMaster::checkAccept`\n\n###### References:\n[Ledger validation](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure#validation) \u00b7 [Validation quorum](https://xrpl.org/docs/concepts/consensus-protocol/negative-unl) \u00b7 [Ledger build](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) \u00b7 [Consensus](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledger-validation)", + "type": "stat", + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 1 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"ledger.validate\"}[$__rate_interval])), \"series\", \"Validations / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "suffix: ledgers/s" + }, + "overrides": [] + }, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2, + "id": 3 + }, { "title": "Ledger Build Duration", "description": "###### What this is:\n*The time taken to build a single ledger, at the 95th percentile.*\n\n###### How it's computed:\n*95th-percentile of ledger-build durations over a 5-minute window, per node.*\n\n###### Reading it:\n*Lower is better; the line should stay well under the ledger interval.*\n\n###### Healthy range:\n*Typically tens to low hundreds of milliseconds; workload-dependent.*\n\n###### Watch for:\n*Sustained rises approaching the close interval, which indicate heavy transaction sets or disk/I/O pressure.*\n\n###### Keywords:\n- **Ledger build** *(per node)* \u2014 constructing the new ledger by applying the agreed transaction set to the prior ledger.\n- **Ledger close interval** *(network-wide)* \u2014 the network's steady ledger rhythm \u2014 roughly one closed ledger every 3-5 seconds on Mainnet.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[BuildLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/BuildLedger.cpp)\n\n###### Function:\n`buildLedgerImpl`\n\n###### References:\n[Ledger build](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) \u00b7 [Ledger close interval](https://xrpl.org/docs/concepts/ledgers/ledger-close-times) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledger-build)", @@ -132,44 +170,6 @@ }, "overrides": [] }, - "id": 3 - }, - { - "title": "Ledger Validation Rate [$xrpl_network_type]", - "description": "###### What this is:\n*How often ledgers reach full validation (accepted by the trusted validator quorum) per second.*\n\n###### How it's computed:\n*Per-second rate of ledger-validation events over 5 minutes, per node.*\n\n###### Reading it:\n*Should closely match the build rate under normal, in-sync operation.*\n\n###### Healthy range:\n*About 0.2-0.3/sec on mainnet; workload-dependent elsewhere.*\n\n###### Watch for:\n*A validation rate that lags the build rate, signalling the node is building ahead of the network consensus it trusts.*\n\n###### Keywords:\n- **Ledger validation** *(network event)* \u2014 the second consensus stage where the node confirms a built ledger matches the trusted validator quorum and marks it final.\n- **Validation quorum** *(network-wide)* \u2014 the minimum number of agreeing trusted validations needed to declare a ledger fully validated.\n- **Ledger build** *(per node)* \u2014 constructing the new ledger by applying the agreed transaction set to the prior ledger.\n- **Consensus** *(network event)* \u2014 the protocol by which validators agree on the next ledger's transaction set and close time.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[LedgerMaster.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/LedgerMaster.cpp)\n\n###### Function:\n`LedgerMaster::checkAccept`\n\n###### References:\n[Ledger validation](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure#validation) \u00b7 [Validation quorum](https://xrpl.org/docs/concepts/consensus-protocol/negative-unl) \u00b7 [Ledger build](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) \u00b7 [Consensus](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledger-validation)", - "type": "stat", - "gridPos": { - "h": 4, - "w": 12, - "x": 0, - "y": 13 - }, - "options": { - "tooltip": { - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"ledger.validate\"}[$__rate_interval])), \"series\", \"Validations / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" - } - ], - "fieldConfig": { - "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "unit": "suffix: ledgers/s" - }, - "overrides": [] - }, - "repeat": "xrpl_network_type", - "repeatDirection": "h", - "maxPerRow": 2, "id": 4 }, { @@ -179,8 +179,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 17 + "x": 12, + "y": 5 }, "options": { "tooltip": { @@ -218,7 +218,7 @@ "h": 1, "w": 24, "x": 0, - "y": 25 + "y": 13 }, "collapsed": false, "panels": [], @@ -232,7 +232,7 @@ "h": 8, "w": 12, "x": 0, - "y": 26 + "y": 14 }, "options": { "tooltip": { @@ -277,7 +277,7 @@ "h": 8, "w": 12, "x": 12, - "y": 26 + "y": 14 }, "options": { "tooltip": { @@ -321,7 +321,7 @@ "h": 1, "w": 24, "x": 0, - "y": 34 + "y": 22 }, "collapsed": false, "panels": [], @@ -335,7 +335,7 @@ "h": 4, "w": 12, "x": 0, - "y": 35 + "y": 23 }, "options": { "tooltip": { @@ -373,7 +373,7 @@ "h": 8, "w": 12, "x": 0, - "y": 39 + "y": 27 }, "options": { "tooltip": { @@ -425,7 +425,7 @@ "h": 8, "w": 12, "x": 12, - "y": 39 + "y": 27 }, "options": { "tooltip": { diff --git a/docker/telemetry/grafana/dashboards/node-health.json b/docker/telemetry/grafana/dashboards/node-health.json index a76d639c16..edb1cad3ac 100644 --- a/docker/telemetry/grafana/dashboards/node-health.json +++ b/docker/telemetry/grafana/dashboards/node-health.json @@ -593,6 +593,81 @@ "title": "Operating Mode", "type": "row" }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*The share of recent wall-clock time the node spent in Full mode.*\n\n###### How it's computed:\n*Per-second rate of the Full-mode duration counter divided by the sum of the per-second rates of all five mode duration counters.*\n\n###### Reading it:\n*Higher is better; 1.0 means the node was fully synced for the entire window.*\n\n###### Healthy range:\n*At or above 0.99.*\n\n###### Watch for:\n*Values dropping below 0.9, meaning the node spent meaningful time outside Full mode.*\n\n###### Keywords:\n- **Operating mode / server state** *(per node)* \u2014 the node's sync level: Disconnected, Connected, Syncing, Tracking, Full (and Validating/Proposing).\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`NetworkOPsImp::Stats`\n\n###### References:\n[Operating mode / server state](https://xrpl.org/docs/references/http-websocket-apis/api-conventions/xrpld-server-states) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#operating-mode-server-state)", + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": 0 + }, + { + "color": "yellow", + "value": 0.9 + }, + { + "color": "green", + "value": 0.99 + } + ] + }, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 21 + }, + "id": 9, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + }, + "wideLayout": true + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(rate(state_accounting_full_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) / (rate(state_accounting_disconnected_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) + rate(state_accounting_connected_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) + rate(state_accounting_syncing_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) + rate(state_accounting_tracking_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) + rate(state_accounting_full_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])), \"series\", \"% Time in Full\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "% Time in Full [$xrpl_network_type]", + "type": "stat", + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 + }, { "datasource": { "type": "prometheus", @@ -661,9 +736,9 @@ "h": 8, "w": 12, "x": 0, - "y": 21 + "y": 25 }, - "id": 9, + "id": 10, "options": { "annotations": { "clustering": -1, @@ -796,9 +871,9 @@ "h": 8, "w": 12, "x": 12, - "y": 21 + "y": 25 }, - "id": 10, + "id": 11, "options": { "annotations": { "clustering": -1, @@ -931,9 +1006,9 @@ "h": 8, "w": 12, "x": 0, - "y": 29 + "y": 33 }, - "id": 11, + "id": 12, "options": { "annotations": { "clustering": -1, @@ -1000,81 +1075,6 @@ "title": "State Duration Rate (All States)", "type": "timeseries" }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "###### What this is:\n*The share of recent wall-clock time the node spent in Full mode.*\n\n###### How it's computed:\n*Per-second rate of the Full-mode duration counter divided by the sum of the per-second rates of all five mode duration counters.*\n\n###### Reading it:\n*Higher is better; 1.0 means the node was fully synced for the entire window.*\n\n###### Healthy range:\n*At or above 0.99.*\n\n###### Watch for:\n*Values dropping below 0.9, meaning the node spent meaningful time outside Full mode.*\n\n###### Keywords:\n- **Operating mode / server state** *(per node)* \u2014 the node's sync level: Disconnected, Connected, Syncing, Tracking, Full (and Validating/Proposing).\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`NetworkOPsImp::Stats`\n\n###### References:\n[Operating mode / server state](https://xrpl.org/docs/references/http-websocket-apis/api-conventions/xrpld-server-states) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#operating-mode-server-state)", - "fieldConfig": { - "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": 0 - }, - { - "color": "yellow", - "value": 0.9 - }, - { - "color": "green", - "value": 0.99 - } - ] - }, - "unit": "percentunit" - } - }, - "gridPos": { - "h": 4, - "w": 12, - "x": 0, - "y": 37 - }, - "id": 12, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "tooltip": { - "maxHeight": 600, - "mode": "single", - "sort": "none" - }, - "wideLayout": true - }, - "pluginVersion": "13.2.0-28926505616", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(rate(state_accounting_full_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) / (rate(state_accounting_disconnected_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) + rate(state_accounting_connected_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) + rate(state_accounting_syncing_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) + rate(state_accounting_tracking_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]) + rate(state_accounting_full_duration{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])), \"series\", \"% Time in Full\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", - "refId": "A" - } - ], - "title": "% Time in Full [$xrpl_network_type]", - "type": "stat", - "repeat": "xrpl_network_type", - "repeatDirection": "h", - "maxPerRow": 2 - }, { "datasource": { "type": "prometheus", @@ -1164,8 +1164,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 41 + "x": 12, + "y": 33 }, "id": 13, "options": { @@ -1214,7 +1214,7 @@ "h": 1, "w": 24, "x": 0, - "y": 49 + "y": 41 }, "id": 14, "title": "NodeStore I/O", @@ -1286,7 +1286,7 @@ "h": 8, "w": 12, "x": 0, - "y": 50 + "y": 42 }, "id": 15, "options": { @@ -1405,7 +1405,7 @@ "h": 8, "w": 12, "x": 12, - "y": 50 + "y": 42 }, "id": 16, "options": { @@ -1512,7 +1512,7 @@ "h": 8, "w": 12, "x": 0, - "y": 58 + "y": 50 }, "id": 17, "options": { @@ -1623,7 +1623,7 @@ "h": 8, "w": 12, "x": 12, - "y": 58 + "y": 50 }, "id": 18, "options": { @@ -1735,7 +1735,7 @@ "h": 8, "w": 12, "x": 0, - "y": 66 + "y": 58 }, "id": 19, "options": { @@ -1854,7 +1854,7 @@ "h": 8, "w": 12, "x": 12, - "y": 66 + "y": 58 }, "id": 20, "options": { @@ -1897,7 +1897,7 @@ "h": 1, "w": 24, "x": 0, - "y": 74 + "y": 66 }, "id": 21, "title": "Jobs", @@ -1970,7 +1970,7 @@ "h": 8, "w": 12, "x": 0, - "y": 75 + "y": 67 }, "id": 22, "options": { @@ -2073,7 +2073,7 @@ "h": 8, "w": 12, "x": 12, - "y": 75 + "y": 67 }, "id": 23, "options": { @@ -2256,7 +2256,7 @@ "h": 8, "w": 12, "x": 0, - "y": 83 + "y": 75 }, "id": 24, "options": { @@ -2439,7 +2439,7 @@ "h": 8, "w": 12, "x": 12, - "y": 83 + "y": 75 }, "id": 25, "options": { @@ -2542,7 +2542,7 @@ "h": 8, "w": 12, "x": 0, - "y": 91 + "y": 83 }, "id": 26, "options": { @@ -2585,7 +2585,7 @@ "h": 1, "w": 24, "x": 0, - "y": 99 + "y": 91 }, "id": 27, "panels": [ @@ -3033,7 +3033,7 @@ "h": 1, "w": 24, "x": 0, - "y": 100 + "y": 92 }, "id": 28, "title": "Server Info", @@ -3174,7 +3174,7 @@ "h": 4, "w": 12, "x": 0, - "y": 101 + "y": 93 }, "id": 29, "options": { @@ -3242,7 +3242,7 @@ "h": 4, "w": 12, "x": 12, - "y": 101 + "y": 93 }, "id": 30, "options": { @@ -3316,7 +3316,7 @@ "h": 4, "w": 12, "x": 0, - "y": 105 + "y": 97 }, "id": 31, "options": { @@ -3379,7 +3379,7 @@ "h": 4, "w": 12, "x": 12, - "y": 105 + "y": 97 }, "id": 32, "options": { @@ -3480,7 +3480,7 @@ "h": 4, "w": 12, "x": 0, - "y": 109 + "y": 101 }, "id": 33, "options": { @@ -3547,7 +3547,7 @@ "h": 4, "w": 12, "x": 12, - "y": 109 + "y": 101 }, "id": 34, "options": { @@ -3612,7 +3612,7 @@ "h": 4, "w": 12, "x": 0, - "y": 113 + "y": 105 }, "id": 35, "options": { @@ -3715,7 +3715,7 @@ "h": 8, "w": 12, "x": 0, - "y": 117 + "y": 109 }, "id": 36, "options": { @@ -3815,7 +3815,7 @@ "h": 8, "w": 12, "x": 12, - "y": 117 + "y": 109 }, "id": 37, "options": { @@ -3952,7 +3952,7 @@ "h": 8, "w": 12, "x": 0, - "y": 125 + "y": 117 }, "id": 38, "options": { @@ -4059,7 +4059,7 @@ "h": 8, "w": 12, "x": 12, - "y": 125 + "y": 117 }, "id": 39, "options": { @@ -4163,7 +4163,7 @@ "h": 8, "w": 12, "x": 0, - "y": 133 + "y": 125 }, "id": 40, "options": { @@ -4206,12 +4206,81 @@ "h": 1, "w": 24, "x": 0, - "y": 141 + "y": 133 }, "id": 41, "title": "Complete Ledgers & DB", "type": "row" }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*How many historical ledgers the node is back-filling per minute.*\n\n###### How it's computed:\n*Current value of the historical-fetch-per-minute gauge.*\n\n###### Reading it:\n*Near zero once history is complete; elevated while back-filling.*\n\n###### Healthy range:\n*Close to zero in steady state.*\n\n###### Watch for:\n*A sustained high rate means the node is still filling gaps in its stored history.*\n\n###### Keywords:\n- **Historical fetch rate** *(per node)* \u2014 how many historical ledgers the node is back-filling per minute.\n- **Back-fill / catch-up** *(per node)* \u2014 fetching missing historical ledgers from peers to fill gaps or reach the network tip.\n- **Complete ledger ranges** *(per node)* \u2014 the contiguous spans of ledgers the node holds locally; one unbroken range is ideal.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerDbMetricsGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#historical-fetch-rate)", + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "suffix: fetches/min" + } + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 134 + }, + "id": 42, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + }, + "wideLayout": true + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(db_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"historical_perminute\"}, \"series\", \"Fetches/min\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Historical Fetch Rate [$xrpl_network_type]", + "type": "stat", + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 + }, { "datasource": { "type": "prometheus", @@ -4250,9 +4319,9 @@ "h": 12, "w": 24, "x": 0, - "y": 142 + "y": 138 }, - "id": 42, + "id": 43, "options": { "cellHeight": "sm", "showHeader": true, @@ -4344,9 +4413,9 @@ "h": 8, "w": 12, "x": 0, - "y": 154 + "y": 150 }, - "id": 43, + "id": 44, "options": { "annotations": { "clustering": -1, @@ -4397,75 +4466,6 @@ "title": "Database Sizes", "type": "timeseries" }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "###### What this is:\n*How many historical ledgers the node is back-filling per minute.*\n\n###### How it's computed:\n*Current value of the historical-fetch-per-minute gauge.*\n\n###### Reading it:\n*Near zero once history is complete; elevated while back-filling.*\n\n###### Healthy range:\n*Close to zero in steady state.*\n\n###### Watch for:\n*A sustained high rate means the node is still filling gaps in its stored history.*\n\n###### Keywords:\n- **Historical fetch rate** *(per node)* \u2014 how many historical ledgers the node is back-filling per minute.\n- **Back-fill / catch-up** *(per node)* \u2014 fetching missing historical ledgers from peers to fill gaps or reach the network tip.\n- **Complete ledger ranges** *(per node)* \u2014 the contiguous spans of ledgers the node holds locally; one unbroken range is ideal.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerDbMetricsGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#historical-fetch-rate)", - "fieldConfig": { - "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "suffix: fetches/min" - } - }, - "gridPos": { - "h": 4, - "w": 12, - "x": 0, - "y": 162 - }, - "id": 44, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "tooltip": { - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - }, - "wideLayout": true - }, - "pluginVersion": "13.2.0-28926505616", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(db_metrics{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"historical_perminute\"}, \"series\", \"Fetches/min\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", - "refId": "A" - } - ], - "title": "Historical Fetch Rate [$xrpl_network_type]", - "type": "stat", - "repeat": "xrpl_network_type", - "repeatDirection": "h", - "maxPerRow": 2 - }, { "datasource": { "type": "prometheus", @@ -4532,8 +4532,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 166 + "x": 12, + "y": 150 }, "id": 45, "options": { @@ -4576,7 +4576,7 @@ "h": 1, "w": 24, "x": 0, - "y": 174 + "y": 158 }, "id": 46, "title": "Ledger Economy", @@ -4611,7 +4611,7 @@ "h": 4, "w": 12, "x": 0, - "y": 175 + "y": 159 }, "id": 47, "options": { @@ -4680,7 +4680,7 @@ "h": 4, "w": 12, "x": 12, - "y": 175 + "y": 159 }, "id": 48, "options": { @@ -4749,7 +4749,7 @@ "h": 4, "w": 12, "x": 0, - "y": 179 + "y": 163 }, "id": 49, "options": { @@ -4859,7 +4859,7 @@ "h": 8, "w": 12, "x": 0, - "y": 183 + "y": 167 }, "id": 50, "options": { @@ -4962,7 +4962,7 @@ "h": 8, "w": 12, "x": 12, - "y": 183 + "y": 167 }, "id": 51, "options": { @@ -5065,7 +5065,7 @@ "h": 8, "w": 12, "x": 0, - "y": 191 + "y": 175 }, "id": 52, "options": { @@ -5168,7 +5168,7 @@ "h": 8, "w": 12, "x": 12, - "y": 191 + "y": 175 }, "id": 53, "options": { @@ -5212,7 +5212,7 @@ "h": 1, "w": 24, "x": 0, - "y": 199 + "y": 183 }, "collapsed": false, "panels": [], @@ -5226,7 +5226,7 @@ "h": 8, "w": 12, "x": 0, - "y": 200 + "y": 184 }, "options": { "tooltip": { diff --git a/docker/telemetry/grafana/dashboards/rpc-pathfinding.json b/docker/telemetry/grafana/dashboards/rpc-pathfinding.json index 5117f65c41..a23656883e 100644 --- a/docker/telemetry/grafana/dashboards/rpc-pathfinding.json +++ b/docker/telemetry/grafana/dashboards/rpc-pathfinding.json @@ -252,96 +252,6 @@ "panels": [], "id": 6 }, - { - "title": "Pathfinding Fast Duration", - "description": "###### What this is:\n*The 95th-percentile time of the fast pathfinding search.*\n\n###### How it's computed:\n*95th-percentile of fast pathfinding durations over 5 minutes, per node.*\n\n###### Reading it:\n*Lower is better; fast mode trades accuracy for speed.*\n\n###### Healthy range:\n*Typically a few to tens of milliseconds; workload-dependent.*\n\n###### Watch for:\n*A rising trend, indicating heavy pathfinding demand or an expensive order book.*\n\n###### Keywords:\n- **Pathfinding (fast / full)** *(per node)* \u2014 searching for payment paths through intermediaries; fast mode trades accuracy for speed, full is exhaustive.\n- **Order book** *(network-wide)* \u2014 the ledger's list of standing offers to trade a currency pair on the decentralized exchange.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[PathRequestManager.h](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/PathRequestManager.h)\n\n###### Function:\n`ctor`\n\n###### References:\n[Pathfinding (fast / full)](https://xrpl.org/docs/concepts/tokens/fungible-tokens/paths) \u00b7 [Order book](https://xrpl.org/docs/concepts/tokens/decentralized-exchange) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#pathfinding-fast-full)", - "type": "timeseries", - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 22 - }, - "options": { - "tooltip": { - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(pathfind_fast_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"P95 Fast Pathfind\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" - } - ], - "fieldConfig": { - "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "unit": "ms", - "custom": { - "axisLabel": "Duration (ms)", - "spanNulls": 1800000, - "insertNulls": false, - "showPoints": "auto", - "pointSize": 5, - "lineWidth": 1, - "fillOpacity": 0, - "gradientMode": "none" - } - }, - "overrides": [] - }, - "id": 7 - }, - { - "title": "Pathfinding Full Duration", - "description": "###### What this is:\n*The 95th-percentile time of the full, exhaustive pathfinding search.*\n\n###### How it's computed:\n*95th-percentile of full pathfinding durations over 5 minutes, per node.*\n\n###### Reading it:\n*Lower is better; full mode is much more expensive than fast mode.*\n\n###### Healthy range:\n*Tens to hundreds of milliseconds; workload-dependent.*\n\n###### Watch for:\n*Sustained high durations, consistent with pathfinding-heavy clients straining the node.*\n\n###### Keywords:\n- **Pathfinding (fast / full)** *(per node)* \u2014 searching for payment paths through intermediaries; fast mode trades accuracy for speed, full is exhaustive.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[PathRequestManager.h](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/PathRequestManager.h)\n\n###### Function:\n`ctor`\n\n###### References:\n[Pathfinding (fast / full)](https://xrpl.org/docs/concepts/tokens/fungible-tokens/paths) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#pathfinding-fast-full)", - "type": "timeseries", - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 22 - }, - "options": { - "tooltip": { - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(pathfind_full_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"P95 Full Pathfind\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" - } - ], - "fieldConfig": { - "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "unit": "ms", - "custom": { - "axisLabel": "Duration (ms)", - "spanNulls": 1800000, - "insertNulls": false, - "showPoints": "auto", - "pointSize": 5, - "lineWidth": 1, - "fillOpacity": 0, - "gradientMode": "none" - } - }, - "overrides": [] - }, - "id": 8 - }, { "title": "Resource Warnings Rate [$xrpl_network_type]", "description": "###### What this is:\n*How often the resource manager warns a peer or client for excessive usage, per second.*\n\n###### How it's computed:\n*Per-second rate of resource-warning events over 5 minutes, per node.*\n\n###### Reading it:\n*Green near zero, yellow above 0.1/sec, red above 1/sec.*\n\n###### Healthy range:\n*Near zero in normal operation.*\n\n###### Watch for:\n*A rising rate, consistent with aggressive clients that may need throttling.*\n\n###### Keywords:\n- **Resource drops / warnings** *(per node)* \u2014 the resource manager warning (then dropping/blocking) a peer or client for excessive usage.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[Logic.h](https://github.com/XRPLF/rippled/blob/develop/include/xrpl/resource/detail/Logic.h)\n\n###### Function:\n`Logic::Stats ctor`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#resource-drops-warnings)", @@ -350,7 +260,7 @@ "h": 4, "w": 12, "x": 0, - "y": 30 + "y": 22 }, "options": { "tooltip": { @@ -394,7 +304,7 @@ "repeat": "xrpl_network_type", "repeatDirection": "h", "maxPerRow": 2, - "id": 9 + "id": 7 }, { "title": "Resource Drops Rate [$xrpl_network_type]", @@ -404,7 +314,7 @@ "h": 4, "w": 12, "x": 12, - "y": 30 + "y": 22 }, "options": { "tooltip": { @@ -448,6 +358,96 @@ "repeat": "xrpl_network_type", "repeatDirection": "h", "maxPerRow": 2, + "id": 8 + }, + { + "title": "Pathfinding Fast Duration", + "description": "###### What this is:\n*The 95th-percentile time of the fast pathfinding search.*\n\n###### How it's computed:\n*95th-percentile of fast pathfinding durations over 5 minutes, per node.*\n\n###### Reading it:\n*Lower is better; fast mode trades accuracy for speed.*\n\n###### Healthy range:\n*Typically a few to tens of milliseconds; workload-dependent.*\n\n###### Watch for:\n*A rising trend, indicating heavy pathfinding demand or an expensive order book.*\n\n###### Keywords:\n- **Pathfinding (fast / full)** *(per node)* \u2014 searching for payment paths through intermediaries; fast mode trades accuracy for speed, full is exhaustive.\n- **Order book** *(network-wide)* \u2014 the ledger's list of standing offers to trade a currency pair on the decentralized exchange.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[PathRequestManager.h](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/PathRequestManager.h)\n\n###### Function:\n`ctor`\n\n###### References:\n[Pathfinding (fast / full)](https://xrpl.org/docs/concepts/tokens/fungible-tokens/paths) \u00b7 [Order book](https://xrpl.org/docs/concepts/tokens/decentralized-exchange) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#pathfinding-fast-full)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 26 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(pathfind_fast_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"P95 Fast Pathfind\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ms", + "custom": { + "axisLabel": "Duration (ms)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" + } + }, + "overrides": [] + }, + "id": 9 + }, + { + "title": "Pathfinding Full Duration", + "description": "###### What this is:\n*The 95th-percentile time of the full, exhaustive pathfinding search.*\n\n###### How it's computed:\n*95th-percentile of full pathfinding durations over 5 minutes, per node.*\n\n###### Reading it:\n*Lower is better; full mode is much more expensive than fast mode.*\n\n###### Healthy range:\n*Tens to hundreds of milliseconds; workload-dependent.*\n\n###### Watch for:\n*Sustained high durations, consistent with pathfinding-heavy clients straining the node.*\n\n###### Keywords:\n- **Pathfinding (fast / full)** *(per node)* \u2014 searching for payment paths through intermediaries; fast mode trades accuracy for speed, full is exhaustive.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[PathRequestManager.h](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/PathRequestManager.h)\n\n###### Function:\n`ctor`\n\n###### References:\n[Pathfinding (fast / full)](https://xrpl.org/docs/concepts/tokens/fungible-tokens/paths) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#pathfinding-fast-full)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 26 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(pathfind_full_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"P95 Full Pathfind\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ms", + "custom": { + "axisLabel": "Duration (ms)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" + } + }, + "overrides": [] + }, "id": 10 }, { diff --git a/docker/telemetry/grafana/dashboards/rpc-performance.json b/docker/telemetry/grafana/dashboards/rpc-performance.json index 797d260d8f..c85d252d1d 100644 --- a/docker/telemetry/grafana/dashboards/rpc-performance.json +++ b/docker/telemetry/grafana/dashboards/rpc-performance.json @@ -51,96 +51,6 @@ }, "panels": [] }, - { - "title": "RPC Request Rate by Command", - "description": "###### What this is:\n*How many times each RPC command runs per second.*\n\n###### How it's computed:\n*Per-second rate of command executions over 5 minutes, grouped by command name and node.*\n\n###### Reading it:\n*Compare bars/lines across commands to see which endpoints drive load.*\n\n###### Healthy range:\n*Workload-dependent; dominated by whichever commands clients call most.*\n\n###### Watch for:\n*A single command spiking far above its norm, consistent with a client hammering one endpoint.*\n\n###### Keywords:\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[RPCHandler.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/RPCHandler.cpp)\n\n###### Function:\n`callMethod`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rpc-command-method)", - "type": "timeseries", - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 1 - }, - "options": { - "tooltip": { - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(sum by (command, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", command=~\"$command\", span_name=~\"rpc.command.*\"}[$__rate_interval])), \"series\", \"$1\", \"command\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" - } - ], - "fieldConfig": { - "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "unit": "reqps", - "custom": { - "axisLabel": "Requests / Sec", - "spanNulls": 1800000, - "insertNulls": false, - "showPoints": "auto", - "pointSize": 5, - "lineWidth": 1, - "fillOpacity": 0, - "gradientMode": "none" - } - }, - "overrides": [] - }, - "id": 2 - }, - { - "title": "RPC Latency P95 by Command", - "description": "###### What this is:\n*The 95th-percentile response time for each RPC command.*\n\n###### How it's computed:\n*95th-percentile of per-command execution durations over 5 minutes, grouped by command and node.*\n\n###### Reading it:\n*Lower is better; watch the slowest commands.*\n\n###### Healthy range:\n*Fast lookups a few ms; heavy queries tens to hundreds of ms. Workload-dependent.*\n\n###### Watch for:\n*A command whose latency climbs steadily, pointing to an expensive or degrading query path.*\n\n###### Keywords:\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[RPCHandler.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/RPCHandler.cpp)\n\n###### Function:\n`callMethod`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rpc-command-method)", - "type": "timeseries", - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 1 - }, - "options": { - "tooltip": { - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, command, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", command=~\"$command\", span_name=~\"rpc.command.*\"}[5m]))), \"series\", \"P95 $1\", \"command\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" - } - ], - "fieldConfig": { - "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "unit": "ms", - "custom": { - "axisLabel": "Latency (ms)", - "spanNulls": 1800000, - "insertNulls": false, - "showPoints": "auto", - "pointSize": 5, - "lineWidth": 1, - "fillOpacity": 0, - "gradientMode": "none" - } - }, - "overrides": [] - }, - "id": 3 - }, { "title": "RPC Error Rate [$xrpl_network_type]", "description": "###### What this is:\n*The percentage of each command's calls that finished with an error.*\n\n###### How it's computed:\n*Error calls divided by total calls per command over 5 minutes, expressed as a percent.*\n\n###### Reading it:\n*Green under 1%, yellow 1-5%, red above 5%.*\n\n###### Healthy range:\n*Below 1% per command in normal operation.*\n\n###### Watch for:\n*A command sitting red, indicating malformed input, overload, or a broken handler.*\n\n###### Keywords:\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[RPCHandler.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/RPCHandler.cpp)\n\n###### Function:\n`callMethod`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rpc-command-method)", @@ -149,7 +59,7 @@ "h": 4, "w": 12, "x": 0, - "y": 9 + "y": 1 }, "options": { "tooltip": { @@ -190,11 +100,177 @@ }, "overrides": [] }, + "id": 2, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 + }, + { + "title": "Top Commands by Volume [$xrpl_network_type]", + "description": "###### What this is:\n*The ten most-called RPC commands over the recent window.*\n\n###### How it's computed:\n*Total invocation counts per command over the last 5 minutes, ranked and capped at ten.*\n\n###### Reading it:\n*The top bars are the hottest endpoints driving node load.*\n\n###### Healthy range:\n*Workload-dependent; typically led by lightweight status queries.*\n\n###### Watch for:\n*An unexpected command dominating the ranking, consistent with automated abuse.*\n\n###### Keywords:\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[RPCHandler.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/RPCHandler.cpp)\n\n###### Function:\n`callMethod`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rpc-command-method)", + "type": "bargauge", + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 1 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(topk(10, sum by (command, service_instance_id, xrpl_branch, xrpl_node_role) (increase(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", command=~\"$command\", span_name=~\"rpc.command.*\"}[$__rate_interval]))), \"series\", \"$1\", \"command\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "short" + }, + "overrides": [] + }, + "id": 3, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 + }, + { + "title": "WebSocket Message Rate [$xrpl_network_type]", + "description": "###### What this is:\n*How many RPC messages arrive over WebSocket connections per second.*\n\n###### How it's computed:\n*Per-second rate of processed WebSocket RPC messages over 5 minutes, per node.*\n\n###### Reading it:\n*Non-zero only when clients use WebSocket; zero is normal for HTTP-only nodes.*\n\n###### Healthy range:\n*Workload-dependent; zero if no WebSocket clients.*\n\n###### Watch for:\n*A sudden surge from a single source, consistent with a chatty or abusive subscription client.*\n\n###### Keywords:\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n- **WebSocket** *(per node)* \u2014 a persistent-connection API transport used by subscription clients; zero on HTTP-only nodes.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[ServerHandler.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/ServerHandler.cpp)\n\n###### Function:\n`ServerHandler::processSession`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [WebSocket](https://xrpl.org/docs/references/http-websocket-apis/api-conventions) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rpc-command-method)", + "type": "stat", + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 5 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", command=~\"$command\", span_name=\"rpc.ws_message\"}[$__rate_interval])), \"series\", \"WS Messages / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "suffix: messages/s" + }, + "overrides": [] + }, "id": 4, "repeat": "xrpl_network_type", "repeatDirection": "h", "maxPerRow": 2 }, + { + "title": "RPC Request Rate by Command", + "description": "###### What this is:\n*How many times each RPC command runs per second.*\n\n###### How it's computed:\n*Per-second rate of command executions over 5 minutes, grouped by command name and node.*\n\n###### Reading it:\n*Compare bars/lines across commands to see which endpoints drive load.*\n\n###### Healthy range:\n*Workload-dependent; dominated by whichever commands clients call most.*\n\n###### Watch for:\n*A single command spiking far above its norm, consistent with a client hammering one endpoint.*\n\n###### Keywords:\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[RPCHandler.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/RPCHandler.cpp)\n\n###### Function:\n`callMethod`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rpc-command-method)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (command, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", command=~\"$command\", span_name=~\"rpc.command.*\"}[$__rate_interval])), \"series\", \"$1\", \"command\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "reqps", + "custom": { + "axisLabel": "Requests / Sec", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" + } + }, + "overrides": [] + }, + "id": 5 + }, + { + "title": "RPC Latency P95 by Command", + "description": "###### What this is:\n*The 95th-percentile response time for each RPC command.*\n\n###### How it's computed:\n*95th-percentile of per-command execution durations over 5 minutes, grouped by command and node.*\n\n###### Reading it:\n*Lower is better; watch the slowest commands.*\n\n###### Healthy range:\n*Fast lookups a few ms; heavy queries tens to hundreds of ms. Workload-dependent.*\n\n###### Watch for:\n*A command whose latency climbs steadily, pointing to an expensive or degrading query path.*\n\n###### Keywords:\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[RPCHandler.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/RPCHandler.cpp)\n\n###### Function:\n`callMethod`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rpc-command-method)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, command, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", command=~\"$command\", span_name=~\"rpc.command.*\"}[5m]))), \"series\", \"P95 $1\", \"command\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "ms", + "custom": { + "axisLabel": "Latency (ms)", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 5, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none" + } + }, + "overrides": [] + }, + "id": 6 + }, { "title": "RPC Latency Heatmap", "description": "###### What this is:\n*The full distribution of RPC response times over the window.*\n\n###### How it's computed:\n*Counts of requests in each latency band per 5-minute window, shown as color density.*\n\n###### Reading it:\n*A tight low band is healthy; multiple bands mean mixed fast/slow requests.*\n\n###### Healthy range:\n*Most mass in the low-latency bands; workload-dependent.*\n\n###### Watch for:\n*A distinct high-latency cluster (bimodal behavior) that percentiles alone would hide.*\n\n###### Keywords:\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[RPCHandler.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/RPCHandler.cpp)\n\n###### Function:\n`callMethod`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rpc-command-method)", @@ -203,7 +279,7 @@ "h": 8, "w": 12, "x": 0, - "y": 13 + "y": 17 }, "options": { "tooltip": { @@ -232,7 +308,7 @@ "unit": "ms" } }, - "id": 5 + "id": 7 }, { "title": "Overall RPC Throughput", @@ -242,7 +318,7 @@ "h": 8, "w": 12, "x": 12, - "y": 13 + "y": 17 }, "options": { "tooltip": { @@ -284,7 +360,7 @@ }, "overrides": [] }, - "id": 6 + "id": 8 }, { "title": "RPC Success vs Error", @@ -294,7 +370,7 @@ "h": 8, "w": 12, "x": 0, - "y": 21 + "y": 25 }, "options": { "tooltip": { @@ -336,83 +412,7 @@ }, "overrides": [] }, - "id": 7 - }, - { - "title": "Top Commands by Volume [$xrpl_network_type]", - "description": "###### What this is:\n*The ten most-called RPC commands over the recent window.*\n\n###### How it's computed:\n*Total invocation counts per command over the last 5 minutes, ranked and capped at ten.*\n\n###### Reading it:\n*The top bars are the hottest endpoints driving node load.*\n\n###### Healthy range:\n*Workload-dependent; typically led by lightweight status queries.*\n\n###### Watch for:\n*An unexpected command dominating the ranking, consistent with automated abuse.*\n\n###### Keywords:\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[RPCHandler.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/RPCHandler.cpp)\n\n###### Function:\n`callMethod`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rpc-command-method)", - "type": "bargauge", - "gridPos": { - "h": 4, - "w": 12, - "x": 0, - "y": 29 - }, - "options": { - "tooltip": { - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(topk(10, sum by (command, service_instance_id, xrpl_branch, xrpl_node_role) (increase(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", command=~\"$command\", span_name=~\"rpc.command.*\"}[$__rate_interval]))), \"series\", \"$1\", \"command\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" - } - ], - "fieldConfig": { - "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "unit": "short" - }, - "overrides": [] - }, - "id": 8, - "repeat": "xrpl_network_type", - "repeatDirection": "h", - "maxPerRow": 2 - }, - { - "title": "WebSocket Message Rate [$xrpl_network_type]", - "description": "###### What this is:\n*How many RPC messages arrive over WebSocket connections per second.*\n\n###### How it's computed:\n*Per-second rate of processed WebSocket RPC messages over 5 minutes, per node.*\n\n###### Reading it:\n*Non-zero only when clients use WebSocket; zero is normal for HTTP-only nodes.*\n\n###### Healthy range:\n*Workload-dependent; zero if no WebSocket clients.*\n\n###### Watch for:\n*A sudden surge from a single source, consistent with a chatty or abusive subscription client.*\n\n###### Keywords:\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n- **WebSocket** *(per node)* \u2014 a persistent-connection API transport used by subscription clients; zero on HTTP-only nodes.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[ServerHandler.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/ServerHandler.cpp)\n\n###### Function:\n`ServerHandler::processSession`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [WebSocket](https://xrpl.org/docs/references/http-websocket-apis/api-conventions) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rpc-command-method)", - "type": "stat", - "gridPos": { - "h": 4, - "w": 12, - "x": 12, - "y": 29 - }, - "options": { - "tooltip": { - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", command=~\"$command\", span_name=\"rpc.ws_message\"}[$__rate_interval])), \"series\", \"WS Messages / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" - } - ], - "fieldConfig": { - "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "unit": "suffix: messages/s" - }, - "overrides": [] - }, - "id": 9, - "repeat": "xrpl_network_type", - "repeatDirection": "h", - "maxPerRow": 2 + "id": 9 }, { "title": "RPC Resource Cost by Command", @@ -529,6 +529,66 @@ }, "panels": [] }, + { + "title": "Current RPC Latency (p99 Gauge) [$xrpl_network_type]", + "description": "###### What this is:\n*Current tail latency (p99) of RPC handling across all methods, as a live gauge.*\n\n###### How it's computed:\n*p99 of the method-duration histogram over the recent window.*\n\n###### Reading it:\n*A single at-a-glance number for current RPC responsiveness.*\n\n###### Healthy range:\n*Low-millisecond under normal load.*\n\n###### Watch for:\n*Sustained elevation, indicating the node is under query pressure.*\n\n###### Keywords:\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordRpcFinished`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rpc-command-method)", + "type": "gauge", + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 50 + }, + "options": { + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "orientation": "auto", + "showThresholdLabels": false, + "showThresholdMarkers": true, + "tooltip": {} + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.99, sum by (service_instance_id, le, xrpl_branch, xrpl_node_role) (rate(rpc_method_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", method=~\"$method\"}[5m]))), \"series\", \"p99 Latency\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "\u00b5s", + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 100000 + }, + { + "color": "red", + "value": 1000000 + } + ] + } + }, + "overrides": [] + }, + "id": 13, + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 + }, { "title": "RPC Call Rate (All Methods)", "description": "###### What this is:\n*Overall rate of RPC method calls that started, finished, and errored, across all methods.*\n\n###### How it's computed:\n*Per-second rate of each counter over a 5-minute window, summed per node.*\n\n###### Reading it:\n*Started should closely track finished; errored should be a small fraction.*\n\n###### Healthy range:\n*Workload-dependent; started \u2248 finished, errored near zero.*\n\n###### Watch for:\n*A growing gap between started and finished (calls hanging), or an errored line that rises with load.*\n\n###### Keywords:\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordRpcStarted / recordRpcFinished / recordRpcErrored`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rpc-command-method)", @@ -537,7 +597,7 @@ "h": 8, "w": 12, "x": 0, - "y": 50 + "y": 54 }, "options": { "tooltip": { @@ -590,7 +650,7 @@ }, "overrides": [] }, - "id": 13 + "id": 14 }, { "title": "Per-Method Call Rate (Top 10)", @@ -600,7 +660,7 @@ "h": 8, "w": 24, "x": 0, - "y": 58 + "y": 62 }, "options": { "tooltip": { @@ -644,7 +704,7 @@ }, "overrides": [] }, - "id": 14 + "id": 15 }, { "title": "Per-Method Error Rate (Top 10)", @@ -654,7 +714,7 @@ "h": 8, "w": 24, "x": 0, - "y": 66 + "y": 70 }, "options": { "tooltip": { @@ -698,7 +758,7 @@ }, "overrides": [] }, - "id": 15 + "id": 16 }, { "title": "RPC Latency - All Methods", @@ -708,7 +768,7 @@ "h": 8, "w": 12, "x": 0, - "y": 74 + "y": 78 }, "options": { "tooltip": { @@ -754,7 +814,7 @@ }, "overrides": [] }, - "id": 16 + "id": 17 }, { "title": "Per-Method Latency (p99, Top 10 Slowest)", @@ -764,7 +824,7 @@ "h": 8, "w": 24, "x": 0, - "y": 82 + "y": 86 }, "options": { "tooltip": { @@ -808,7 +868,7 @@ }, "overrides": [] }, - "id": 17 + "id": 18 }, { "title": "RPC Error Ratio by Method", @@ -818,7 +878,7 @@ "h": 8, "w": 24, "x": 0, - "y": 90 + "y": 94 }, "options": { "tooltip": { @@ -880,67 +940,7 @@ }, "overrides": [] }, - "id": 18 - }, - { - "title": "Current RPC Latency (p99 Gauge) [$xrpl_network_type]", - "description": "###### What this is:\n*Current tail latency (p99) of RPC handling across all methods, as a live gauge.*\n\n###### How it's computed:\n*p99 of the method-duration histogram over the recent window.*\n\n###### Reading it:\n*A single at-a-glance number for current RPC responsiveness.*\n\n###### Healthy range:\n*Low-millisecond under normal load.*\n\n###### Watch for:\n*Sustained elevation, indicating the node is under query pressure.*\n\n###### Keywords:\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordRpcFinished`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rpc-command-method)", - "type": "gauge", - "gridPos": { - "h": 4, - "w": 12, - "x": 0, - "y": 98 - }, - "options": { - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "orientation": "auto", - "showThresholdLabels": false, - "showThresholdMarkers": true, - "tooltip": {} - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(histogram_quantile(0.99, sum by (service_instance_id, le, xrpl_branch, xrpl_node_role) (rate(rpc_method_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", method=~\"$method\"}[5m]))), \"series\", \"p99 Latency\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" - } - ], - "fieldConfig": { - "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "unit": "\u00b5s", - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 100000 - }, - { - "color": "red", - "value": 1000000 - } - ] - } - }, - "overrides": [] - }, - "id": 19, - "repeat": "xrpl_network_type", - "repeatDirection": "h", - "maxPerRow": 2 + "id": 19 } ], "schemaVersion": 39,