From d13f2fe6b483d84d207e41cfd17eff024db9f1cb Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Mon, 27 Jul 2026 21:22:33 +0100 Subject: [PATCH] fix(telemetry): show discrete sync events as counts, not per-second rates Seventeen panels on ledger-sync-health drew low-frequency discrete counters as timeseries with an ops/s unit. The rate is arithmetically right but unreadable: "0.1394 ops/s" is 278 abandoned tree phases over 33 minutes, and no reader can recover the 278. rate() also extrapolates, so whole events rendered as fractions -- peer disconnects showed 8.008 for 8 actual disconnects. Measured every candidate against live Prometheus over a 33-minute mainnet sync before converting; none exceeded 0.48 events/s, so all are count-shaped: serve_refused_total 955 events 0.4785/s ledger_quorum_shortfall_total 670 0.3354/s sync_acquire_source_total 511 0.2559/s overlay_connect_total 130 0.0652/s sync_acquire_no_progress_total 109 0.0545/s unl_fetch_total 12 0.0061/s peer_disconnect_total 8 0.0040/s Each becomes a bargauge over round(increase(...[$__range])) with unit short and decimals 0, matching the existing Mode Transitions panel. Legends follow instructions.md OTel rule 7 -- "MetricName [labels]" -- and keep node identity via xrpl_ident, so a nine-node view no longer collapses to one bar. Panel 49 also gained node identity in its aggregation, which it was missing entirely. Label values stay as emitted (upgrade_fail, not "Upgrade Fail"). They are wire identifiers: rewriting them in the legend would hide what the metric reports and break silently when a new value appears. The Title Case sits in the metric name. The five zero-valued panels were checked rather than assumed dead -- ledger_replay_*, sweep_malloc_trim_* and rotation_copy_node_restore_total all have real emit sites and are feature-gated off in this configuration. Verified against Grafana Cloud Prometheus: 0 parse errors across all panel queries, whole-number results throughout. Co-Authored-By: Claude Opus 5 (1M context) --- .../dashboards/ledger-sync-health.json | 3813 +++++++---------- 1 file changed, 1668 insertions(+), 2145 deletions(-) diff --git a/docker/telemetry/grafana/dashboards/ledger-sync-health.json b/docker/telemetry/grafana/dashboards/ledger-sync-health.json index a79a1ffff5..75fa4e55d4 100644 --- a/docker/telemetry/grafana/dashboards/ledger-sync-health.json +++ b/docker/telemetry/grafana/dashboards/ledger-sync-health.json @@ -57,62 +57,26 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*Rate of peer-hostname DNS resolutions, split by outcome (resolved or empty).*\n\n###### How it's computed:\n*Per-second rate of completed resolutions grouped by outcome, per node.*\n\n###### Reading it:\n*Resolved should account for every attempt; the empty line should stay flat at zero.*\n\n###### Healthy range:\n*A short burst of resolved at startup, then flat. Non-zero empty is always a defect.*\n\n###### Watch for:\n*Any empty rate means a configured bootstrap or [ips_fixed] hostname returned no address, so the node never even tries to dial that peer.*\n\n###### Keywords:\n- **DNS resolve** *(per node)* — turning a configured peer hostname into IP addresses before any dial is attempted; `outcome=empty` means the name resolved to nothing.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl::reportDnsResolve`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#dns-resolve)", + "description": "###### What this is:\n*Count of peer-hostname DNS resolutions, split by outcome (resolved or empty).*\n\n###### How it's computed:\n*Count over the selected range of completed resolutions grouped by outcome, per node.*\n\n###### Reading it:\n*Resolved should account for every attempt; the empty line should stay flat at zero.*\n\n###### Healthy range:\n*A short burst of resolved at startup, then flat. Non-zero empty is always a defect.*\n\n###### Watch for:\n*Any empty rate means a configured bootstrap or [ips_fixed] hostname returned no address, so the node never even tries to dial that peer.*\n\n###### Keywords:\n- **DNS resolve** *(per node)* — turning a configured peer hostname into IP addresses before any dial is attempted; `outcome=empty` means the name resolved to nothing.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl::reportDnsResolve`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#dns-resolve)", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Resolutions / Sec", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": 1800000, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "displayName": "DNS Resolve [${__field.labels.series}] ${__field.labels.xrpl_ident}", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 + "value": null } ] }, - "unit": "ops" - } + "unit": "short", + "decimals": 0 + }, + "overrides": [] }, "gridPos": { "h": 12, @@ -122,24 +86,26 @@ }, "id": 3, "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, + "displayMode": "gradient", "legend": { "calcs": [], "displayMode": "list", - "enableFacetedFilter": false, - "overflow": "ellipsis", "placement": "bottom", - "showLegend": true + "showLegend": false }, - "tooltip": { - "hideZeros": false, - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "left", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "manual", + "valueMode": "color" }, "pluginVersion": "13.2.0-28926505616", "targets": [ @@ -148,12 +114,13 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "label_replace(label_join(label_replace(sum by (outcome, service_instance_id, xrpl_branch, xrpl_node_role) (rate(dns_resolve_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])), \"series\", \"$1\", \"outcome\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", - "refId": "A" + "expr": "label_replace(label_join(label_replace(sum by (outcome, service_instance_id, xrpl_branch, xrpl_work_item) (round(increase(dns_resolve_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__range]))), \"series\", \"$1\", \"outcome\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A", + "instant": true } ], - "title": "DNS Resolve Outcome Rate", - "type": "timeseries" + "title": "DNS Resolve Outcomes (Count By Outcome)", + "type": "bargauge" }, { "datasource": { @@ -251,7 +218,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(dns_resolve_latency_ms_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"P95 DNS Resolve\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_work_item) (rate(dns_resolve_latency_ms_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"P95 DNS Resolve\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", "refId": "A" } ], @@ -263,62 +230,26 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*Rate of outbound peer connection attempts, split by terminal outcome.*\n\n###### How it's computed:\n*Per-second rate of finished dials grouped by outcome, per node. Filter the outcome set with the Dial Outcome variable.*\n\n###### Reading it:\n*Connected should dominate. The failure lines name the stage that broke: tcp_fail (no route or refused), tls_fail (TLS handshake), upgrade_fail (HTTP upgrade or protocol negotiation), timeout (no terminal state in time).*\n\n###### Healthy range:\n*Connected rising to the configured peer count, then flat with failures near zero.*\n\n###### Watch for:\n*All attempts landing on one failure outcome and no connected line — the node has no outbound peers and can never sync.*\n\n###### Keywords:\n- **Outbound dial latency** *(per node)* — an outbound peer connection attempt from TCP connect through TLS to protocol upgrade; each attempt ends in exactly one outcome.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[ConnectAttempt.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/ConnectAttempt.cpp)\n\n###### Function:\n`ConnectAttempt::reportOutcome`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#outbound-dial-latency)", + "description": "###### What this is:\n*Count of outbound peer connection attempts, split by terminal outcome.*\n\n###### How it's computed:\n*Count over the selected range of finished dials grouped by outcome, per node. Filter the outcome set with the Dial Outcome variable.*\n\n###### Reading it:\n*Connected should dominate. The failure lines name the stage that broke: tcp_fail (no route or refused), tls_fail (TLS handshake), upgrade_fail (HTTP upgrade or protocol negotiation), timeout (no terminal state in time).*\n\n###### Healthy range:\n*Connected rising to the configured peer count, then flat with failures near zero.*\n\n###### Watch for:\n*All attempts landing on one failure outcome and no connected line — the node has no outbound peers and can never sync.*\n\n###### Keywords:\n- **Outbound dial latency** *(per node)* — an outbound peer connection attempt from TCP connect through TLS to protocol upgrade; each attempt ends in exactly one outcome.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[ConnectAttempt.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/ConnectAttempt.cpp)\n\n###### Function:\n`ConnectAttempt::reportOutcome`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#outbound-dial-latency)", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Attempts / Sec", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": 1800000, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "displayName": "Dial Outcome [${__field.labels.series}] ${__field.labels.xrpl_ident}", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 + "value": null } ] }, - "unit": "ops" - } + "unit": "short", + "decimals": 0 + }, + "overrides": [] }, "gridPos": { "h": 12, @@ -328,24 +259,26 @@ }, "id": 5, "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, + "displayMode": "gradient", "legend": { "calcs": [], "displayMode": "list", - "enableFacetedFilter": false, - "overflow": "ellipsis", "placement": "bottom", - "showLegend": true + "showLegend": false }, - "tooltip": { - "hideZeros": false, - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "left", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "manual", + "valueMode": "color" }, "pluginVersion": "13.2.0-28926505616", "targets": [ @@ -354,12 +287,13 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "label_replace(label_join(label_replace(sum by (outcome, service_instance_id, xrpl_branch, xrpl_node_role) (rate(overlay_connect_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", outcome=~\"$dial_outcome\"}[$__rate_interval])), \"series\", \"$1\", \"outcome\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", - "refId": "A" + "expr": "label_replace(label_join(label_replace(sum by (outcome, service_instance_id, xrpl_branch, xrpl_work_item) (round(increase(overlay_connect_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", outcome=~\"$dial_outcome\"}[$__range]))), \"series\", \"$1\", \"outcome\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A", + "instant": true } ], - "title": "Outbound Dial Outcome Rate", - "type": "timeseries" + "title": "Outbound Dial Outcomes (Count By Outcome)", + "type": "bargauge" }, { "datasource": { @@ -457,7 +391,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(overlay_dial_latency_ms_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"P95 Outbound Dial\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_work_item) (rate(overlay_dial_latency_ms_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"P95 Outbound Dial\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", "refId": "A" } ], @@ -469,62 +403,26 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*Rate of peer handshakes rejected during protocol negotiation, split by reason.*\n\n###### How it's computed:\n*Per-second rate of rejected handshakes grouped by reason, per node. Filter the reason set with the Handshake Reason variable.*\n\n###### Reading it:\n*Flat at zero is healthy. The reason names the exact check that rejected the peer, so one dominant reason is the fault to fix.*\n\n###### Healthy range:\n*Zero, or a low background rate of self_connection and remote_ip_mismatch on a NAT'd host.*\n\n###### Watch for:\n*wrong_network or invalid_network_id — the node is configured for a different network than its peers and will never reach a quorum. clock_skew points at the local clock; session_verify_failed and bad_public_key at a misbehaving peer.*\n\n###### Keywords:\n- **Handshake negotiation failure** *(per node)* — a peer connection rejected after TLS while checking network id, clock, keys and addresses; the reason label names the failing check.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[Handshake.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/Handshake.cpp)\n\n###### Function:\n`throwNegotiationFailure`\n\n###### References:\n[Peer protocol on xrpl.org](https://xrpl.org/docs/concepts/networks-and-servers/peer-protocol) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#handshake-negotiation-failure)", + "description": "###### What this is:\n*Count of peer handshakes rejected during protocol negotiation, split by reason.*\n\n###### How it's computed:\n*Count over the selected range of rejected handshakes grouped by reason, per node. Filter the reason set with the Handshake Reason variable.*\n\n###### Reading it:\n*Flat at zero is healthy. The reason names the exact check that rejected the peer, so one dominant reason is the fault to fix.*\n\n###### Healthy range:\n*Zero, or a low background rate of self_connection and remote_ip_mismatch on a NAT'd host.*\n\n###### Watch for:\n*wrong_network or invalid_network_id — the node is configured for a different network than its peers and will never reach a quorum. clock_skew points at the local clock; session_verify_failed and bad_public_key at a misbehaving peer.*\n\n###### Keywords:\n- **Handshake negotiation failure** *(per node)* — a peer connection rejected after TLS while checking network id, clock, keys and addresses; the reason label names the failing check.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[Handshake.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/Handshake.cpp)\n\n###### Function:\n`throwNegotiationFailure`\n\n###### References:\n[Peer protocol on xrpl.org](https://xrpl.org/docs/concepts/networks-and-servers/peer-protocol) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#handshake-negotiation-failure)", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Failures / Sec", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": 1800000, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "displayName": "Handshake Fail [${__field.labels.series}] ${__field.labels.xrpl_ident}", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 + "value": null } ] }, - "unit": "ops" - } + "unit": "short", + "decimals": 0 + }, + "overrides": [] }, "gridPos": { "h": 12, @@ -534,24 +432,26 @@ }, "id": 7, "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, + "displayMode": "gradient", "legend": { "calcs": [], "displayMode": "list", - "enableFacetedFilter": false, - "overflow": "ellipsis", "placement": "bottom", - "showLegend": true + "showLegend": false }, - "tooltip": { - "hideZeros": false, - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "left", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "manual", + "valueMode": "color" }, "pluginVersion": "13.2.0-28926505616", "targets": [ @@ -560,74 +460,39 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "label_replace(label_join(label_replace(sum by (reason, service_instance_id, xrpl_branch, xrpl_node_role) (rate(handshake_negotiation_fail_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", reason=~\"$handshake_reason\"}[$__rate_interval])), \"series\", \"$1\", \"reason\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", - "refId": "A" + "expr": "label_replace(label_join(label_replace(sum by (reason, service_instance_id, xrpl_branch, xrpl_work_item) (round(increase(handshake_negotiation_fail_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", reason=~\"$handshake_reason\"}[$__range]))), \"series\", \"$1\", \"reason\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A", + "instant": true } ], - "title": "Handshake Negotiation Failures by Reason", - "type": "timeseries" + "title": "Handshake Negotiation Failures (Count By Reason)", + "type": "bargauge" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*Rate of validator-list fetches from each configured UNL site, split by outcome.*\n\n###### How it's computed:\n*Per-second rate of fetch attempts grouped by site and outcome, per node. Filter with the UNL Site and UNL Fetch Outcome variables.*\n\n###### Reading it:\n*accepted is the only success value. same_sequence and known_sequence are normal no-op refreshes of a list the node already holds. fetch_error, bad_status and parse_error are transport or content faults; expired, stale, untrusted, invalid and unsupported_version mean the list was retrieved but rejected.*\n\n###### Healthy range:\n*A first accepted per site at startup, then a steady low rate of same_sequence refreshes.*\n\n###### Watch for:\n*A site with only fetch_error or bad_status is unreachable. Only expired or invalid means the site is reachable but its list is unusable, so no trusted keys are loaded from it.*\n\n###### Keywords:\n- **UNL fetch outcome** *(per node)* — the result of retrieving and applying a validator list from a configured site; `accepted` is the only success.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[ValidatorSite.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/detail/ValidatorSite.cpp)\n\n###### Function:\n`ValidatorSite::reportFetchOutcome`\n\n###### References:\n[UNL (Unique Node List)](https://xrpl.org/docs/concepts/consensus-protocol/unl) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#unl-fetch-outcome)", + "description": "###### What this is:\n*Count of validator-list fetches from each configured UNL site, split by outcome.*\n\n###### How it's computed:\n*Count over the selected range of fetch attempts grouped by site and outcome, per node. Filter with the UNL Site and UNL Fetch Outcome variables.*\n\n###### Reading it:\n*accepted is the only success value. same_sequence and known_sequence are normal no-op refreshes of a list the node already holds. fetch_error, bad_status and parse_error are transport or content faults; expired, stale, untrusted, invalid and unsupported_version mean the list was retrieved but rejected.*\n\n###### Healthy range:\n*A first accepted per site at startup, then a steady low rate of same_sequence refreshes.*\n\n###### Watch for:\n*A site with only fetch_error or bad_status is unreachable. Only expired or invalid means the site is reachable but its list is unusable, so no trusted keys are loaded from it.*\n\n###### Keywords:\n- **UNL fetch outcome** *(per node)* — the result of retrieving and applying a validator list from a configured site; `accepted` is the only success.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[ValidatorSite.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/detail/ValidatorSite.cpp)\n\n###### Function:\n`ValidatorSite::reportFetchOutcome`\n\n###### References:\n[UNL (Unique Node List)](https://xrpl.org/docs/concepts/consensus-protocol/unl) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#unl-fetch-outcome)", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Fetches / Sec", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": 1800000, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "displayName": "UNL Fetch [${__field.labels.series}] ${__field.labels.xrpl_ident}", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 + "value": null } ] }, - "unit": "ops" - } + "unit": "short", + "decimals": 0 + }, + "overrides": [] }, "gridPos": { "h": 12, @@ -637,24 +502,26 @@ }, "id": 8, "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, + "displayMode": "gradient", "legend": { "calcs": [], "displayMode": "list", - "enableFacetedFilter": false, - "overflow": "ellipsis", "placement": "bottom", - "showLegend": true + "showLegend": false }, - "tooltip": { - "hideZeros": false, - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "left", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "manual", + "valueMode": "color" }, "pluginVersion": "13.2.0-28926505616", "targets": [ @@ -663,12 +530,13 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "label_replace(label_join(label_join(sum by (outcome, site, service_instance_id, xrpl_branch, xrpl_node_role) (rate(unl_fetch_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", site=~\"$unl_site\", outcome=~\"$unl_outcome\"}[$__rate_interval])), \"series\", \" \", \"outcome\", \"site\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", - "refId": "A" + "expr": "label_replace(label_join(label_join(sum by (outcome, site, service_instance_id, xrpl_branch, xrpl_work_item) (round(increase(unl_fetch_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", site=~\"$unl_site\", outcome=~\"$unl_outcome\"}[$__range]))), \"series\", \" \", \"outcome\", \"site\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A", + "instant": true } ], - "title": "UNL Fetch Rate by Site & Outcome", - "type": "timeseries" + "title": "UNL Fetch Outcomes (Count By Site & Outcome)", + "type": "bargauge" }, { "datasource": { @@ -1089,7 +957,7 @@ }, "custom": { "axisBorderShow": false, - "axisCenteredZero": true, + "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Ledgers", "axisPlacement": "auto", @@ -1119,7 +987,8 @@ }, "thresholdsStyle": { "mode": "line" - } + }, + "axisSoftMin": 0 }, "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", "thresholds": { @@ -1151,6 +1020,14 @@ { "id": "custom.axisLabel", "value": "Ledgers behind tip" + }, + { + "id": "custom.axisSoftMin", + "value": -5 + }, + { + "id": "custom.axisSoftMax", + "value": 5 } ] } @@ -1408,47 +1285,13 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*Rate of inbound peer connection handoffs, split by terminal outcome.*\n\n###### How it's computed:\n*Per-second rate of finished inbound handoffs grouped by outcome, per node. Filter the outcome set with the Accept Outcome variable.*\n\n###### Reading it:\n*accepted should dominate. The failure outcomes name what rejected the connection: resource_limit and no_slot are this node's own capacity, protocol_mismatch and bad_cookie are the peer or the network identity, handshake_error and local_endpoint_fail are the transport.*\n\n###### Healthy range:\n*accepted dominant, failures near zero.*\n\n###### Watch for:\n*Read this together with Outbound Dial Outcome Rate above, which is the outbound twin on overlay_connect_total{outcome}. The two share the same one-outcome-per-attempt shape, so together they give the full in/out split: a node that accepts nothing but dials successfully has a very different fault from one that can neither dial nor accept. A steady no_slot or resource_limit is this node refusing peers it has no room for, which is capacity rather than a fault.*\n\n###### Keywords:\n- **Inbound peer accept outcome** *(per node)* — the terminal result of an inbound peer connection handoff; each handoff ends in exactly one outcome.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl::onHandoff`\n\n###### References:\n[Overlay](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#overlay) · [Outbound dial latency](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#outbound-dial-latency)", + "description": "###### What this is:\n*Count of inbound peer connection handoffs, split by terminal outcome.*\n\n###### How it's computed:\n*Count over the selected range of finished inbound handoffs grouped by outcome, per node. Filter the outcome set with the Accept Outcome variable.*\n\n###### Reading it:\n*accepted should dominate. The failure outcomes name what rejected the connection: resource_limit and no_slot are this node's own capacity, protocol_mismatch and bad_cookie are the peer or the network identity, handshake_error and local_endpoint_fail are the transport.*\n\n###### Healthy range:\n*accepted dominant, failures near zero.*\n\n###### Watch for:\n*Read this together with Outbound Dial Outcome Rate above, which is the outbound twin on overlay_connect_total{outcome}. The two share the same one-outcome-per-attempt shape, so together they give the full in/out split: a node that accepts nothing but dials successfully has a very different fault from one that can neither dial nor accept. A steady no_slot or resource_limit is this node refusing peers it has no room for, which is capacity rather than a fault.*\n\n###### Keywords:\n- **Inbound peer accept outcome** *(per node)* — the terminal result of an inbound peer connection handoff; each handoff ends in exactly one outcome.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl::onHandoff`\n\n###### References:\n[Overlay](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#overlay) · [Outbound dial latency](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#outbound-dial-latency)", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Handoffs / Sec", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": 1800000, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "displayName": "Peer Accept [${__field.labels.series}] ${__field.labels.xrpl_ident}", "thresholds": { "mode": "absolute", "steps": [ @@ -1458,8 +1301,10 @@ } ] }, - "unit": "ops" - } + "unit": "short", + "decimals": 0 + }, + "overrides": [] }, "gridPos": { "h": 12, @@ -1469,24 +1314,26 @@ }, "id": 32, "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, + "displayMode": "gradient", "legend": { "calcs": [], "displayMode": "list", - "enableFacetedFilter": false, - "overflow": "ellipsis", "placement": "bottom", - "showLegend": true + "showLegend": false }, - "tooltip": { - "hideZeros": false, - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "left", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "manual", + "valueMode": "color" }, "pluginVersion": "13.2.0-28926505616", "targets": [ @@ -1495,59 +1342,26 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "label_replace(label_join(label_replace(sum by (outcome, service_instance_id, xrpl_branch, xrpl_work_item) (rate(peer_accept_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", outcome=~\"$accept_outcome\"}[$__rate_interval])), \"series\", \"$1\", \"outcome\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", - "refId": "A" + "expr": "label_replace(label_join(label_replace(sum by (outcome, service_instance_id, xrpl_branch, xrpl_work_item) (round(increase(peer_accept_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", outcome=~\"$accept_outcome\"}[$__range]))), \"series\", \"$1\", \"outcome\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A", + "instant": true } ], - "title": "Inbound Peer Accept Outcomes", - "type": "timeseries" + "title": "Inbound Peer Accept Outcomes (Count By Outcome)", + "type": "bargauge" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*Rate at which peer connections are closed, split by why they closed and by which side opened them.*\n\n###### How it's computed:\n*Per-second rate of peer closes grouped by reason and direction (inbound or outbound), per node. Filter with the Disconnect Reason and Disconnect Direction variables.*\n\n###### Reading it:\n*The split separates our-fault backpressure from topology and network faults. large_sendq and charge_resources are this node shedding a peer it cannot keep up with. not_useful and ping_timeout are topology and liveness. read_error and write_error are the transport. graceful, stopping and shutdown are ordinary lifecycle, not faults.*\n\n###### Healthy range:\n*Mostly graceful; the fault reasons near zero.*\n\n###### Watch for:\n*A sustained large_sendq or charge_resources means this node is the bottleneck and is dropping peers — which removes the very peers it needs to sync from, so a slow node makes itself slower. A rising not_useful or ping_timeout on outbound points at the peer set instead. Distinct from the single unlabelled peer-disconnect total on the Node Health dashboard, which cannot say which of these is happening.*\n\n###### Keywords:\n- **Peer disconnect reason** *(per node)* — the cause recorded when a peer connection is closed, paired with whether that connection was inbound or outbound.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[PeerImp.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/PeerImp.cpp)\n\n###### Function:\n`PeerImp::close`\n\n###### References:\n[Resource disconnect](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#resource-disconnect) · [Insane / diverged peers](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#insane-diverged-peers)", + "description": "###### What this is:\n*Rate at which peer connections are closed, split by why they closed and by which side opened them.*\n\n###### How it's computed:\n*Count over the selected range of peer closes grouped by reason and direction (inbound or outbound), per node. Filter with the Disconnect Reason and Disconnect Direction variables.*\n\n###### Reading it:\n*The split separates our-fault backpressure from topology and network faults. large_sendq and charge_resources are this node shedding a peer it cannot keep up with. not_useful and ping_timeout are topology and liveness. read_error and write_error are the transport. graceful, stopping and shutdown are ordinary lifecycle, not faults.*\n\n###### Healthy range:\n*Mostly graceful; the fault reasons near zero.*\n\n###### Watch for:\n*A sustained large_sendq or charge_resources means this node is the bottleneck and is dropping peers — which removes the very peers it needs to sync from, so a slow node makes itself slower. A rising not_useful or ping_timeout on outbound points at the peer set instead. Distinct from the single unlabelled peer-disconnect total on the Node Health dashboard, which cannot say which of these is happening.*\n\n###### Keywords:\n- **Peer disconnect reason** *(per node)* — the cause recorded when a peer connection is closed, paired with whether that connection was inbound or outbound.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[PeerImp.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/PeerImp.cpp)\n\n###### Function:\n`PeerImp::close`\n\n###### References:\n[Resource disconnect](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#resource-disconnect) · [Insane / diverged peers](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#insane-diverged-peers)", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Disconnects / Sec", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": 1800000, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "displayName": "Peer Disconnect [${__field.labels.series}] ${__field.labels.xrpl_ident}", "thresholds": { "mode": "absolute", "steps": [ @@ -1557,8 +1371,10 @@ } ] }, - "unit": "ops" - } + "unit": "short", + "decimals": 0 + }, + "overrides": [] }, "gridPos": { "h": 12, @@ -1568,24 +1384,26 @@ }, "id": 33, "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, + "displayMode": "gradient", "legend": { "calcs": [], "displayMode": "list", - "enableFacetedFilter": false, - "overflow": "ellipsis", "placement": "bottom", - "showLegend": true + "showLegend": false }, - "tooltip": { - "hideZeros": false, - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "left", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "manual", + "valueMode": "color" }, "pluginVersion": "13.2.0-28926505616", "targets": [ @@ -1594,12 +1412,13 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "label_replace(label_join(label_join(sum by (reason, direction, service_instance_id, xrpl_branch, xrpl_work_item) (rate(peer_disconnect_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", reason=~\"$disconnect_reason\", direction=~\"$disconnect_direction\"}[$__rate_interval])), \"series\", \" \", \"reason\", \"direction\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", - "refId": "A" + "expr": "label_replace(label_join(label_join(sum by (reason, direction, service_instance_id, xrpl_branch, xrpl_work_item) (round(increase(peer_disconnect_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", reason=~\"$disconnect_reason\", direction=~\"$disconnect_direction\"}[$__range]))), \"series\", \" \", \"reason\", \"direction\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A", + "instant": true } ], - "title": "Peer Disconnects by Reason", - "type": "timeseries" + "title": "Peer Disconnects (Count By Reason & Direction)", + "type": "bargauge" }, { "collapsed": false, @@ -1918,62 +1737,26 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*How often the server enters a NEW stall episode, as opposed to how long one stall lasts.*\n\n###### How it's computed:\n*Per-second rate of server_stall_events_total, which counts once per stall episode rather than once per stalled second.*\n\n###### Reading it:\n*Flat at zero is healthy. Read it beside the Server Stall stat: a rising rate means repeated fresh stalls, while a flat rate with a large stall value means one long unresolved stall.*\n\n###### Healthy range:\n*0.*\n\n###### Watch for:\n*Any repeating rate. Recurring short stalls and one long stall have different causes, and this panel is what separates them.*\n\n###### Keywords:\n- **Server stall** *(per node)* — the main loop failing to check in with the load monitor, measured in seconds of unresponsiveness.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[LoadManager.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/main/LoadManager.cpp)\n\n###### Function:\n`LoadManager::updateStallState`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#server-stall)", + "description": "###### What this is:\n*How often the server enters a NEW stall episode, as opposed to how long one stall lasts.*\n\n###### How it's computed:\n*Count over the selected range of server_stall_events_total, which counts once per stall episode rather than once per stalled second.*\n\n###### Reading it:\n*Flat at zero is healthy. Read it beside the Server Stall stat: a rising rate means repeated fresh stalls, while a flat rate with a large stall value means one long unresolved stall.*\n\n###### Healthy range:\n*0.*\n\n###### Watch for:\n*Any repeating rate. Recurring short stalls and one long stall have different causes, and this panel is what separates them.*\n\n###### Keywords:\n- **Server stall** *(per node)* — the main loop failing to check in with the load monitor, measured in seconds of unresponsiveness.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[LoadManager.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/main/LoadManager.cpp)\n\n###### Function:\n`LoadManager::updateStallState`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#server-stall)", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Episodes / Sec", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": 1800000, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 1 + "value": null } ] }, - "unit": "short" - } + "unit": "short", + "decimals": 0 + }, + "overrides": [] }, "gridPos": { "h": 12, @@ -1983,24 +1766,26 @@ }, "id": 16, "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, + "displayMode": "gradient", "legend": { "calcs": [], "displayMode": "list", - "enableFacetedFilter": false, - "overflow": "ellipsis", "placement": "bottom", - "showLegend": true + "showLegend": false }, - "tooltip": { - "hideZeros": false, - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "left", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "manual", + "valueMode": "color" }, "pluginVersion": "13.2.0-28926505616", "targets": [ @@ -2009,12 +1794,13 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "label_replace(label_join(label_replace(rate(server_stall_events_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Stall Episodes\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", - "refId": "A" + "expr": "label_replace(label_join(label_replace(round(increase(server_stall_events_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__range])), \"series\", \"Stall Episodes\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A", + "instant": true } ], - "title": "Server Stall Event Rate", - "type": "timeseries" + "title": "Server Stall Episodes (Count)", + "type": "bargauge" }, { "datasource": { @@ -2208,62 +1994,26 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*Rate of ledger-acquire timeouts where not a single new node arrived since the previous timeout.*\n\n###### How it's computed:\n*rate of sync_acquire_no_progress_total, incremented on each acquire timeout whose progress flag was false. The acquire timer fires every 3 seconds at most.*\n\n###### Reading it:\n*Zero means every timeout window saw at least some new data. Any sustained rate means acquires are repeatedly timing out with nothing received.*\n\n###### Healthy range:\n*0 on a synced node; brief non-zero bursts during initial sync are normal.*\n\n###### Watch for:\n*A sustained rate together with a flat Missing SHAMap Nodes panel: the node is asking and no peer is answering. Check peer count and whether any peer holds the ledger range being requested.*\n\n###### Keywords:\n- **Acquire stall** *(per node)* — an acquire timeout in which no new SHAMap node was received, so the acquire made no progress at all.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (call-site metric macro, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[InboundLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/InboundLedger.cpp)\n\n###### Function:\n`InboundLedger::onTimer`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#acquire-stall)", + "description": "###### What this is:\n*Count of ledger-acquire timeouts where not a single new node arrived since the previous timeout.*\n\n###### How it's computed:\n*rate of sync_acquire_no_progress_total, incremented on each acquire timeout whose progress flag was false. The acquire timer fires every 3 seconds at most.*\n\n###### Reading it:\n*Zero means every timeout window saw at least some new data. Any sustained rate means acquires are repeatedly timing out with nothing received.*\n\n###### Healthy range:\n*0 on a synced node; brief non-zero bursts during initial sync are normal.*\n\n###### Watch for:\n*A sustained rate together with a flat Missing SHAMap Nodes panel: the node is asking and no peer is answering. Check peer count and whether any peer holds the ledger range being requested.*\n\n###### Keywords:\n- **Acquire stall** *(per node)* — an acquire timeout in which no new SHAMap node was received, so the acquire made no progress at all.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (call-site metric macro, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[InboundLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/InboundLedger.cpp)\n\n###### Function:\n`InboundLedger::onTimer`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#acquire-stall)", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Stalls / Sec", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": 1800000, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "displayName": "Acquire Stalls ${__field.labels.xrpl_ident}", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null - }, - { - "color": "red", - "value": 0.1 } ] }, - "unit": "ops" - } + "unit": "short", + "decimals": 0 + }, + "overrides": [] }, "gridPos": { "h": 12, @@ -2273,24 +2023,26 @@ }, "id": 19, "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, + "displayMode": "gradient", "legend": { "calcs": [], "displayMode": "list", - "enableFacetedFilter": false, - "overflow": "ellipsis", "placement": "bottom", - "showLegend": true + "showLegend": false }, - "tooltip": { - "hideZeros": false, - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "left", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "manual", + "valueMode": "color" }, "pluginVersion": "13.2.0-28926505616", "targets": [ @@ -2299,12 +2051,13 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "label_replace(label_join(label_replace(rate(sync_acquire_no_progress_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Stalled Timeouts\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", - "refId": "A" + "expr": "label_replace(label_join(label_replace(round(increase(sync_acquire_no_progress_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__range])), \"series\", \"Stalled Timeouts\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A", + "instant": true } ], - "title": "Acquire Stall Rate (no progress)", - "type": "timeseries" + "title": "Acquire Stalls — No Progress (Count)", + "type": "bargauge" }, { "datasource": { @@ -2513,47 +2266,13 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*Rate of ledger acquires split by whether the local node store already held the whole ledger or the data had to come from peers.*\n\n###### How it's computed:\n*rate of sync_acquire_source_total by source, emitted once per new acquire right after the first local-store lookup.*\n\n###### Reading it:\n*network dominant during initial sync is expected — nothing is local yet. local dominant on a warm node means the store is serving requests without peer traffic.*\n\n###### Healthy range:\n*Mostly local on a warm node with complete history.*\n\n###### Watch for:\n*Sustained network on a node that should already hold the range: the local store is not retaining data, so sync is disk-bound rather than peer-bound. Read with the SHAMap cache hit-rate panel.*\n\n###### Keywords:\n- **Acquire source** *(per node)* — whether a ledger acquire was satisfied entirely from the local node store or required fetching from peers.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (call-site metric macro, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[InboundLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/InboundLedger.cpp)\n\n###### Function:\n`InboundLedger::init`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#acquire-source)", + "description": "###### What this is:\n*Count of ledger acquires split by whether the local node store already held the whole ledger or the data had to come from peers.*\n\n###### How it's computed:\n*rate of sync_acquire_source_total by source, emitted once per new acquire right after the first local-store lookup.*\n\n###### Reading it:\n*network dominant during initial sync is expected — nothing is local yet. local dominant on a warm node means the store is serving requests without peer traffic.*\n\n###### Healthy range:\n*Mostly local on a warm node with complete history.*\n\n###### Watch for:\n*Sustained network on a node that should already hold the range: the local store is not retaining data, so sync is disk-bound rather than peer-bound. Read with the SHAMap cache hit-rate panel.*\n\n###### Keywords:\n- **Acquire source** *(per node)* — whether a ledger acquire was satisfied entirely from the local node store or required fetching from peers.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (call-site metric macro, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[InboundLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/InboundLedger.cpp)\n\n###### Function:\n`InboundLedger::init`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#acquire-source)", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Acquires / Sec", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 30, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": 1800000, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "displayName": "Acquire Source [${__field.labels.series}] ${__field.labels.xrpl_ident}", "thresholds": { "mode": "absolute", "steps": [ @@ -2563,8 +2282,10 @@ } ] }, - "unit": "ops" - } + "unit": "short", + "decimals": 0 + }, + "overrides": [] }, "gridPos": { "h": 12, @@ -2574,24 +2295,26 @@ }, "id": 21, "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, + "displayMode": "gradient", "legend": { "calcs": [], "displayMode": "list", - "enableFacetedFilter": false, - "overflow": "ellipsis", "placement": "bottom", - "showLegend": true + "showLegend": false }, - "tooltip": { - "hideZeros": false, - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "left", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "manual", + "valueMode": "color" }, "pluginVersion": "13.2.0-28926505616", "targets": [ @@ -2600,12 +2323,13 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "label_replace(label_join(label_replace(sum by (source, service_instance_id, xrpl_branch, xrpl_work_item) (rate(sync_acquire_source_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", source=~\"$acquire_source\"}[$__rate_interval])), \"series\", \"$1\", \"source\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", - "refId": "A" + "expr": "label_replace(label_join(label_replace(sum by (source, service_instance_id, xrpl_branch, xrpl_work_item) (round(increase(sync_acquire_source_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", source=~\"$acquire_source\"}[$__range]))), \"series\", \"$1\", \"source\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A", + "instant": true } ], - "title": "Acquire Source (local vs network)", - "type": "timeseries" + "title": "Acquire Source (Count: Local vs Network)", + "type": "bargauge" }, { "datasource": { @@ -3019,62 +2743,26 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*Rate at which the pre-accept gate refused to declare a candidate ledger validated because its trusted validations were below quorum.*\n\n###### How it's computed:\n*Rate of ledger_quorum_shortfall_total by stage. One increment per rejected gate evaluation, emitted from the early return that was trace-log-only before, so a node that peers and receives validations yet never validates is no longer indistinguishable from an idle one. The gate is re-entered on every fresh trusted validation for the candidate ledger, so one ledger that eventually validates can contribute several increments on its way there.*\n\n###### Reading it:\n*A steady low rate is NORMAL and is not a fault. The gate is evaluated the instant this node finishes building a ledger, before its peers' validations for that ledger have arrived, so the first evaluation of each round routinely tallies short and is retried as validations come in. Read this against the rate of ledger closes and against Trusted Validations vs Quorum Target — it is the ratio and the accompanying tally that carry the signal, never the bare presence of a rate.*\n\n###### Healthy range:\n*A low steady rate on the order of one per ledger close or less, on a node that is validating.*\n\n###### Watch for:\n*A rate that climbs well above the ledger-close rate while Publish Lag grows and Time to First Validated Ledger stays at zero — that combination is the retry loop never converging, so the tally is not merely early, it never reaches the target. Confirm on Trusted Validations vs Quorum Target: a climbing tally is slow and will finish, a flat tally below the target is stuck (too few trusted validators reachable, or a UNL / negative-UNL misconfiguration). Check UNL Quorum Headroom in the Bootstrap row before anything else in this row, because a trusted list that cannot satisfy quorum makes every panel below it look starved.*\n\n###### Keywords:\n- **Quorum shortfall** *(per node)* — trusted validations for a candidate ledger falling short of the quorum needed to declare it validated, so the node holds the ledger and still cannot call it validated.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (call-site metric macro, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[LedgerMaster.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/LedgerMaster.cpp)\n\n###### Function:\n`LedgerMaster::checkAccept`\n\n###### References:\n[Negative UNL and validation quorum on xrpl.org](https://xrpl.org/docs/concepts/consensus-protocol/negative-unl) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#quorum-shortfall)", + "description": "###### What this is:\n*Rate at which the pre-accept gate refused to declare a candidate ledger validated because its trusted validations were below quorum.*\n\n###### How it's computed:\n*Count of ledger_quorum_shortfall_total by stage. One increment per rejected gate evaluation, emitted from the early return that was trace-log-only before, so a node that peers and receives validations yet never validates is no longer indistinguishable from an idle one. The gate is re-entered on every fresh trusted validation for the candidate ledger, so one ledger that eventually validates can contribute several increments on its way there.*\n\n###### Reading it:\n*A steady low rate is NORMAL and is not a fault. The gate is evaluated the instant this node finishes building a ledger, before its peers' validations for that ledger have arrived, so the first evaluation of each round routinely tallies short and is retried as validations come in. Read this against the rate of ledger closes and against Trusted Validations vs Quorum Target — it is the ratio and the accompanying tally that carry the signal, never the bare presence of a rate.*\n\n###### Healthy range:\n*A low steady rate on the order of one per ledger close or less, on a node that is validating.*\n\n###### Watch for:\n*A rate that climbs well above the ledger-close rate while Publish Lag grows and Time to First Validated Ledger stays at zero — that combination is the retry loop never converging, so the tally is not merely early, it never reaches the target. Confirm on Trusted Validations vs Quorum Target: a climbing tally is slow and will finish, a flat tally below the target is stuck (too few trusted validators reachable, or a UNL / negative-UNL misconfiguration). Check UNL Quorum Headroom in the Bootstrap row before anything else in this row, because a trusted list that cannot satisfy quorum makes every panel below it look starved.*\n\n###### Keywords:\n- **Quorum shortfall** *(per node)* — trusted validations for a candidate ledger falling short of the quorum needed to declare it validated, so the node holds the ledger and still cannot call it validated.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (call-site metric macro, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[LedgerMaster.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/LedgerMaster.cpp)\n\n###### Function:\n`LedgerMaster::checkAccept`\n\n###### References:\n[Negative UNL and validation quorum on xrpl.org](https://xrpl.org/docs/concepts/consensus-protocol/negative-unl) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#quorum-shortfall)", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Shortfalls / Sec", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": 1800000, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "displayName": "Quorum Shortfall [${__field.labels.series}] ${__field.labels.xrpl_ident}", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null - }, - { - "color": "red", - "value": 0.1 } ] }, - "unit": "ops" - } + "unit": "short", + "decimals": 0 + }, + "overrides": [] }, "gridPos": { "h": 12, @@ -3084,24 +2772,26 @@ }, "id": 44, "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, + "displayMode": "gradient", "legend": { "calcs": [], "displayMode": "list", - "enableFacetedFilter": false, - "overflow": "ellipsis", "placement": "bottom", - "showLegend": true + "showLegend": false }, - "tooltip": { - "hideZeros": false, - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "left", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "manual", + "valueMode": "color" }, "pluginVersion": "13.2.0-28926505616", "targets": [ @@ -3110,12 +2800,13 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "label_replace(label_join(label_replace(sum by (stage, service_instance_id, xrpl_branch, xrpl_work_item) (rate(ledger_quorum_shortfall_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", stage=~\"$shortfall_stage\"}[$__rate_interval])), \"series\", \"$1\", \"stage\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", - "refId": "A" + "expr": "label_replace(label_join(label_replace(sum by (stage, service_instance_id, xrpl_branch, xrpl_work_item) (round(increase(ledger_quorum_shortfall_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", stage=~\"$shortfall_stage\"}[$__range]))), \"series\", \"$1\", \"stage\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A", + "instant": true } ], - "title": "Pre-Accept Quorum Shortfall Rate", - "type": "timeseries" + "title": "Pre-Accept Quorum Shortfalls (Count By Stage)", + "type": "bargauge" }, { "datasource": { @@ -3506,7 +3197,19 @@ ] }, "unit": "s", - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}" + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "mappings": [ + { + "type": "value", + "options": { + "-1": { + "text": "None Pending", + "color": "green", + "index": 0 + } + } + } + ] } }, "gridPos": { @@ -3653,7 +3356,160 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*Peer requests that this node declined to serve, split by why it declined and by what was asked for.*\n\n###### How it's computed:\n*Per-second rate of refused serve attempts grouped by reason and request type (ledger, tx-set, object or fetchpack), per node. Filter with the Serve Reason and Serve Request variables, which list the emitted label values verbatim (the tx-set one is a single unhyphenated word).*\n\n###### Reading it:\n*This is the other side of the sync exchange: every other panel here measures what this node fetches, this one measures what it refuses to give back. sendq_full and load_shed are self-inflicted backpressure — the node holds the data but will not send it. not_found and no_map mean the requester asked for history this node does not hold.*\n\n###### Healthy range:\n*Near zero.*\n\n###### Watch for:\n*sendq_full or load_shed climbing. This node is starving its peers, and on a network of similarly loaded nodes that is exactly the condition that makes everybody's sync slow, so a refusal here can be the cause of another operator's stall. A high not_found is usually benign on a node configured with short history, but it tells peers to look elsewhere.*\n\n###### Keywords:\n- **Serve refusal** *(per node)* — a peer request for ledger, transaction-set, object or fetch-pack data that this node declined, together with the reason it declined.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[PeerImp.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/PeerImp.cpp)\n\n###### Function:\n`PeerImp::processLedgerRequest` · `PeerImp::onMessage(TMGetObjectByHash)` · `PeerImp::doFetchPack`\n\n###### References:\n[Fetch-pack](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#fetch-pack) · [GetObject / object fetch](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#getobject-object-fetch)", + "description": "###### What this is:\n*Peer requests that this node declined to serve, split by why it declined and by what was asked for.*\n\n###### How it's computed:\n*Count over the selected range of refused serve attempts grouped by reason and request type (ledger, tx-set, object or fetchpack), per node. Filter with the Serve Reason and Serve Request variables, which list the emitted label values verbatim (the tx-set one is a single unhyphenated word).*\n\n###### Reading it:\n*This is the other side of the sync exchange: every other panel here measures what this node fetches, this one measures what it refuses to give back. sendq_full and load_shed are self-inflicted backpressure — the node holds the data but will not send it. not_found and no_map mean the requester asked for history this node does not hold.*\n\n###### Healthy range:\n*Near zero.*\n\n###### Watch for:\n*sendq_full or load_shed climbing. This node is starving its peers, and on a network of similarly loaded nodes that is exactly the condition that makes everybody's sync slow, so a refusal here can be the cause of another operator's stall. A high not_found is usually benign on a node configured with short history, but it tells peers to look elsewhere.*\n\n###### Keywords:\n- **Serve refusal** *(per node)* — a peer request for ledger, transaction-set, object or fetch-pack data that this node declined, together with the reason it declined.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[PeerImp.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/PeerImp.cpp)\n\n###### Function:\n`PeerImp::processLedgerRequest` · `PeerImp::onMessage(TMGetObjectByHash)` · `PeerImp::doFetchPack`\n\n###### References:\n[Fetch-pack](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#fetch-pack) · [GetObject / object fetch](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#getobject-object-fetch)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "displayName": "Serve Refused [${__field.labels.series}] ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short", + "decimals": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 235 + }, + "id": 34, + "options": { + "displayMode": "gradient", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "left", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "manual", + "valueMode": "color" + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_join(sum by (reason, request, service_instance_id, xrpl_branch, xrpl_work_item) (round(increase(serve_refused_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", reason=~\"$serve_reason\", request=~\"$serve_request\"}[$__range]))), \"series\", \" \", \"reason\", \"request\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A", + "instant": true + } + ], + "title": "Ledger/Object Serve Refusals (Count By Reason & Request)", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Rate at which this node abandoned the chain tip it had built and switched to a different last-closed ledger reported by the network.*\n\n###### How it's computed:\n*Count over the selected range of last-closed-ledger switches where the ledger the network reported was not the one this node built on, per node. The counter carries no labels, so the sum is over one series per node.*\n\n###### Reading it:\n*Any non-zero value means this node was told the network's last-closed ledger is not the one it built on, and it discarded its own chain tip in response. A single jump around startup or a restart is ordinary recovery.*\n\n###### Healthy range:\n*0.*\n\n###### Watch for:\n*Repeated jumps — that is wrong-chain thrash, not one-off recovery. Check the peer set and the configured network id: a node peered to the wrong network, or into a minority partition, keeps being overruled and keeps throwing away work. Pair with the Bootstrap row and with Ledgers Behind Network.*\n\n###### Keywords:\n- **Byzantine ledger jump** *(per node)* — the node replaced its own last-closed ledger with a different one reported by the network, discarding the chain tip it had built.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`NetworkOPsImp::switchLastClosedLedger`\n\n###### References:\n[Fork](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#fork) · [Ledger history mismatch](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledger-history-mismatch)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short", + "decimals": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 235 + }, + "id": 35, + "options": { + "displayMode": "gradient", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "left", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "manual", + "valueMode": "color" + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_work_item) (round(increase(ledger_jump_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__range]))), \"series\", \"Ledger Jumps\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A", + "instant": true + } + ], + "title": "Byzantine Ledger Jumps (Count)", + "type": "bargauge" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 247 + }, + "id": 62, + "panels": [], + "title": "Back-fill & persistence", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Node-store write latency next to read latency, in microseconds per operation. The write side is the signal: a node with a large existing database back-fills slower than a fresh one, and back-fill is write-bound, so no read-side metric can show it.*\n\n###### How it's computed:\n*nodestore_latency series write_mean_us and read_mean_us, each divided by its own count series so the reading is the latency during the selected interval rather than the average since boot. The write numerator comes from a store-duration total that was declared but never written before this signal existed.*\n\n###### Reading it:\n*Compare the two lines. Reads far above writes points at the read path or a cold cache; writes far above reads points at backend write pressure, which is the large-existing-database case.*\n\n###### Healthy range:\n*Both well under a few hundred microseconds on healthy local storage.*\n\n###### Watch for:\n*A rising write line during history back-fill: the backend cannot absorb writes fast enough and sync will stay slow no matter how many peers are available. Read with Fetch-Pack Peer Starvation to tell a data-supply problem from a disk problem. This is a mean, not a percentile — a tail that matters will move it, but p99 is not available from this signal.*\n\n###### Keywords:\n- **Node-store write latency** *(per node)* — how long the node store takes to persist one object.\n- **Node-store read latency** *(per node)* — how long the node store takes to retrieve one object.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerNodeStoreLatencyGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#node-store-write-latency)", "fieldConfig": { "defaults": { "color": { @@ -3663,12 +3519,12 @@ "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "Refusals / Sec", + "axisLabel": "Microseconds / Op", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, @@ -3677,7 +3533,950 @@ }, "insertNulls": false, "lineInterpolation": "linear", - "lineWidth": 1, + "lineWidth": 2, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "µs" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 248 + }, + "id": 38, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_work_item) (rate(nodestore_latency{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"write_duration_us\"}[$__rate_interval])) / clamp_min(sum by (service_instance_id, xrpl_branch, xrpl_work_item) (rate(nodestore_latency{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"write_count\"}[$__rate_interval])), 1), \"series\", \"Write us/op (interval)\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_work_item) (rate(nodestore_latency{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"read_duration_us\"}[$__rate_interval])) / clamp_min(sum by (service_instance_id, xrpl_branch, xrpl_work_item) (rate(nodestore_latency{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"read_count\"}[$__rate_interval])), 1), \"series\", \"Read us/op (interval)\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "B" + } + ], + "title": "NodeStore Write vs Read Latency (us/op)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Node-store write and read operation rates — the denominators behind the latency panel.*\n\n###### How it's computed:\n*Rate of the nodestore_latency write_count and read_count series.*\n\n###### Reading it:\n*Writes climb while a node is back-filling history and fall to near the ledger-close rate once it is caught up.*\n\n###### Healthy range:\n*Non-zero writes whenever the node is ingesting ledgers.*\n\n###### Watch for:\n*Write rate at zero while the node is still behind the network: nothing is being persisted, so the stall is upstream of the node store — check peer supply and the acquire panels rather than storage. A flat latency with a collapsing operation rate also means the latency figure above has gone stale rather than good.*\n\n###### Keywords:\n- **Node-store operation rate** *(per node)* — stores and fetches per second against the node store.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerNodeStoreLatencyGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#node-store-operation-rate)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Ops / Sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 248 + }, + "id": 39, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_work_item) (rate(nodestore_latency{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"write_count\"}[$__rate_interval])), \"series\", \"Writes/s\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_work_item) (rate(nodestore_latency{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"read_count\"}[$__rate_interval])), \"series\", \"Reads/s\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "B" + } + ], + "title": "NodeStore Operation Rate (writes vs reads)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Rate at which ledger-replay sub-tasks give up and fall back to acquiring a whole ledger, split by which sub-task gave up.*\n\n###### How it's computed:\n*Count of ledger_replay_fallback_total by stage. The skip-list stage and the delta stage each emit once, on the transition into fallback, from the branch that was debug-log-only before.*\n\n###### Reading it:\n*Zero when replay-capable peers are available. Any sustained rate means the replay optimisation is being defeated and back-fill has reverted to the slower full-acquire path.*\n\n###### Healthy range:\n*Zero, or brief spikes while the peer set changes.*\n\n###### Watch for:\n*A persistent rate on either stage: too few connected peers support the ledger-replay feature, so every historical ledger is fetched in full instead of as a delta. Read with Replay Outcomes.*\n\n###### Keywords:\n- **Replay fallback** *(per node)* — a replay sub-task abandoning the delta shortcut and acquiring the entire ledger instead.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[SkipListAcquire.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/SkipListAcquire.cpp)\n\n###### Function:\n`SkipListAcquire::trigger` / `LedgerDeltaAcquire::trigger`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#replay-fallback)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "displayName": "Replay Fallback [${__field.labels.series}] ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short", + "decimals": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 260 + }, + "id": 40, + "options": { + "displayMode": "gradient", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "left", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "manual", + "valueMode": "color" + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (stage, service_instance_id, xrpl_branch, xrpl_work_item) (round(increase(ledger_replay_fallback_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", stage=~\"$replay_stage\"}[$__range]))), \"series\", \"$1\", \"stage\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A", + "instant": true + } + ], + "title": "Replay Fallback To Full Acquire (Count By Stage)", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Count of ledger-replay tasks reaching each terminal state: success, timeout, build failure or parameter failure.*\n\n###### How it's computed:\n*Count of ledger_replay_outcome_total by outcome. Every terminal path in the replay task emits exactly once; before this, all four only set an internal flag and wrote a log line.*\n\n###### Reading it:\n*Successes only is healthy. Timeouts mean deltas never arrived; build failures mean a delta would not apply to its parent; parameter failures mean a peer served an inconsistent skip list.*\n\n###### Healthy range:\n*Successes non-zero while replaying, all failure outcomes at zero.*\n\n###### Watch for:\n*Any failure outcome climbing while successes stay flat: replay is running but never completing, so history back-fill is silently falling back to the slower path. The outcome value tells you which layer to look at — timeouts point at peers, build and parameter failures point at the data those peers served.*\n\n###### Keywords:\n- **Ledger replay** *(per node)* — rebuilding a range of historical ledgers from a start ledger plus per-ledger deltas instead of downloading each one whole.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[LedgerReplayTask.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/LedgerReplayTask.cpp)\n\n###### Function:\n`LedgerReplayTask::recordOutcome`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledger-replay)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "displayName": "Replay Outcome [${__field.labels.series}] ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short", + "decimals": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 260 + }, + "id": 41, + "options": { + "displayMode": "gradient", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "left", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "manual", + "valueMode": "color" + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (outcome, service_instance_id, xrpl_branch, xrpl_work_item) (round(increase(ledger_replay_outcome_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", outcome=~\"$replay_outcome\"}[$__range]))), \"series\", \"$1\", \"outcome\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A", + "instant": true + } + ], + "title": "Replay Outcomes (Count By Terminal State)", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*How long the heap trim that ends every cache sweep takes. The trim returns free heap pages to the kernel, and its cost scales with the resident heap — so this is the panel that shows a node with a large existing database paying a per-sweep penalty a fresh node never pays.*\n\n###### How it's computed:\n*p50 and p95 of the sweep_malloc_trim_us histogram. Recorded once per sweep, so the sample rate is one per sweep interval (10 s on a tiny node through 120 s on a huge one). The measurement used to run only at debug log level, so on an ordinary node there was no value at all.*\n\n###### Reading it:\n*Read the two lines together against the sweep interval. Sub-millisecond is free. Tens of milliseconds against a 10 s interval is still a small duty cycle but means the trim is walking a large heap, and it runs on the sweep job — so the cost lands on the job queue, not in the background.*\n\n###### Healthy range:\n*Under a millisecond on a warm node with a modest heap.*\n\n###### Watch for:\n*p95 climbing as the database grows, especially alongside a rising sweep-job queue wait. IMPORTANT LIMITATION: the fault counter beside this panel covers the trim call only. It proves the trim itself faults; it does NOT prove the trim causes the later faults taken as the caches refill. Correlate this duration against sweep-job queueing rather than concluding the trim caused a slow sync.*\n\n###### Keywords:\n- **Heap trim** *(per node)* — returning free heap pages from the allocator's arenas back to the kernel.\n- **Sweep interval** *(per node)* — how often the periodic cache sweep runs, set by node size.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[Application.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/main/Application.cpp)\n\n###### Function:\n`ApplicationImp::trimHeapAndRecord`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#heap-trim)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Trim Duration", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "µs" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 272 + }, + "id": 64, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.5, sum by (le, service_instance_id, xrpl_branch, xrpl_work_item) (rate(sweep_malloc_trim_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"P50 Trim\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_work_item) (rate(sweep_malloc_trim_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"P95 Trim\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "B" + } + ], + "title": "Sweep Heap-Trim Duration (p50/p95)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Two rates side by side: minor page faults taken inside the heap trim, and the resident memory the trim actually returned to the kernel. Together they say whether the trim is buying anything for what it costs.*\n\n###### How it's computed:\n*Rate of sweep_malloc_trim_minor_faults_total and of sweep_malloc_trim_reclaimed_kb_total. Both are cumulative counters exported per sweep, so the panel rates them rather than plotting the totals. Reclaim is published only when resident memory actually fell; a sweep during which another thread grew the heap contributes nothing rather than a negative amount.*\n\n###### Reading it:\n*Reclaim rate with a near-zero fault rate is a cheap, useful trim. A fault rate that moves with the reclaim rate means the pages are being handed back and immediately taken again, which is churn rather than savings.*\n\n###### Healthy range:\n*A reclaim rate that tracks cache turnover, with faults near zero.*\n\n###### Watch for:\n*Reclaim near zero while the trim duration panel shows real time being spent: the trim is walking the heap and freeing nothing, which is pure cost. IMPORTANT LIMITATION: the fault delta is scoped to the trim call, so it cannot show the re-fault cost paid later as the caches refill and touch the returned pages. That later cost is real but is NOT measured here; do not read a low fault rate as proof the trim was free.*\n\n###### Keywords:\n- **Minor page fault** *(per node)* — a memory access satisfied without disk I/O, by mapping a page the kernel already holds.\n- **Reclaimed resident memory** *(per node)* — resident kilobytes the allocator handed back to the kernel.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[Application.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/main/Application.cpp)\n\n###### Function:\n`ApplicationImp::trimHeapAndRecord`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#minor-page-fault)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Faults & KB / Sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "cps" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 272 + }, + "id": 65, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_work_item) (rate(sweep_malloc_trim_minor_faults_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])), \"series\", \"Minor Faults / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_work_item) (rate(sweep_malloc_trim_reclaimed_kb_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])), \"series\", \"Reclaimed KB / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "B" + } + ], + "title": "Sweep Heap-Trim Faults & Reclaim Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*When an online-delete rotation is running, and the rate of the extra writes it forces. A rotation rewrites into the new backend any node body the doomed archive serves, which is I/O an ordinary fetch would never perform — and it exists only on a populated, already-rotated database, which is why it never appears on a fresh node.*\n\n###### How it's computed:\n*rotation_state with metric=in_flight plotted raw (it is a 0/1 state flag), and metric=copy_forward — a cumulative write total — plotted as a rate. Both are read from the node store on each collection tick. The copy-forward count existed before as a log-only per-rotation tally that reset on every swap; the total behind this panel never resets, so it can be rated.*\n\n###### Reading it:\n*The two must move together: copy-forward writes should only appear while the window flag is 1. Read the write rate against the node-store write latency panel above — that is what tells extra rotation writes from a slow backend.*\n\n###### Healthy range:\n*Flag at 0 most of the time, rising to 1 briefly once per delete interval, with the write rate non-zero only inside those windows.*\n\n###### Watch for:\n*A copy-forward rate that is large enough to move node-store write latency: rotation is competing with sync I/O, which is the whole hypothesis this panel tests. Copy-forward writes while the flag reads 0 would mean the window flag leaked, not that rotation is cheap. NO SERIES AT ALL on either query means online_delete is not configured on this node, which is different from a rotation that costs nothing.*\n\n###### Keywords:\n- **Rotation window** *(per node)* — the interval during which an online-delete backend swap is in progress.\n- **Copy-forward write** *(per node)* — rewriting a node body from the backend about to be deleted into the one replacing it.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerRotationStateGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rotation-window)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "State & Writes / Sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "cps" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 284 + }, + "id": 66, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_work_item) (rotation_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"in_flight\"}), \"series\", \"Rotation In Flight (0/1)\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_work_item) (rate(rotation_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"copy_forward\"}[$__rate_interval])), \"series\", \"Copy-Forward Writes / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "B" + } + ], + "title": "Online-Delete Rotation Window & Copy-Forward Writes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Rate at which a rotation has to rescue a tree node that was missing from BOTH backends and rewrite it from memory. Each one is an extra write on top of the whole-state-map walk the rotation already performs.*\n\n###### How it's computed:\n*Count of rotation_copy_node_restore_total. Incremented once per rescued node in the rotation's state-map walk, from a branch that was warn-log-only before, so the volume was invisible unless someone was reading logs.*\n\n###### Reading it:\n*Zero on a healthy node. Any sustained rate means clean nodes reachable from the validated state map have no on-disk copy left, because the backend holding them was removed by an earlier rotation and they were never rewritten.*\n\n###### Healthy range:\n*Flat at zero.*\n\n###### Watch for:\n*A non-zero rate is the signal that earlier rotations dropped data the current state map still needs: each rescue is a write that competes with sync I/O, and without the rescue the node would later surface as an unresolvable missing-node error. Read with the copy-forward panel — both are rotation-time writes, but this one also indicates prior data loss rather than merely cost. The node hash is deliberately not a label (unbounded cardinality); get it from the copyNode warning in Loki, correlated by node and time.*\n\n###### Keywords:\n- **Node re-store** *(per node)* — rewriting an in-memory tree node whose only on-disk copy was removed by an earlier rotation.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[SHAMapStoreImp.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/SHAMapStoreImp.cpp)\n\n###### Function:\n`SHAMapStoreImp::copyNode`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#node-re-store)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short", + "decimals": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 284 + }, + "id": 67, + "options": { + "displayMode": "gradient", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "left", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "manual", + "valueMode": "color" + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_work_item) (round(increase(rotation_copy_node_restore_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__range]))), \"series\", \"Nodes Re-Stored\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A", + "instant": true + } + ], + "title": "Rotation Node Re-Stores (Count)", + "type": "bargauge" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 248 + }, + "id": 63, + "panels": [], + "title": "Spans & traces", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*95th-percentile duration of each of the three phases of one ledger acquisition: the header wait, the account-state tree fetch, and the transaction tree fetch.*\n\n###### How it's computed:\n*95th percentile of the span-derived duration histogram for the three ledger.acquire.header / .astree / .txtree child spans, one series per phase. The parent ledger.acquire span is flat and cannot separate them, which matters because the account-state tree is nearly all of the work in a real fresh sync.*\n\n###### Reading it:\n*The astree series dominating is expected and healthy. The value of the split is the comparison: a header series that is large means the node is waiting to be told what to fetch, which is a peer-supply problem upstream of either tree.*\n\n###### Healthy range:\n*The astree series largest, the txtree series small, the header series near zero.*\n\n###### Watch for:\n*A hot astree band that keeps growing means account-state nodes are not being served — check Ledger Acquire Phase Outcomes beside it for timed_out, and Outbound Dial Outcome Rate in the Bootstrap row for refused or timed-out dials. A large header series instead means no peer is answering the header request at all.*\n\n###### Keywords:\n- **Acquire phase** *(per node)* — one of the three sequential fetches a ledger acquisition is made of; the header gates both trees, because it is what names their root hashes.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Derived by the OTel Collector's spanmetrics connector from spans emitted by xrpld code; the collector counts the spans and their durations, and the Grafana query selects and aggregates the resulting series.*\n\n###### Source:\n[InboundLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/InboundLedger.cpp)\n\n###### Function:\n`InboundLedger::syncPhaseSpans`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#acquire-phase)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Duration", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ms" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 249 + }, + "id": 48, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, span_name, service_instance_id, xrpl_branch, xrpl_work_item) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"ledger.acquire.header\"}[$__rate_interval]))), \"series\", \"P95 header\", \"span_name\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, span_name, service_instance_id, xrpl_branch, xrpl_work_item) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"ledger.acquire.astree\"}[$__rate_interval]))), \"series\", \"P95 account-state tree\", \"span_name\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, span_name, service_instance_id, xrpl_branch, xrpl_work_item) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"ledger.acquire.txtree\"}[$__rate_interval]))), \"series\", \"P95 transaction tree\", \"span_name\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "C" + } + ], + "title": "Ledger Acquire Phase Duration (p95 by phase)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*How many ledger-acquire phases finished each way over the selected range, as whole counts.*\n\n###### How it's computed:\n*`increase(span_calls_total[$__range])` per `span_name` and `outcome`, so the number shown is the count of phases that ended that way inside the dashboard time range.*\n\n###### Reading it:\n*Counts, not rates. These are discrete, low-frequency events: a rate of 0.14/s is 278 phases over 33 minutes, which no reader can infer from the rate. Compare the Complete and Abandoned bars for the same phase. Header needs one round trip; both tree phases need many.*\n\n###### Healthy range:\n*Abandoned near zero for every phase, and Complete rising with ledger progress.*\n\n###### Watch for:\n*Abandoned dominating the tree phases while Header still completes. That means requests reach peers and headers arrive, but the many-round-trip phases never finish. `Abandoned` here means the acquire object was destroyed while still fetching, not that it timed out — see Acquire Stall Rate and the runbook for the sweep mechanism.*\n\n###### Keywords:\n- **Acquire phase** *(per node)* — one of the three sequential fetches of a ledger: header, account-state tree, transaction tree.\n- **Abandoned** *(per node)* — the acquire was destroyed with no result, i.e. dropped while still fetching rather than failing or timing out.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Spans are emitted by xrpld code; the collector's spanmetrics connector turns them into `span_calls_total`; the Grafana query counts the increase over the range.*\n\n###### Source:\n[InboundLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/InboundLedger.cpp)\n\n###### Function:\n`InboundLedger::endPhaseSpan`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#fresh-node-sync-diagnostics)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "displayName": "Acquire Phase [${__field.labels.series}] ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short", + "decimals": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 249 + }, + "id": 49, + "options": { + "displayMode": "gradient", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "left", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "manual", + "valueMode": "color" + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_join(label_replace(label_replace(label_replace(label_replace(label_replace(label_replace(label_replace(sum by (span_name, outcome, service_instance_id, xrpl_branch, xrpl_work_item) (round(increase(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", outcome=~\"$span_outcome\", span_name=~\"ledger\\\\.acquire\\\\..*\"}[$__range]))), \"phase\", \"Header\", \"span_name\", \"ledger\\\\.acquire\\\\.header\"), \"phase\", \"Account State Tree\", \"span_name\", \"ledger\\\\.acquire\\\\.astree\"), \"phase\", \"Transaction Tree\", \"span_name\", \"ledger\\\\.acquire\\\\.txtree\"), \"outcome_t\", \"Complete\", \"outcome\", \"complete\"), \"outcome_t\", \"Abandoned\", \"outcome\", \"abandoned\"), \"outcome_t\", \"Timeout\", \"outcome\", \"timeout\"), \"outcome_t\", \"Failed\", \"outcome\", \"failed\"), \"series\", \" \", \"phase\", \"outcome_t\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A", + "instant": true + } + ], + "title": "Ledger Acquire Phase Outcomes (Count By Phase & Outcome)", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Rate at which attempts to fetch a transaction set referenced by a consensus proposal reach each terminal state.*\n\n###### How it's computed:\n*Rate of span-derived call counts for the txset.acquire span, split by outcome. The span is new: transaction-set acquisition had no telemetry at all before, so a consensus round stalled waiting on a set looked identical to an idle one. Every exit stamps an outcome, including the sweep that drops a set which never arrived.*\n\n###### Reading it:\n*Only complete is healthy. Timeout means peers never supplied the set. Abandoned means the round moved on and the set was dropped mid-fetch. Failed means a peer served data that would not build.*\n\n###### Healthy range:\n*A low rate of complete during consensus, everything else at zero. Zero everywhere is also normal on a node that already holds every proposed set locally.*\n\n###### Watch for:\n*Timeout or abandoned climbing: proposed sets are not arriving, so rounds are waiting on data rather than on agreement. Read with the Tx-Set Acquire Duration panel beside it — a rising p95 with outcomes still complete is slow-but-working, while the same p95 with timeouts is the set never arriving at all.*\n\n###### Keywords:\n- **Tx-set acquire** *(per node)* — one attempt to fetch the transaction set a consensus proposal referenced but this node did not hold.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Derived by the OTel Collector's spanmetrics connector from spans emitted by xrpld code; the collector counts the spans and their durations, and the Grafana query selects and aggregates the resulting series.*\n\n###### Source:\n[TransactionAcquire.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/TransactionAcquire.cpp)\n\n###### Function:\n`TransactionAcquire::finalizeAcquireSpan`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#tx-set-acquire)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Acquisitions / Sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, "pointSize": 3, "scaleDistribution": { "type": "linear" @@ -3710,9 +4509,9 @@ "h": 12, "w": 12, "x": 0, - "y": 235 + "y": 261 }, - "id": 34, + "id": 46, "options": { "annotations": { "clustering": -1, @@ -3740,11 +4539,11 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "label_replace(label_join(label_join(sum by (reason, request, service_instance_id, xrpl_branch, xrpl_work_item) (rate(serve_refused_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", reason=~\"$serve_reason\", request=~\"$serve_request\"}[$__rate_interval])), \"series\", \" \", \"reason\", \"request\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "expr": "label_replace(label_join(label_replace(sum by (outcome, service_instance_id, xrpl_branch, xrpl_work_item) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", outcome=~\"$span_outcome\", span_name=\"txset.acquire\"}[$__rate_interval])), \"series\", \"$1\", \"outcome\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", "refId": "A" } ], - "title": "Ledger/Object Serve Refusals", + "title": "Tx-Set Acquire Outcomes", "type": "timeseries" }, { @@ -3752,7 +4551,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*Rate at which this node abandoned the chain tip it had built and switched to a different last-closed ledger reported by the network.*\n\n###### How it's computed:\n*Per-second rate of last-closed-ledger switches where the ledger the network reported was not the one this node built on, per node. The counter carries no labels, so the sum is over one series per node.*\n\n###### Reading it:\n*Any non-zero value means this node was told the network's last-closed ledger is not the one it built on, and it discarded its own chain tip in response. A single jump around startup or a restart is ordinary recovery.*\n\n###### Healthy range:\n*0.*\n\n###### Watch for:\n*Repeated jumps — that is wrong-chain thrash, not one-off recovery. Check the peer set and the configured network id: a node peered to the wrong network, or into a minority partition, keeps being overruled and keeps throwing away work. Pair with the Bootstrap row and with Ledgers Behind Network.*\n\n###### Keywords:\n- **Byzantine ledger jump** *(per node)* — the node replaced its own last-closed ledger with a different one reported by the network, discarding the chain tip it had built.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`NetworkOPsImp::switchLastClosedLedger`\n\n###### References:\n[Fork](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#fork) · [Ledger history mismatch](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledger-history-mismatch)", + "description": "###### What this is:\n*95th-percentile wall time of one transaction-set acquisition, split by how it ended.*\n\n###### How it's computed:\n*95th percentile of the span-derived duration histogram for the txset.acquire span, grouped by outcome.*\n\n###### Reading it:\n*Splitting by outcome is what makes this readable: the complete series is how long a successful fetch takes, while the timeout series is pinned near the retry budget by construction and carries no information about speed.*\n\n###### Healthy range:\n*A complete-series p95 well under a consensus round interval.*\n\n###### Watch for:\n*A complete-series p95 approaching the round interval: sets are arriving, but so late that they delay the round they belong to. This is the case that a pure outcome rate cannot show, because those acquisitions do succeed.*\n\n###### Keywords:\n- **Tx-set acquire** *(per node)* — one attempt to fetch the transaction set a consensus proposal referenced but this node did not hold.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Derived by the OTel Collector's spanmetrics connector from spans emitted by xrpld code; the collector counts the spans and their durations, and the Grafana query selects and aggregates the resulting series.*\n\n###### Source:\n[TransactionAcquire.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/TransactionAcquire.cpp)\n\n###### Function:\n`TransactionAcquire::init`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#tx-set-acquire)", "fieldConfig": { "defaults": { "color": { @@ -3762,7 +4561,7 @@ "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "Jumps / Sec", + "axisLabel": "Duration", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, @@ -3789,7 +4588,7 @@ "mode": "none" }, "thresholdsStyle": { - "mode": "line" + "mode": "off" } }, "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", @@ -3799,23 +4598,19 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 0.001 } ] }, - "unit": "ops" + "unit": "ms" } }, "gridPos": { "h": 12, "w": 12, "x": 12, - "y": 235 + "y": 261 }, - "id": 35, + "id": 47, "options": { "annotations": { "clustering": -1, @@ -3843,1582 +4638,310 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_work_item) (rate(ledger_jump_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])), \"series\", \"Ledger Jumps\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, outcome, service_instance_id, xrpl_branch, xrpl_work_item) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", outcome=~\"$span_outcome\", span_name=\"txset.acquire\"}[$__rate_interval]))), \"series\", \"P95 $1\", \"outcome\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", "refId": "A" } ], - "title": "Byzantine Ledger Jumps", + "title": "Tx-Set Acquire Duration (p95)", "type": "timeseries" }, { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 247 + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - "id": 62, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "description": "###### What this is:\n*Rate of outbound peer dials reaching each terminal outcome, derived from the per-attempt peer.dial span.*\n\n###### How it's computed:\n*Rate of span-derived call counts for the peer.dial span, split by outcome. The same five outcome values the overlay_connect_total counter carries, set from the same funnel in the dial state machine so the two cannot disagree.*\n\n###### Reading it:\n*Read alongside Outbound Dial Outcome Rate in the Bootstrap row, which is the native counter for the same events. This panel exists for what the counter cannot do: each point here is backed by traces, so clicking through gives the individual attempt and the peer address it was dialling, which is never a metric label because one series per peer address would be unbounded.*\n\n###### Healthy range:\n*The connected series non-zero, failure series at or near zero.*\n\n###### Watch for:\n*A failure series dominating while connected stays at zero means the node has no outbound peers and cannot sync at all. Use the trace drill-down to find which endpoint keeps failing — the aggregate rate cannot tell one bad peer from a broken local network.*\n\n###### Keywords:\n- **Outbound dial** *(per node)* — one attempt by this node to open a peer connection, spanning the TCP connect, the TLS handshake and the protocol upgrade.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Derived by the OTel Collector's spanmetrics connector from spans emitted by xrpld code; the collector counts the spans and their durations, and the Grafana query selects and aggregates the resulting series.*\n\n###### Source:\n[ConnectAttempt.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/ConnectAttempt.cpp)\n\n###### Function:\n`ConnectAttempt::reportOutcome`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#outbound-dial-latency)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "description": "###### What this is:\n*Node-store write latency next to read latency, in microseconds per operation. The write side is the signal: a node with a large existing database back-fills slower than a fresh one, and back-fill is write-bound, so no read-side metric can show it.*\n\n###### How it's computed:\n*nodestore_latency series write_mean_us and read_mean_us, each divided by its own count series so the reading is the latency during the selected interval rather than the average since boot. The write numerator comes from a store-duration total that was declared but never written before this signal existed.*\n\n###### Reading it:\n*Compare the two lines. Reads far above writes points at the read path or a cold cache; writes far above reads points at backend write pressure, which is the large-existing-database case.*\n\n###### Healthy range:\n*Both well under a few hundred microseconds on healthy local storage.*\n\n###### Watch for:\n*A rising write line during history back-fill: the backend cannot absorb writes fast enough and sync will stay slow no matter how many peers are available. Read with Fetch-Pack Peer Starvation to tell a data-supply problem from a disk problem. This is a mean, not a percentile — a tail that matters will move it, but p99 is not available from this signal.*\n\n###### Keywords:\n- **Node-store write latency** *(per node)* — how long the node store takes to persist one object.\n- **Node-store read latency** *(per node)* — how long the node store takes to retrieve one object.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerNodeStoreLatencyGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#node-store-write-latency)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Microseconds / Op", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": 1800000, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "µs" + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Dials / Sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "gridPos": { - "h": 12, - "w": 12, - "x": 0, - "y": 248 + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] }, - "id": 38, - "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, - "legend": { - "calcs": [], - "displayMode": "list", - "enableFacetedFilter": false, - "overflow": "ellipsis", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "13.2.0-28926505616", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_work_item) (rate(nodestore_latency{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"write_duration_us\"}[$__rate_interval])) / clamp_min(sum by (service_instance_id, xrpl_branch, xrpl_work_item) (rate(nodestore_latency{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"write_count\"}[$__rate_interval])), 1), \"series\", \"Write us/op (interval)\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_work_item) (rate(nodestore_latency{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"read_duration_us\"}[$__rate_interval])) / clamp_min(sum by (service_instance_id, xrpl_branch, xrpl_work_item) (rate(nodestore_latency{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"read_count\"}[$__rate_interval])), 1), \"series\", \"Read us/op (interval)\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", - "refId": "B" - } - ], - "title": "NodeStore Write vs Read Latency (us/op)", - "type": "timeseries" + "unit": "ops" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 273 + }, + "id": 50, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "###### What this is:\n*Node-store write and read operation rates — the denominators behind the latency panel.*\n\n###### How it's computed:\n*Rate of the nodestore_latency write_count and read_count series.*\n\n###### Reading it:\n*Writes climb while a node is back-filling history and fall to near the ledger-close rate once it is caught up.*\n\n###### Healthy range:\n*Non-zero writes whenever the node is ingesting ledgers.*\n\n###### Watch for:\n*Write rate at zero while the node is still behind the network: nothing is being persisted, so the stall is upstream of the node store — check peer supply and the acquire panels rather than storage. A flat latency with a collapsing operation rate also means the latency figure above has gone stale rather than good.*\n\n###### Keywords:\n- **Node-store operation rate** *(per node)* — stores and fetches per second against the node store.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerNodeStoreLatencyGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#node-store-operation-rate)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Ops / Sec", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": 1800000, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "ops" - } - }, - "gridPos": { - "h": 12, - "w": 12, - "x": 12, - "y": 248 - }, - "id": 39, - "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, - "legend": { - "calcs": [], - "displayMode": "list", - "enableFacetedFilter": false, - "overflow": "ellipsis", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "13.2.0-28926505616", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_work_item) (rate(nodestore_latency{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"write_count\"}[$__rate_interval])), \"series\", \"Writes/s\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_work_item) (rate(nodestore_latency{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"read_count\"}[$__rate_interval])), \"series\", \"Reads/s\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", - "refId": "B" - } - ], - "title": "NodeStore Operation Rate (writes vs reads)", - "type": "timeseries" + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*Rate at which ledger-replay sub-tasks give up and fall back to acquiring a whole ledger, split by which sub-task gave up.*\n\n###### How it's computed:\n*Rate of ledger_replay_fallback_total by stage. The skip-list stage and the delta stage each emit once, on the transition into fallback, from the branch that was debug-log-only before.*\n\n###### Reading it:\n*Zero when replay-capable peers are available. Any sustained rate means the replay optimisation is being defeated and back-fill has reverted to the slower full-acquire path.*\n\n###### Healthy range:\n*Zero, or brief spikes while the peer set changes.*\n\n###### Watch for:\n*A persistent rate on either stage: too few connected peers support the ledger-replay feature, so every historical ledger is fetched in full instead of as a delta. Read with Replay Outcomes.*\n\n###### Keywords:\n- **Replay fallback** *(per node)* — a replay sub-task abandoning the delta shortcut and acquiring the entire ledger instead.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[SkipListAcquire.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/SkipListAcquire.cpp)\n\n###### Function:\n`SkipListAcquire::trigger` / `LedgerDeltaAcquire::trigger`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#replay-fallback)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Fallbacks / Sec", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": 1800000, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "ops" - } - }, - "gridPos": { - "h": 12, - "w": 12, - "x": 0, - "y": 260 - }, - "id": 40, - "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, - "legend": { - "calcs": [], - "displayMode": "list", - "enableFacetedFilter": false, - "overflow": "ellipsis", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "13.2.0-28926505616", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(sum by (stage, service_instance_id, xrpl_branch, xrpl_work_item) (rate(ledger_replay_fallback_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", stage=~\"$replay_stage\"}[$__rate_interval])), \"series\", \"$1\", \"stage\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", - "refId": "A" - } - ], - "title": "Replay Fallback to Full Acquire (by stage)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "###### What this is:\n*Rate of ledger-replay tasks reaching each terminal state: success, timeout, build failure or parameter failure.*\n\n###### How it's computed:\n*Rate of ledger_replay_outcome_total by outcome. Every terminal path in the replay task emits exactly once; before this, all four only set an internal flag and wrote a log line.*\n\n###### Reading it:\n*Successes only is healthy. Timeouts mean deltas never arrived; build failures mean a delta would not apply to its parent; parameter failures mean a peer served an inconsistent skip list.*\n\n###### Healthy range:\n*Successes non-zero while replaying, all failure outcomes at zero.*\n\n###### Watch for:\n*Any failure outcome climbing while successes stay flat: replay is running but never completing, so history back-fill is silently falling back to the slower path. The outcome value tells you which layer to look at — timeouts point at peers, build and parameter failures point at the data those peers served.*\n\n###### Keywords:\n- **Ledger replay** *(per node)* — rebuilding a range of historical ledgers from a start ledger plus per-ledger deltas instead of downloading each one whole.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[LedgerReplayTask.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/LedgerReplayTask.cpp)\n\n###### Function:\n`LedgerReplayTask::recordOutcome`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledger-replay)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Tasks / Sec", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": 1800000, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "ops" - } - }, - "gridPos": { - "h": 12, - "w": 12, - "x": 12, - "y": 260 - }, - "id": 41, - "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, - "legend": { - "calcs": [], - "displayMode": "list", - "enableFacetedFilter": false, - "overflow": "ellipsis", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "13.2.0-28926505616", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(sum by (outcome, service_instance_id, xrpl_branch, xrpl_work_item) (rate(ledger_replay_outcome_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", outcome=~\"$replay_outcome\"}[$__rate_interval])), \"series\", \"$1\", \"outcome\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", - "refId": "A" - } - ], - "title": "Replay Outcomes (by terminal state)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "###### What this is:\n*How long the heap trim that ends every cache sweep takes. The trim returns free heap pages to the kernel, and its cost scales with the resident heap — so this is the panel that shows a node with a large existing database paying a per-sweep penalty a fresh node never pays.*\n\n###### How it's computed:\n*p50 and p95 of the sweep_malloc_trim_us histogram. Recorded once per sweep, so the sample rate is one per sweep interval (10 s on a tiny node through 120 s on a huge one). The measurement used to run only at debug log level, so on an ordinary node there was no value at all.*\n\n###### Reading it:\n*Read the two lines together against the sweep interval. Sub-millisecond is free. Tens of milliseconds against a 10 s interval is still a small duty cycle but means the trim is walking a large heap, and it runs on the sweep job — so the cost lands on the job queue, not in the background.*\n\n###### Healthy range:\n*Under a millisecond on a warm node with a modest heap.*\n\n###### Watch for:\n*p95 climbing as the database grows, especially alongside a rising sweep-job queue wait. IMPORTANT LIMITATION: the fault counter beside this panel covers the trim call only. It proves the trim itself faults; it does NOT prove the trim causes the later faults taken as the caches refill. Correlate this duration against sweep-job queueing rather than concluding the trim caused a slow sync.*\n\n###### Keywords:\n- **Heap trim** *(per node)* — returning free heap pages from the allocator's arenas back to the kernel.\n- **Sweep interval** *(per node)* — how often the periodic cache sweep runs, set by node size.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[Application.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/main/Application.cpp)\n\n###### Function:\n`ApplicationImp::trimHeapAndRecord`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#heap-trim)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Trim Duration", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": 1800000, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "µs" - } - }, - "gridPos": { - "h": 12, - "w": 12, - "x": 0, - "y": 272 - }, - "id": 64, - "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, - "legend": { - "calcs": [], - "displayMode": "list", - "enableFacetedFilter": false, - "overflow": "ellipsis", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "13.2.0-28926505616", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(histogram_quantile(0.5, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(sweep_malloc_trim_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"P50 Trim\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(sweep_malloc_trim_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"P95 Trim\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", - "refId": "B" - } - ], - "title": "Sweep Heap-Trim Duration (p50/p95)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "###### What this is:\n*Two rates side by side: minor page faults taken inside the heap trim, and the resident memory the trim actually returned to the kernel. Together they say whether the trim is buying anything for what it costs.*\n\n###### How it's computed:\n*Rate of sweep_malloc_trim_minor_faults_total and of sweep_malloc_trim_reclaimed_kb_total. Both are cumulative counters exported per sweep, so the panel rates them rather than plotting the totals. Reclaim is published only when resident memory actually fell; a sweep during which another thread grew the heap contributes nothing rather than a negative amount.*\n\n###### Reading it:\n*Reclaim rate with a near-zero fault rate is a cheap, useful trim. A fault rate that moves with the reclaim rate means the pages are being handed back and immediately taken again, which is churn rather than savings.*\n\n###### Healthy range:\n*A reclaim rate that tracks cache turnover, with faults near zero.*\n\n###### Watch for:\n*Reclaim near zero while the trim duration panel shows real time being spent: the trim is walking the heap and freeing nothing, which is pure cost. IMPORTANT LIMITATION: the fault delta is scoped to the trim call, so it cannot show the re-fault cost paid later as the caches refill and touch the returned pages. That later cost is real but is NOT measured here; do not read a low fault rate as proof the trim was free.*\n\n###### Keywords:\n- **Minor page fault** *(per node)* — a memory access satisfied without disk I/O, by mapping a page the kernel already holds.\n- **Reclaimed resident memory** *(per node)* — resident kilobytes the allocator handed back to the kernel.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[Application.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/main/Application.cpp)\n\n###### Function:\n`ApplicationImp::trimHeapAndRecord`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#minor-page-fault)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Faults & KB / Sec", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": 1800000, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "cps" - } - }, - "gridPos": { - "h": 12, - "w": 12, - "x": 12, - "y": 272 - }, - "id": 65, - "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, - "legend": { - "calcs": [], - "displayMode": "list", - "enableFacetedFilter": false, - "overflow": "ellipsis", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "13.2.0-28926505616", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_work_item) (rate(sweep_malloc_trim_minor_faults_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])), \"series\", \"Minor Faults / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_work_item) (rate(sweep_malloc_trim_reclaimed_kb_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])), \"series\", \"Reclaimed KB / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", - "refId": "B" - } - ], - "title": "Sweep Heap-Trim Faults & Reclaim Rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "###### What this is:\n*When an online-delete rotation is running, and the rate of the extra writes it forces. A rotation rewrites into the new backend any node body the doomed archive serves, which is I/O an ordinary fetch would never perform — and it exists only on a populated, already-rotated database, which is why it never appears on a fresh node.*\n\n###### How it's computed:\n*rotation_state with metric=in_flight plotted raw (it is a 0/1 state flag), and metric=copy_forward — a cumulative write total — plotted as a rate. Both are read from the node store on each collection tick. The copy-forward count existed before as a log-only per-rotation tally that reset on every swap; the total behind this panel never resets, so it can be rated.*\n\n###### Reading it:\n*The two must move together: copy-forward writes should only appear while the window flag is 1. Read the write rate against the node-store write latency panel above — that is what tells extra rotation writes from a slow backend.*\n\n###### Healthy range:\n*Flag at 0 most of the time, rising to 1 briefly once per delete interval, with the write rate non-zero only inside those windows.*\n\n###### Watch for:\n*A copy-forward rate that is large enough to move node-store write latency: rotation is competing with sync I/O, which is the whole hypothesis this panel tests. Copy-forward writes while the flag reads 0 would mean the window flag leaked, not that rotation is cheap. NO SERIES AT ALL on either query means online_delete is not configured on this node, which is different from a rotation that costs nothing.*\n\n###### Keywords:\n- **Rotation window** *(per node)* — the interval during which an online-delete backend swap is in progress.\n- **Copy-forward write** *(per node)* — rewriting a node body from the backend about to be deleted into the one replacing it.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerRotationStateGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rotation-window)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "State & Writes / Sec", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": 1800000, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "cps" - } - }, - "gridPos": { - "h": 12, - "w": 12, - "x": 0, - "y": 284 - }, - "id": 66, - "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, - "legend": { - "calcs": [], - "displayMode": "list", - "enableFacetedFilter": false, - "overflow": "ellipsis", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "13.2.0-28926505616", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_work_item) (rotation_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"in_flight\"}), \"series\", \"Rotation In Flight (0/1)\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_work_item) (rate(rotation_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"copy_forward\"}[$__rate_interval])), \"series\", \"Copy-Forward Writes / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", - "refId": "B" - } - ], - "title": "Online-Delete Rotation Window & Copy-Forward Writes", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "###### What this is:\n*Rate at which a rotation has to rescue a tree node that was missing from BOTH backends and rewrite it from memory. Each one is an extra write on top of the whole-state-map walk the rotation already performs.*\n\n###### How it's computed:\n*Rate of rotation_copy_node_restore_total. Incremented once per rescued node in the rotation's state-map walk, from a branch that was warn-log-only before, so the volume was invisible unless someone was reading logs.*\n\n###### Reading it:\n*Zero on a healthy node. Any sustained rate means clean nodes reachable from the validated state map have no on-disk copy left, because the backend holding them was removed by an earlier rotation and they were never rewritten.*\n\n###### Healthy range:\n*Flat at zero.*\n\n###### Watch for:\n*A non-zero rate is the signal that earlier rotations dropped data the current state map still needs: each rescue is a write that competes with sync I/O, and without the rescue the node would later surface as an unresolvable missing-node error. Read with the copy-forward panel — both are rotation-time writes, but this one also indicates prior data loss rather than merely cost. The node hash is deliberately not a label (unbounded cardinality); get it from the copyNode warning in Loki, correlated by node and time.*\n\n###### Keywords:\n- **Node re-store** *(per node)* — rewriting an in-memory tree node whose only on-disk copy was removed by an earlier rotation.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[SHAMapStoreImp.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/SHAMapStoreImp.cpp)\n\n###### Function:\n`SHAMapStoreImp::copyNode`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#node-re-store)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Re-Stores / Sec", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": 1800000, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "ops" - } - }, - "gridPos": { - "h": 12, - "w": 12, - "x": 12, - "y": 284 - }, - "id": 67, - "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, - "legend": { - "calcs": [], - "displayMode": "list", - "enableFacetedFilter": false, - "overflow": "ellipsis", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "13.2.0-28926505616", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_work_item) (rate(rotation_copy_node_restore_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])), \"series\", \"Nodes Re-Stored / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", - "refId": "A" - } - ], - "title": "Rotation Node Re-Store Rate", - "type": "timeseries" + "expr": "label_replace(label_join(label_replace(sum by (outcome, service_instance_id, xrpl_branch, xrpl_work_item) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", outcome=~\"$span_outcome\", span_name=\"peer.dial\"}[$__rate_interval])), \"series\", \"$1\", \"outcome\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" } ], - "title": "Back-fill & persistence", - "type": "row" + "title": "Outbound Dial Outcomes (span-derived, per attempt)", + "type": "timeseries" }, { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 248 + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - "id": 63, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "description": "###### What this is:\n*Rate at which this node answers peers' ledger-data requests, split by what was asked for and how the reply ended.*\n\n###### How it's computed:\n*Rate of span-derived call counts for the ledger.serve span, split by object_type (header, transaction tree, account-state tree, or a proposed transaction set) and by outcome. The outcome is derived from the reply itself, so it is refused whenever nothing was sent.*\n\n###### Reading it:\n*This is the supply side — what this node does for its peers, not what it receives. The account-state series is the one that matters to a syncing peer, since that tree is the bulk of a fresh sync.*\n\n###### Healthy range:\n*Non-zero complete series on the types peers ask for, refused near zero.*\n\n###### Watch for:\n*A refused series climbing means this node is declining to serve; the paired serve_refused_total counter on Ledger/Object Serve Refusals gives the specific cause. A partial series means replies keep hitting the size cap, so peers must make repeated round trips for one tree. Note this panel does not explain this node's own sync — it explains its peers'.*\n\n###### Keywords:\n- **Ledger serve** *(per node)* — this node answering a peer's request for ledger data, as opposed to requesting data for itself.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Derived by the OTel Collector's spanmetrics connector from spans emitted by xrpld code; the collector counts the spans and their durations, and the Grafana query selects and aggregates the resulting series.*\n\n###### Source:\n[PeerImp.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/PeerImp.cpp)\n\n###### Function:\n`PeerImp::processLedgerRequest`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledger-serve)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "description": "###### What this is:\n*95th-percentile duration of each of the three phases of one ledger acquisition: the header wait, the account-state tree fetch, and the transaction tree fetch.*\n\n###### How it's computed:\n*95th percentile of the span-derived duration histogram for the three ledger.acquire.header / .astree / .txtree child spans, one series per phase. The parent ledger.acquire span is flat and cannot separate them, which matters because the account-state tree is nearly all of the work in a real fresh sync.*\n\n###### Reading it:\n*The astree series dominating is expected and healthy. The value of the split is the comparison: a header series that is large means the node is waiting to be told what to fetch, which is a peer-supply problem upstream of either tree.*\n\n###### Healthy range:\n*The astree series largest, the txtree series small, the header series near zero.*\n\n###### Watch for:\n*A hot astree band that keeps growing means account-state nodes are not being served — check Ledger Acquire Phase Outcomes beside it for timed_out, and Outbound Dial Outcome Rate in the Bootstrap row for refused or timed-out dials. A large header series instead means no peer is answering the header request at all.*\n\n###### Keywords:\n- **Acquire phase** *(per node)* — one of the three sequential fetches a ledger acquisition is made of; the header gates both trees, because it is what names their root hashes.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Derived by the OTel Collector's spanmetrics connector from spans emitted by xrpld code; the collector counts the spans and their durations, and the Grafana query selects and aggregates the resulting series.*\n\n###### Source:\n[InboundLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/InboundLedger.cpp)\n\n###### Function:\n`InboundLedger::syncPhaseSpans`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#acquire-phase)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Duration", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": 1800000, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "ms" + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Requests / Sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "gridPos": { - "h": 12, - "w": 12, - "x": 0, - "y": 249 + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] }, - "id": 48, - "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, - "legend": { - "calcs": [], - "displayMode": "list", - "enableFacetedFilter": false, - "overflow": "ellipsis", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "13.2.0-28926505616", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, span_name, service_instance_id, xrpl_branch, xrpl_work_item) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"ledger.acquire.header\"}[$__rate_interval]))), \"series\", \"P95 header\", \"span_name\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, span_name, service_instance_id, xrpl_branch, xrpl_work_item) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"ledger.acquire.astree\"}[$__rate_interval]))), \"series\", \"P95 account-state tree\", \"span_name\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, span_name, service_instance_id, xrpl_branch, xrpl_work_item) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"ledger.acquire.txtree\"}[$__rate_interval]))), \"series\", \"P95 transaction tree\", \"span_name\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", - "refId": "C" - } - ], - "title": "Ledger Acquire Phase Duration (p95 by phase)", - "type": "timeseries" + "unit": "ops" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 273 + }, + "id": 51, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "###### What this is:\n*Rate at which each ledger-acquire phase ends, split by which phase it was and whether it ran out of retry budget.*\n\n###### How it's computed:\n*Rate of span-derived call counts for the three ledger.acquire.* child spans, split by span name and by the timed_out flag. The flag is a separate dimension from the outcome because a phase can time out and still be retried by its parent acquire.*\n\n###### Reading it:\n*Series with timed_out=false are phases that finished on their own. Any series with timed_out=true is a phase whose retry budget expired, which names the exact stage of the fetch that peers are not serving.*\n\n###### Healthy range:\n*All series timed_out=false; zero everywhere on a node with no back-fill to do.*\n\n###### Watch for:\n*Any sustained timed_out=true rate. Pair it with the missing-node count on Missing SHAMap Nodes per Acquire: a timed-out phase together with a flat, non-zero missing-node count is the definitive stuck-sync signature, and the phase name says which tree to chase.*\n\n###### Keywords:\n- **Acquire phase timeout** *(per node)* — an acquire phase that ended because its retry budget expired rather than because its data arrived.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Derived by the OTel Collector's spanmetrics connector from spans emitted by xrpld code; the collector counts the spans and their durations, and the Grafana query selects and aggregates the resulting series.*\n\n###### Source:\n[InboundLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/InboundLedger.cpp)\n\n###### Function:\n`InboundLedger::endPhaseSpan`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#acquire-phase)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Phases / sec", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": 1800000, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "ops" - } - }, - "gridPos": { - "h": 12, - "w": 12, - "x": 12, - "y": 249 - }, - "id": 49, - "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, - "legend": { - "calcs": [], - "displayMode": "list", - "enableFacetedFilter": false, - "overflow": "ellipsis", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "13.2.0-28926505616", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_join(label_replace(sum by (span_name, outcome, service_instance_id, xrpl_branch, xrpl_work_item) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", outcome=~\"$span_outcome\", span_name=~\"ledger\\\\.acquire\\\\..*\"}[$__rate_interval])), \"phase\", \"$1\", \"span_name\", \"ledger\\\\.acquire\\\\.(.*)\"), \"series\", \" \", \"phase\", \"outcome\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", - "refId": "A" - } - ], - "title": "Ledger Acquire Phase Outcome Rate (by phase & outcome)", - "type": "timeseries" + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*Rate at which attempts to fetch a transaction set referenced by a consensus proposal reach each terminal state.*\n\n###### How it's computed:\n*Rate of span-derived call counts for the txset.acquire span, split by outcome. The span is new: transaction-set acquisition had no telemetry at all before, so a consensus round stalled waiting on a set looked identical to an idle one. Every exit stamps an outcome, including the sweep that drops a set which never arrived.*\n\n###### Reading it:\n*Only complete is healthy. Timeout means peers never supplied the set. Abandoned means the round moved on and the set was dropped mid-fetch. Failed means a peer served data that would not build.*\n\n###### Healthy range:\n*A low rate of complete during consensus, everything else at zero. Zero everywhere is also normal on a node that already holds every proposed set locally.*\n\n###### Watch for:\n*Timeout or abandoned climbing: proposed sets are not arriving, so rounds are waiting on data rather than on agreement. Read with the Tx-Set Acquire Duration panel beside it — a rising p95 with outcomes still complete is slow-but-working, while the same p95 with timeouts is the set never arriving at all.*\n\n###### Keywords:\n- **Tx-set acquire** *(per node)* — one attempt to fetch the transaction set a consensus proposal referenced but this node did not hold.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Derived by the OTel Collector's spanmetrics connector from spans emitted by xrpld code; the collector counts the spans and their durations, and the Grafana query selects and aggregates the resulting series.*\n\n###### Source:\n[TransactionAcquire.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/TransactionAcquire.cpp)\n\n###### Function:\n`TransactionAcquire::finalizeAcquireSpan`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#tx-set-acquire)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Acquisitions / Sec", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": 1800000, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "ops" - } - }, - "gridPos": { - "h": 12, - "w": 12, - "x": 0, - "y": 261 - }, - "id": 46, - "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, - "legend": { - "calcs": [], - "displayMode": "list", - "enableFacetedFilter": false, - "overflow": "ellipsis", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "13.2.0-28926505616", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(sum by (outcome, service_instance_id, xrpl_branch, xrpl_work_item) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", outcome=~\"$span_outcome\", span_name=\"txset.acquire\"}[$__rate_interval])), \"series\", \"$1\", \"outcome\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", - "refId": "A" - } - ], - "title": "Tx-Set Acquire Outcomes", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "###### What this is:\n*95th-percentile wall time of one transaction-set acquisition, split by how it ended.*\n\n###### How it's computed:\n*95th percentile of the span-derived duration histogram for the txset.acquire span, grouped by outcome.*\n\n###### Reading it:\n*Splitting by outcome is what makes this readable: the complete series is how long a successful fetch takes, while the timeout series is pinned near the retry budget by construction and carries no information about speed.*\n\n###### Healthy range:\n*A complete-series p95 well under a consensus round interval.*\n\n###### Watch for:\n*A complete-series p95 approaching the round interval: sets are arriving, but so late that they delay the round they belong to. This is the case that a pure outcome rate cannot show, because those acquisitions do succeed.*\n\n###### Keywords:\n- **Tx-set acquire** *(per node)* — one attempt to fetch the transaction set a consensus proposal referenced but this node did not hold.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Derived by the OTel Collector's spanmetrics connector from spans emitted by xrpld code; the collector counts the spans and their durations, and the Grafana query selects and aggregates the resulting series.*\n\n###### Source:\n[TransactionAcquire.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/TransactionAcquire.cpp)\n\n###### Function:\n`TransactionAcquire::init`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#tx-set-acquire)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Duration", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": 1800000, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "ms" - } - }, - "gridPos": { - "h": 12, - "w": 12, - "x": 12, - "y": 261 - }, - "id": 47, - "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, - "legend": { - "calcs": [], - "displayMode": "list", - "enableFacetedFilter": false, - "overflow": "ellipsis", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "13.2.0-28926505616", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, outcome, service_instance_id, xrpl_branch, xrpl_work_item) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", outcome=~\"$span_outcome\", span_name=\"txset.acquire\"}[$__rate_interval]))), \"series\", \"P95 $1\", \"outcome\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", - "refId": "A" - } - ], - "title": "Tx-Set Acquire Duration (p95)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "###### What this is:\n*Rate of outbound peer dials reaching each terminal outcome, derived from the per-attempt peer.dial span.*\n\n###### How it's computed:\n*Rate of span-derived call counts for the peer.dial span, split by outcome. The same five outcome values the overlay_connect_total counter carries, set from the same funnel in the dial state machine so the two cannot disagree.*\n\n###### Reading it:\n*Read alongside Outbound Dial Outcome Rate in the Bootstrap row, which is the native counter for the same events. This panel exists for what the counter cannot do: each point here is backed by traces, so clicking through gives the individual attempt and the peer address it was dialling, which is never a metric label because one series per peer address would be unbounded.*\n\n###### Healthy range:\n*The connected series non-zero, failure series at or near zero.*\n\n###### Watch for:\n*A failure series dominating while connected stays at zero means the node has no outbound peers and cannot sync at all. Use the trace drill-down to find which endpoint keeps failing — the aggregate rate cannot tell one bad peer from a broken local network.*\n\n###### Keywords:\n- **Outbound dial** *(per node)* — one attempt by this node to open a peer connection, spanning the TCP connect, the TLS handshake and the protocol upgrade.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Derived by the OTel Collector's spanmetrics connector from spans emitted by xrpld code; the collector counts the spans and their durations, and the Grafana query selects and aggregates the resulting series.*\n\n###### Source:\n[ConnectAttempt.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/ConnectAttempt.cpp)\n\n###### Function:\n`ConnectAttempt::reportOutcome`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#outbound-dial-latency)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Dials / Sec", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": 1800000, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "ops" - } - }, - "gridPos": { - "h": 12, - "w": 12, - "x": 0, - "y": 273 - }, - "id": 50, - "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, - "legend": { - "calcs": [], - "displayMode": "list", - "enableFacetedFilter": false, - "overflow": "ellipsis", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "13.2.0-28926505616", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(sum by (outcome, service_instance_id, xrpl_branch, xrpl_work_item) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", outcome=~\"$span_outcome\", span_name=\"peer.dial\"}[$__rate_interval])), \"series\", \"$1\", \"outcome\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", - "refId": "A" - } - ], - "title": "Outbound Dial Outcomes (span-derived, per attempt)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "###### What this is:\n*Rate at which this node answers peers' ledger-data requests, split by what was asked for and how the reply ended.*\n\n###### How it's computed:\n*Rate of span-derived call counts for the ledger.serve span, split by object_type (header, transaction tree, account-state tree, or a proposed transaction set) and by outcome. The outcome is derived from the reply itself, so it is refused whenever nothing was sent.*\n\n###### Reading it:\n*This is the supply side — what this node does for its peers, not what it receives. The account-state series is the one that matters to a syncing peer, since that tree is the bulk of a fresh sync.*\n\n###### Healthy range:\n*Non-zero complete series on the types peers ask for, refused near zero.*\n\n###### Watch for:\n*A refused series climbing means this node is declining to serve; the paired serve_refused_total counter on Ledger/Object Serve Refusals gives the specific cause. A partial series means replies keep hitting the size cap, so peers must make repeated round trips for one tree. Note this panel does not explain this node's own sync — it explains its peers'.*\n\n###### Keywords:\n- **Ledger serve** *(per node)* — this node answering a peer's request for ledger data, as opposed to requesting data for itself.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Derived by the OTel Collector's spanmetrics connector from spans emitted by xrpld code; the collector counts the spans and their durations, and the Grafana query selects and aggregates the resulting series.*\n\n###### Source:\n[PeerImp.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/PeerImp.cpp)\n\n###### Function:\n`PeerImp::processLedgerRequest`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledger-serve)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Requests / Sec", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": 1800000, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "ops" - } - }, - "gridPos": { - "h": 12, - "w": 12, - "x": 12, - "y": 273 - }, - "id": 51, - "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, - "legend": { - "calcs": [], - "displayMode": "list", - "enableFacetedFilter": false, - "overflow": "ellipsis", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "13.2.0-28926505616", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_join(sum by (object_type, outcome, service_instance_id, xrpl_branch, xrpl_work_item) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", object_type=~\"$object_type\", outcome=~\"$span_outcome\", span_name=\"ledger.serve\"}[$__rate_interval])), \"series\", \" \", \"object_type\", \"outcome\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", - "refId": "A" - } - ], - "title": "Ledger Serve Rate by Object Type", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "###### What this is:\n*Rate of trusted validations reaching the ledger-acceptance gate, split by what the validation store did with each one and by whether it actually reached the gate.*\n\n###### How it's computed:\n*Rate of span-derived calls for the consensus.validation.accept span, by validation_status and accept_gated. The span is emitted once per trusted validation and carries the trace id of the ledger it validates, so any point here can be opened as the full trace for that ledger — the validation, the acceptance decision it drove, and the acquire and store spans for the same ledger.*\n\n###### Reading it:\n*Nearly all of the rate should be validation_status=current, which is the only status that continues to the gate. accept_gated=true means another thread was already accepting that ledger, so no acceptance followed this validation; a modest share is normal when validations for one ledger arrive together.*\n\n###### Healthy range:\n*Dominated by current, at roughly the trusted-validator count per ledger close.*\n\n###### Watch for:\n*Rate concentrated in stale, bad_seq, multiple or conflicting: validations are arriving and being counted for nothing, which is the difference between a node that is slow to validate and one that never will — from the outside the two look identical. Confirm on Trusted Validations vs Quorum Target and Pre-Accept Quorum Shortfall Rate, and check UNL Quorum Headroom in the Bootstrap row first, since a trusted list that cannot satisfy quorum starves everything below it. A flat zero on a peered node means no trusted validations are arriving at all.*\n\n###### Keywords:\n- **Validation status** *(per node)* — what this node's validation store did with an arriving validation; only current counts toward accepting a ledger.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Derived by the OTel Collector's spanmetrics connector from spans emitted by xrpld code; the collector counts the spans, and the Grafana query selects and aggregates the resulting series.*\n\n###### Source:\n[RCLValidations.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/RCLValidations.cpp)\n\n###### Function:\n`handleNewValidation`\n\n###### References:\n[Negative UNL and validation quorum on xrpl.org](https://xrpl.org/docs/concepts/consensus-protocol/negative-unl) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#validation-status)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Validations / Sec", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": 1800000, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 12, - "w": 24, - "x": 0, - "y": 285 - }, - "id": 54, - "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, - "legend": { - "calcs": [], - "displayMode": "list", - "enableFacetedFilter": false, - "overflow": "ellipsis", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "13.2.0-28926505616", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_join(sum by (validation_status, accept_gated, service_instance_id, xrpl_branch, xrpl_work_item) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"consensus.validation.accept\", validation_status=~\"$validation_status\"}[$__rate_interval])), \"series\", \" gated=\", \"validation_status\", \"accept_gated\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", - "refId": "A" - } - ], - "title": "Trusted Validation Accept Rate by Status", - "type": "timeseries" + "expr": "label_replace(label_join(label_join(sum by (object_type, outcome, service_instance_id, xrpl_branch, xrpl_work_item) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", object_type=~\"$object_type\", outcome=~\"$span_outcome\", span_name=\"ledger.serve\"}[$__rate_interval])), \"series\", \" \", \"object_type\", \"outcome\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" } ], - "title": "Spans & traces", - "type": "row" + "title": "Ledger Serve Rate by Object Type", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Rate of trusted validations reaching the ledger-acceptance gate, split by what the validation store did with each one and by whether it actually reached the gate.*\n\n###### How it's computed:\n*Rate of span-derived calls for the consensus.validation.accept span, by validation_status and accept_gated. The span is emitted once per trusted validation and carries the trace id of the ledger it validates, so any point here can be opened as the full trace for that ledger — the validation, the acceptance decision it drove, and the acquire and store spans for the same ledger.*\n\n###### Reading it:\n*Nearly all of the rate should be validation_status=current, which is the only status that continues to the gate. accept_gated=true means another thread was already accepting that ledger, so no acceptance followed this validation; a modest share is normal when validations for one ledger arrive together.*\n\n###### Healthy range:\n*Dominated by current, at roughly the trusted-validator count per ledger close.*\n\n###### Watch for:\n*Rate concentrated in stale, bad_seq, multiple or conflicting: validations are arriving and being counted for nothing, which is the difference between a node that is slow to validate and one that never will — from the outside the two look identical. Confirm on Trusted Validations vs Quorum Target and Pre-Accept Quorum Shortfall Rate, and check UNL Quorum Headroom in the Bootstrap row first, since a trusted list that cannot satisfy quorum starves everything below it. A flat zero on a peered node means no trusted validations are arriving at all.*\n\n###### Keywords:\n- **Validation status** *(per node)* — what this node's validation store did with an arriving validation; only current counts toward accepting a ledger.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Derived by the OTel Collector's spanmetrics connector from spans emitted by xrpld code; the collector counts the spans, and the Grafana query selects and aggregates the resulting series.*\n\n###### Source:\n[RCLValidations.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/RCLValidations.cpp)\n\n###### Function:\n`handleNewValidation`\n\n###### References:\n[Negative UNL and validation quorum on xrpl.org](https://xrpl.org/docs/concepts/consensus-protocol/negative-unl) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#validation-status)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Validations / Sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 285 + }, + "id": 54, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_join(sum by (validation_status, accept_gated, service_instance_id, xrpl_branch, xrpl_work_item) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"consensus.validation.accept\", validation_status=~\"$validation_status\"}[$__rate_interval])), \"series\", \" Gated=\", \"validation_status\", \"accept_gated\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Trusted Validation Accept Rate by Status", + "type": "timeseries" } ], "schemaVersion": 39,