diff --git a/docker/telemetry/grafana/dashboards/ledger-sync-health.json b/docker/telemetry/grafana/dashboards/ledger-sync-health.json index d6dec81f37..013d6944da 100644 --- a/docker/telemetry/grafana/dashboards/ledger-sync-health.json +++ b/docker/telemetry/grafana/dashboards/ledger-sync-health.json @@ -57,7 +57,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*Count of peer-hostname DNS resolutions, split by outcome (resolved or empty).*\n\n###### How it's computed:\n*Count over the selected range of completed resolutions grouped by outcome, per node.*\n\n###### Reading it:\n*Resolved should account for every attempt; the empty line should stay flat at zero.*\n\n###### Healthy range:\n*A short burst of resolved at startup, then flat. Non-zero empty is always a defect.*\n\n###### Watch for:\n*Any empty rate means a configured bootstrap or [ips_fixed] hostname returned no address, so the node never even tries to dial that peer.*\n\n###### Keywords:\n- **DNS resolve** *(per node)* — turning a configured peer hostname into IP addresses before any dial is attempted; `outcome=empty` means the name resolved to nothing.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl::reportDnsResolve`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#dns-resolve)", + "description": "###### What this is:\n*Count of peer-hostname DNS resolutions, split by outcome (resolved or empty).*\n\n###### How it's computed:\n*Count over the selected range of completed resolutions grouped by outcome, per node.*\n\n###### Reading it:\n*Resolved should account for every attempt; the empty line should stay flat at zero.*\n\n###### Healthy range:\n*A short burst of resolved at startup, then flat. Non-zero empty is always a defect.*\n\n###### Watch for:\n*Any non-zero empty count means a configured bootstrap or [ips_fixed] hostname returned no address, so the node never even tries to dial that peer.*\n\n###### Keywords:\n- **DNS resolve** *(per node)* — turning a configured peer hostname into IP addresses before any dial is attempted; `outcome=empty` means the name resolved to nothing.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl::reportDnsResolve`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#dns-resolve)", "fieldConfig": { "defaults": { "color": { @@ -114,7 +114,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "label_replace(label_join(label_replace(sum by (outcome, service_instance_id, xrpl_branch, xrpl_work_item) (round(increase(dns_resolve_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__range]))), \"series\", \"$1\", \"outcome\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "expr": "label_replace(label_join(label_replace(sum by (outcome, service_instance_id, xrpl_branch, xrpl_work_item) (last_over_time(dns_resolve_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__range])), \"series\", \"$1\", \"outcome\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", "refId": "A", "instant": true } @@ -287,7 +287,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "label_replace(label_join(label_replace(sum by (outcome, service_instance_id, xrpl_branch, xrpl_work_item) (round(increase(overlay_connect_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", outcome=~\"$dial_outcome\"}[$__range]))), \"series\", \"$1\", \"outcome\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "expr": "label_replace(label_join(label_replace(sum by (outcome, service_instance_id, xrpl_branch, xrpl_work_item) (last_over_time(overlay_connect_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", outcome=~\"$dial_outcome\"}[$__range])), \"series\", \"$1\", \"outcome\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", "refId": "A", "instant": true } @@ -403,7 +403,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*Count of peer handshakes rejected during protocol negotiation, split by reason.*\n\n###### How it's computed:\n*Count over the selected range of rejected handshakes grouped by reason, per node. Filter the reason set with the Handshake Reason variable.*\n\n###### Reading it:\n*Flat at zero is healthy. The reason names the exact check that rejected the peer, so one dominant reason is the fault to fix.*\n\n###### Healthy range:\n*Zero, or a low background rate of self_connection and remote_ip_mismatch on a NAT'd host.*\n\n###### Watch for:\n*wrong_network or invalid_network_id — the node is configured for a different network than its peers and will never reach a quorum. clock_skew points at the local clock; session_verify_failed and bad_public_key at a misbehaving peer.*\n\n###### Keywords:\n- **Handshake negotiation failure** *(per node)* — a peer connection rejected after TLS while checking network id, clock, keys and addresses; the reason label names the failing check.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[Handshake.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/Handshake.cpp)\n\n###### Function:\n`throwNegotiationFailure`\n\n###### References:\n[Peer protocol on xrpl.org](https://xrpl.org/docs/concepts/networks-and-servers/peer-protocol) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#handshake-negotiation-failure)", + "description": "###### What this is:\n*Count of peer handshakes rejected during protocol negotiation, split by reason.*\n\n###### How it's computed:\n*Count over the selected range of rejected handshakes grouped by reason, per node. Filter the reason set with the Handshake Reason variable.*\n\n###### Reading it:\n*Flat at zero is healthy. The reason names the exact check that rejected the peer, so one dominant reason is the fault to fix.*\n\n###### Healthy range:\n*Zero, or a low background count of self_connection and remote_ip_mismatch on a NAT'd host.*\n\n###### Watch for:\n*wrong_network or invalid_network_id — the node is configured for a different network than its peers and will never reach a quorum. clock_skew points at the local clock; session_verify_failed and bad_public_key at a misbehaving peer.*\n\n###### Keywords:\n- **Handshake negotiation failure** *(per node)* — a peer connection rejected after TLS while checking network id, clock, keys and addresses; the reason label names the failing check.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[Handshake.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/Handshake.cpp)\n\n###### Function:\n`throwNegotiationFailure`\n\n###### References:\n[Peer protocol on xrpl.org](https://xrpl.org/docs/concepts/networks-and-servers/peer-protocol) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#handshake-negotiation-failure)", "fieldConfig": { "defaults": { "color": { @@ -460,7 +460,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "label_replace(label_join(label_replace(sum by (reason, service_instance_id, xrpl_branch, xrpl_work_item) (round(increase(handshake_negotiation_fail_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", reason=~\"$handshake_reason\"}[$__range]))), \"series\", \"$1\", \"reason\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "expr": "label_replace(label_join(label_replace(sum by (reason, service_instance_id, xrpl_branch, xrpl_work_item) (last_over_time(handshake_negotiation_fail_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", reason=~\"$handshake_reason\"}[$__range])), \"series\", \"$1\", \"reason\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", "refId": "A", "instant": true } @@ -473,7 +473,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*Count of validator-list fetches from each configured UNL site, split by outcome.*\n\n###### How it's computed:\n*Count over the selected range of fetch attempts grouped by site and outcome, per node. Filter with the UNL Site and UNL Fetch Outcome variables.*\n\n###### Reading it:\n*accepted is the only success value. same_sequence and known_sequence are normal no-op refreshes of a list the node already holds. fetch_error, bad_status and parse_error are transport or content faults; expired, stale, untrusted, invalid and unsupported_version mean the list was retrieved but rejected.*\n\n###### Healthy range:\n*A first accepted per site at startup, then a steady low rate of same_sequence refreshes.*\n\n###### Watch for:\n*A site with only fetch_error or bad_status is unreachable. Only expired or invalid means the site is reachable but its list is unusable, so no trusted keys are loaded from it.*\n\n###### Keywords:\n- **UNL fetch outcome** *(per node)* — the result of retrieving and applying a validator list from a configured site; `accepted` is the only success.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[ValidatorSite.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/detail/ValidatorSite.cpp)\n\n###### Function:\n`ValidatorSite::reportFetchOutcome`\n\n###### References:\n[UNL (Unique Node List)](https://xrpl.org/docs/concepts/consensus-protocol/unl) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#unl-fetch-outcome)", + "description": "###### What this is:\n*Count of validator-list fetches from each configured UNL site, split by outcome.*\n\n###### How it's computed:\n*Count over the selected range of fetch attempts grouped by site and outcome, per node. Filter with the UNL Site and UNL Fetch Outcome variables.*\n\n###### Reading it:\n*accepted is the only success value. same_sequence and known_sequence are normal no-op refreshes of a list the node already holds. fetch_error, bad_status and parse_error are transport or content faults; expired, stale, untrusted, invalid and unsupported_version mean the list was retrieved but rejected.*\n\n###### Healthy range:\n*A first accepted per site at startup, then a steady low count of same_sequence refreshes.*\n\n###### Watch for:\n*A site with only fetch_error or bad_status is unreachable. Only expired or invalid means the site is reachable but its list is unusable, so no trusted keys are loaded from it.*\n\n###### Keywords:\n- **UNL fetch outcome** *(per node)* — the result of retrieving and applying a validator list from a configured site; `accepted` is the only success.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[ValidatorSite.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/detail/ValidatorSite.cpp)\n\n###### Function:\n`ValidatorSite::reportFetchOutcome`\n\n###### References:\n[UNL (Unique Node List)](https://xrpl.org/docs/concepts/consensus-protocol/unl) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#unl-fetch-outcome)", "fieldConfig": { "defaults": { "color": { @@ -530,7 +530,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "label_replace(label_join(label_join(sum by (outcome, site, service_instance_id, xrpl_branch, xrpl_work_item) (round(increase(unl_fetch_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", site=~\"$unl_site\", outcome=~\"$unl_outcome\"}[$__range]))), \"series\", \" \", \"outcome\", \"site\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "expr": "label_replace(label_join(label_join(sum by (outcome, site, service_instance_id, xrpl_branch, xrpl_work_item) (last_over_time(unl_fetch_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", site=~\"$unl_site\", outcome=~\"$unl_outcome\"}[$__range])), \"series\", \" \", \"outcome\", \"site\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", "refId": "A", "instant": true } @@ -654,7 +654,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*Spare trusted validator keys above the required quorum — the single number that says whether this node can ever validate.*\n\n###### How it's computed:\n*Trusted key count minus the required quorum, matched per node.*\n\n###### Reading it:\n*Positive is healthy. Zero or negative (red) means the trusted UNL is too small to ever satisfy quorum, so the node will stay short of a validated ledger.*\n\n###### Healthy range:\n*Positive; the exact figure depends on UNL size and the configured quorum.*\n\n###### Watch for:\n*Zero or below. Pair it with UNL Fetch Rate by Site & Outcome: a site stuck on fetch_error or expired is the usual cause of a UNL too small to meet quorum.*\n\n###### Keywords:\n- **UNL quorum headroom** *(per node)* — trusted UNL key count minus the required quorum; at or below zero the node can never declare a ledger validated.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerUnlQuorumGauge`\n\n###### References:\n[Validation quorum on xrpl.org](https://xrpl.org/docs/concepts/consensus-protocol/negative-unl) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#unl-quorum-headroom)", + "description": "###### What this is:\n*Spare trusted validator keys above the required quorum — the single number that says whether this node can ever validate.*\n\n###### How it's computed:\n*Trusted key count minus the required quorum, matched per node.*\n\n###### Reading it:\n*Positive is healthy. Zero or negative (red) means the trusted UNL is too small to ever satisfy quorum, so the node will stay short of a validated ledger.*\n\n###### Healthy range:\n*Positive; the exact figure depends on UNL size and the configured quorum.*\n\n###### Watch for:\n*Zero or below. Pair it with UNL Fetch Outcomes (Count By Site & Outcome): a site stuck on fetch_error or expired is the usual cause of a UNL too small to meet quorum.*\n\n###### Keywords:\n- **UNL quorum headroom** *(per node)* — trusted UNL key count minus the required quorum; at or below zero the node can never declare a ledger validated.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerUnlQuorumGauge`\n\n###### References:\n[Validation quorum on xrpl.org](https://xrpl.org/docs/concepts/consensus-protocol/negative-unl) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#unl-quorum-headroom)", "fieldConfig": { "defaults": { "color": { @@ -1285,7 +1285,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*Count of inbound peer connection handoffs, split by terminal outcome.*\n\n###### How it's computed:\n*Count over the selected range of finished inbound handoffs grouped by outcome, per node. Filter the outcome set with the Accept Outcome variable.*\n\n###### Reading it:\n*accepted should dominate. The failure outcomes name what rejected the connection: resource_limit and no_slot are this node's own capacity, protocol_mismatch and bad_cookie are the peer or the network identity, handshake_error and local_endpoint_fail are the transport.*\n\n###### Healthy range:\n*accepted dominant, failures near zero.*\n\n###### Watch for:\n*Read this together with Outbound Dial Outcome Rate above, which is the outbound twin on overlay_connect_total{outcome}. The two share the same one-outcome-per-attempt shape, so together they give the full in/out split: a node that accepts nothing but dials successfully has a very different fault from one that can neither dial nor accept. A steady no_slot or resource_limit is this node refusing peers it has no room for, which is capacity rather than a fault.*\n\n###### Keywords:\n- **Inbound peer accept outcome** *(per node)* — the terminal result of an inbound peer connection handoff; each handoff ends in exactly one outcome.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl::onHandoff`\n\n###### References:\n[Overlay](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#overlay) · [Outbound dial latency](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#outbound-dial-latency)", + "description": "###### What this is:\n*Count of inbound peer connection handoffs, split by terminal outcome.*\n\n###### How it's computed:\n*Count over the selected range of finished inbound handoffs grouped by outcome, per node. Filter the outcome set with the Accept Outcome variable.*\n\n###### Reading it:\n*accepted should dominate. The failure outcomes name what rejected the connection: resource_limit and no_slot are this node's own capacity, protocol_mismatch and bad_cookie are the peer or the network identity, handshake_error and local_endpoint_fail are the transport.*\n\n###### Healthy range:\n*accepted dominant, failures near zero.*\n\n###### Watch for:\n*Read this together with Outbound Dial Outcomes (Count By Outcome) above, which is the outbound twin on overlay_connect_total{outcome}. The two share the same one-outcome-per-attempt shape, so together they give the full in/out split: a node that accepts nothing but dials successfully has a very different fault from one that can neither dial nor accept. A steady no_slot or resource_limit is this node refusing peers it has no room for, which is capacity rather than a fault.*\n\n###### Keywords:\n- **Inbound peer accept outcome** *(per node)* — the terminal result of an inbound peer connection handoff; each handoff ends in exactly one outcome.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl::onHandoff`\n\n###### References:\n[Overlay](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#overlay) · [Outbound dial latency](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#outbound-dial-latency)", "fieldConfig": { "defaults": { "color": { @@ -1342,7 +1342,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "label_replace(label_join(label_replace(sum by (outcome, service_instance_id, xrpl_branch, xrpl_work_item) (round(increase(peer_accept_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", outcome=~\"$accept_outcome\"}[$__range]))), \"series\", \"$1\", \"outcome\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "expr": "label_replace(label_join(label_replace(sum by (outcome, service_instance_id, xrpl_branch, xrpl_work_item) (last_over_time(peer_accept_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", outcome=~\"$accept_outcome\"}[$__range])), \"series\", \"$1\", \"outcome\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", "refId": "A", "instant": true } @@ -1355,7 +1355,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*Rate at which peer connections are closed, split by why they closed and by which side opened them.*\n\n###### How it's computed:\n*Count over the selected range of peer closes grouped by reason and direction (inbound or outbound), per node. Filter with the Disconnect Reason and Disconnect Direction variables.*\n\n###### Reading it:\n*The split separates our-fault backpressure from topology and network faults. large_sendq and charge_resources are this node shedding a peer it cannot keep up with. not_useful and ping_timeout are topology and liveness. read_error and write_error are the transport. graceful, stopping and shutdown are ordinary lifecycle, not faults.*\n\n###### Healthy range:\n*Mostly graceful; the fault reasons near zero.*\n\n###### Watch for:\n*A sustained large_sendq or charge_resources means this node is the bottleneck and is dropping peers — which removes the very peers it needs to sync from, so a slow node makes itself slower. A rising not_useful or ping_timeout on outbound points at the peer set instead. Distinct from the single unlabelled peer-disconnect total on the Node Health dashboard, which cannot say which of these is happening.*\n\n###### Keywords:\n- **Peer disconnect reason** *(per node)* — the cause recorded when a peer connection is closed, paired with whether that connection was inbound or outbound.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[PeerImp.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/PeerImp.cpp)\n\n###### Function:\n`PeerImp::close`\n\n###### References:\n[Resource disconnect](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#resource-disconnect) · [Insane / diverged peers](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#insane-diverged-peers)", + "description": "###### What this is:\n*Number of times peer connections are closed, split by why they closed and by which side opened them.*\n\n###### How it's computed:\n*Count over the selected range of peer closes grouped by reason and direction (inbound or outbound), per node. Filter with the Disconnect Reason and Disconnect Direction variables.*\n\n###### Reading it:\n*The split separates our-fault backpressure from topology and network faults. large_sendq and charge_resources are this node shedding a peer it cannot keep up with. not_useful and ping_timeout are topology and liveness. read_error and write_error are the transport. graceful, stopping and shutdown are ordinary lifecycle, not faults.*\n\n###### Healthy range:\n*Mostly graceful; the fault reasons near zero.*\n\n###### Watch for:\n*A sustained large_sendq or charge_resources means this node is the bottleneck and is dropping peers — which removes the very peers it needs to sync from, so a slow node makes itself slower. A rising not_useful or ping_timeout on outbound points at the peer set instead. Distinct from the single unlabelled peer-disconnect total on the Node Health dashboard, which cannot say which of these is happening.*\n\n###### Keywords:\n- **Peer disconnect reason** *(per node)* — the cause recorded when a peer connection is closed, paired with whether that connection was inbound or outbound.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[PeerImp.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/PeerImp.cpp)\n\n###### Function:\n`PeerImp::close`\n\n###### References:\n[Resource disconnect](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#resource-disconnect) · [Insane / diverged peers](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#insane-diverged-peers)", "fieldConfig": { "defaults": { "color": { @@ -1412,7 +1412,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "label_replace(label_join(label_join(sum by (reason, direction, service_instance_id, xrpl_branch, xrpl_work_item) (round(increase(peer_disconnect_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", reason=~\"$disconnect_reason\", direction=~\"$disconnect_direction\"}[$__range]))), \"series\", \" \", \"reason\", \"direction\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "expr": "label_replace(label_join(label_join(sum by (reason, direction, service_instance_id, xrpl_branch, xrpl_work_item) (last_over_time(peer_disconnect_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", reason=~\"$disconnect_reason\", direction=~\"$disconnect_direction\"}[$__range])), \"series\", \" \", \"reason\", \"direction\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", "refId": "A", "instant": true } @@ -1737,7 +1737,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*How often the server enters a NEW stall episode, as opposed to how long one stall lasts.*\n\n###### How it's computed:\n*Count over the selected range of server_stall_events_total, which counts once per stall episode rather than once per stalled second.*\n\n###### Reading it:\n*Flat at zero is healthy. Read it beside the Server Stall stat: a rising rate means repeated fresh stalls, while a flat rate with a large stall value means one long unresolved stall.*\n\n###### Healthy range:\n*0.*\n\n###### Watch for:\n*Any repeating rate. Recurring short stalls and one long stall have different causes, and this panel is what separates them.*\n\n###### Keywords:\n- **Server stall** *(per node)* — the main loop failing to check in with the load monitor, measured in seconds of unresponsiveness.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[LoadManager.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/main/LoadManager.cpp)\n\n###### Function:\n`LoadManager::updateStallState`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#server-stall)", + "description": "###### What this is:\n*How often the server enters a NEW stall episode, as opposed to how long one stall lasts.*\n\n###### How it's computed:\n*Count over the selected range of server_stall_events_total, which counts once per stall episode rather than once per stalled second.*\n\n###### Reading it:\n*Flat at zero is healthy. Read it beside the Server Stall stat: a rising count means repeated fresh stalls, while a flat count with a large stall value means one long unresolved stall.*\n\n###### Healthy range:\n*0.*\n\n###### Watch for:\n*Any repeating count. Recurring short stalls and one long stall have different causes, and this panel is what separates them.*\n\n###### Keywords:\n- **Server stall** *(per node)* — the main loop failing to check in with the load monitor, measured in seconds of unresponsiveness.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[LoadManager.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/main/LoadManager.cpp)\n\n###### Function:\n`LoadManager::updateStallState`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#server-stall)", "fieldConfig": { "defaults": { "color": { @@ -1794,7 +1794,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "label_replace(label_join(label_replace(round(increase(server_stall_events_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__range])), \"series\", \"Stall Episodes\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "expr": "label_replace(label_join(label_replace(last_over_time(server_stall_events_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__range]), \"series\", \"Stall Episodes\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", "refId": "A", "instant": true } @@ -1887,7 +1887,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*Outstanding SHAMap nodes the busiest in-flight ledger acquire still needs, split by tree. This is the signal that separates a sync that is merely slow from one that will never finish.*\n\n###### How it's computed:\n*sync_acquire series missing_state_nodes_max and missing_tx_nodes_max: the largest outstanding node count across all in-flight acquires, refreshed after each getMissingNodes sweep. The maximum, not the sum, so one stuck acquire stays visible instead of being averaged away.*\n\n###### Reading it:\n*Falling toward zero means the acquire is progressing. A value pinned at 256 is the sweep cap, meaning there are at least that many nodes outstanding. Zero on one tree with a value on the other means that tree is already complete.*\n\n###### Healthy range:\n*Falling to 0 within seconds per ledger.*\n\n###### Watch for:\n*A flat, non-zero value across several minutes: no peer is serving that tree, so this acquire will never complete. Pair with Acquire Stall Rate — both flat and climbing together is the definitive stuck-sync signature.*\n\n###### Keywords:\n- **Missing SHAMap node** *(per node)* — a tree node this node needs to complete a ledger but does not yet hold.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerSyncAcquireGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#missing-shamap-node)", + "description": "###### What this is:\n*Outstanding SHAMap nodes the busiest in-flight ledger acquire still needs, split by tree. This is the signal that separates a sync that is merely slow from one that will never finish.*\n\n###### How it's computed:\n*sync_acquire series missing_state_nodes_max and missing_tx_nodes_max: the largest outstanding node count across all in-flight acquires, refreshed after each getMissingNodes sweep. The maximum, not the sum, so one stuck acquire stays visible instead of being averaged away.*\n\n###### Reading it:\n*Falling toward zero means the acquire is progressing. A value pinned at 256 is the sweep cap, meaning there are at least that many nodes outstanding. Zero on one tree with a value on the other means that tree is already complete.*\n\n###### Healthy range:\n*Falling to 0 within seconds per ledger.*\n\n###### Watch for:\n*A flat, non-zero value across several minutes: no peer is serving that tree, so this acquire will never complete. Pair with Acquire Stalls — No Progress (Count) — both flat and climbing together is the definitive stuck-sync signature.*\n\n###### Keywords:\n- **Missing SHAMap node** *(per node)* — a tree node this node needs to complete a ledger but does not yet hold.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerSyncAcquireGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#missing-shamap-node)", "fieldConfig": { "defaults": { "color": { @@ -1982,7 +1982,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "label_replace(label_join(label_replace(sync_acquire{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=~\"$acquire_metric\", metric=~\"missing_(state|tx)_nodes_max\"}, \"series\", \"$1 tree\", \"metric\", \"missing_(.*)_nodes_max\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "expr": "label_replace(label_join(label_replace(sync_acquire{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=~\"missing_(state|tx)_nodes_max\"}, \"series\", \"$1 tree\", \"metric\", \"missing_(.*)_nodes_max\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", "refId": "A" } ], @@ -1994,7 +1994,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*Count of ledger-acquire timeouts where not a single new node arrived since the previous timeout.*\n\n###### How it's computed:\n*rate of sync_acquire_no_progress_total, incremented on each acquire timeout whose progress flag was false. The acquire timer fires every 3 seconds at most.*\n\n###### Reading it:\n*Zero means every timeout window saw at least some new data. Any sustained rate means acquires are repeatedly timing out with nothing received.*\n\n###### Healthy range:\n*0 on a synced node; brief non-zero bursts during initial sync are normal.*\n\n###### Watch for:\n*A sustained rate together with a flat Missing SHAMap Nodes panel: the node is asking and no peer is answering. Check peer count and whether any peer holds the ledger range being requested.*\n\n###### Keywords:\n- **Acquire stall** *(per node)* — an acquire timeout in which no new SHAMap node was received, so the acquire made no progress at all.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (call-site metric macro, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[InboundLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/InboundLedger.cpp)\n\n###### Function:\n`InboundLedger::onTimer`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#acquire-stall)", + "description": "###### What this is:\n*Count of ledger-acquire timeouts where not a single new node arrived since the previous timeout.*\n\n###### How it's computed:\n*count of sync_acquire_no_progress_total, incremented on each acquire timeout whose progress flag was false. The acquire timer fires every 3 seconds at most.*\n\n###### Reading it:\n*Zero means every timeout window saw at least some new data. Any sustained count means acquires are repeatedly timing out with nothing received.*\n\n###### Healthy range:\n*0 on a synced node; brief non-zero bursts during initial sync are normal.*\n\n###### Watch for:\n*A sustained count together with a flat Missing SHAMap Nodes panel: the node is asking and no peer is answering. Check peer count and whether any peer holds the ledger range being requested.*\n\n###### Keywords:\n- **Acquire stall** *(per node)* — an acquire timeout in which no new SHAMap node was received, so the acquire made no progress at all.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (call-site metric macro, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[InboundLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/InboundLedger.cpp)\n\n###### Function:\n`InboundLedger::onTimer`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#acquire-stall)", "fieldConfig": { "defaults": { "color": { @@ -2051,7 +2051,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "label_replace(label_join(label_replace(round(increase(sync_acquire_no_progress_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__range])), \"series\", \"Stalled Timeouts\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "expr": "label_replace(label_join(label_replace(last_over_time(sync_acquire_no_progress_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__range]), \"series\", \"Stalled Timeouts\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", "refId": "A", "instant": true } @@ -2200,7 +2200,7 @@ "mode": "none" }, "thresholdsStyle": { - "mode": "line" + "mode": "off" } }, "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", @@ -2210,15 +2210,43 @@ { "color": "green", "value": null - }, - { - "color": "yellow", - "value": 16 } ] }, "unit": "short" - } + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*received_data_depth.*" + }, + "properties": [ + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 16 + } + ] + } + }, + { + "id": "custom.thresholdsStyle", + "value": { + "mode": "line" + } + } + ] + } + ] }, "gridPos": { "h": 12, @@ -2254,7 +2282,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "label_replace(label_join(label_replace(sync_acquire{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=~\"$acquire_metric\", metric=~\"received_data_depth|in_flight\"}, \"series\", \"$1\", \"metric\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "expr": "label_replace(label_join(label_replace(sync_acquire{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=~\"received_data_depth|in_flight\"}, \"series\", \"$1\", \"metric\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", "refId": "A" } ], @@ -2266,7 +2294,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*Count of ledger acquires split by whether the local node store already held the whole ledger or the data had to come from peers.*\n\n###### How it's computed:\n*rate of sync_acquire_source_total by source, emitted once per new acquire right after the first local-store lookup.*\n\n###### Reading it:\n*network dominant during initial sync is expected — nothing is local yet. local dominant on a warm node means the store is serving requests without peer traffic.*\n\n###### Healthy range:\n*Mostly local on a warm node with complete history.*\n\n###### Watch for:\n*Sustained network on a node that should already hold the range: the local store is not retaining data, so sync is disk-bound rather than peer-bound. Read with the SHAMap cache hit-rate panel.*\n\n###### Keywords:\n- **Acquire source** *(per node)* — whether a ledger acquire was satisfied entirely from the local node store or required fetching from peers.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (call-site metric macro, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[InboundLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/InboundLedger.cpp)\n\n###### Function:\n`InboundLedger::init`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#acquire-source)", + "description": "###### What this is:\n*Count of ledger acquires split by whether the local node store already held the whole ledger or the data had to come from peers.*\n\n###### How it's computed:\n*count of sync_acquire_source_total by source, emitted once per new acquire right after the first local-store lookup.*\n\n###### Reading it:\n*network dominant during initial sync is expected — nothing is local yet. local dominant on a warm node means the store is serving requests without peer traffic.*\n\n###### Healthy range:\n*Mostly local on a warm node with complete history.*\n\n###### Watch for:\n*Sustained network on a node that should already hold the range: the local store is not retaining data, so sync is disk-bound rather than peer-bound. Read with the SHAMap cache hit-rate panel.*\n\n###### Keywords:\n- **Acquire source** *(per node)* — whether a ledger acquire was satisfied entirely from the local node store or required fetching from peers.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (call-site metric macro, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[InboundLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/InboundLedger.cpp)\n\n###### Function:\n`InboundLedger::init`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#acquire-source)", "fieldConfig": { "defaults": { "color": { @@ -2323,7 +2351,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "label_replace(label_join(label_replace(sum by (source, service_instance_id, xrpl_branch, xrpl_work_item) (round(increase(sync_acquire_source_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", source=~\"$acquire_source\"}[$__range]))), \"series\", \"$1\", \"source\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "expr": "label_replace(label_join(label_replace(sum by (source, service_instance_id, xrpl_branch, xrpl_work_item) (last_over_time(sync_acquire_source_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", source=~\"$acquire_source\"}[$__range])), \"series\", \"$1\", \"source\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", "refId": "A", "instant": true } @@ -2456,7 +2484,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*Share of the worker-thread pool currently executing a job. This is the pool-wide view: when the pool itself is exhausted, every subsystem queued behind it looks independently slow, and this panel attributes the whole slowdown once.*\n\n###### How it's computed:\n*jobq_saturation running_tasks divided by worker_threads (the denominator is clamped to at least 1). The thread count is exported rather than hardcoded because it is derived at startup from [workers], node size and hardware concurrency.*\n\n###### Reading it:\n*Below 80% (green) means the pool has spare capacity, so a slow stage is that stage's own fault. At 100% every worker is busy — read Total Jobs Queued next: 100% with a queue is an exhausted pool, 100% with an empty queue is merely busy.*\n\n###### Healthy range:\n*< 80%.*\n\n###### Watch for:\n*A sustained 100% together with a non-zero backlog. Every job type is then starved by the pool, so fix pool capacity or the long-running jobs holding it, not the individual victim subsystems.*\n\n###### Keywords:\n- **Worker-pool saturation** *(per node)* — worker threads executing a job as a share of the threads the pool is configured to run.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerJobQueueSaturationGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#worker-pool-saturation)", + "description": "###### What this is:\n*Share of the worker-thread pool currently executing a job. This is the pool-wide view: when the pool itself is exhausted, every subsystem queued behind it looks independently slow, and this panel attributes the whole slowdown once.*\n\n###### How it's computed:\n*jobq_saturation running_tasks divided by worker_threads (the denominator is clamped to at least 1). The thread count is exported rather than hardcoded because it is derived at startup from [workers], node size and hardware concurrency.*\n\n###### Reading it:\n*Below 80% (green) means the pool has spare capacity, so a slow stage is that stage's own fault. At 100% every worker is busy — read Worker Pool Capacity & Total Backlog next: 100% with a queue is an exhausted pool, 100% with an empty queue is merely busy.*\n\n###### Healthy range:\n*< 80%.*\n\n###### Watch for:\n*A sustained 100% together with a non-zero backlog. Every job type is then starved by the pool, so fix pool capacity or the long-running jobs holding it, not the individual victim subsystems.*\n\n###### Keywords:\n- **Worker-pool saturation** *(per node)* — worker threads executing a job as a share of the threads the pool is configured to run.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerJobQueueSaturationGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#worker-pool-saturation)", "fieldConfig": { "defaults": { "color": { @@ -2743,7 +2771,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*Rate at which the pre-accept gate refused to declare a candidate ledger validated because its trusted validations were below quorum.*\n\n###### How it's computed:\n*Count of ledger_quorum_shortfall_total by stage. One increment per rejected gate evaluation, emitted from the early return that was trace-log-only before, so a node that peers and receives validations yet never validates is no longer indistinguishable from an idle one. The gate is re-entered on every fresh trusted validation for the candidate ledger, so one ledger that eventually validates can contribute several increments on its way there.*\n\n###### Reading it:\n*A steady low rate is NORMAL and is not a fault. The gate is evaluated the instant this node finishes building a ledger, before its peers' validations for that ledger have arrived, so the first evaluation of each round routinely tallies short and is retried as validations come in. Read this against the rate of ledger closes and against Trusted Validations vs Quorum Target — it is the ratio and the accompanying tally that carry the signal, never the bare presence of a rate.*\n\n###### Healthy range:\n*A low steady rate on the order of one per ledger close or less, on a node that is validating.*\n\n###### Watch for:\n*A rate that climbs well above the ledger-close rate while Publish Lag grows and Time to First Validated Ledger stays at zero — that combination is the retry loop never converging, so the tally is not merely early, it never reaches the target. Confirm on Trusted Validations vs Quorum Target: a climbing tally is slow and will finish, a flat tally below the target is stuck (too few trusted validators reachable, or a UNL / negative-UNL misconfiguration). Check UNL Quorum Headroom in the Bootstrap row before anything else in this row, because a trusted list that cannot satisfy quorum makes every panel below it look starved.*\n\n###### Keywords:\n- **Quorum shortfall** *(per node)* — trusted validations for a candidate ledger falling short of the quorum needed to declare it validated, so the node holds the ledger and still cannot call it validated.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (call-site metric macro, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[LedgerMaster.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/LedgerMaster.cpp)\n\n###### Function:\n`LedgerMaster::checkAccept`\n\n###### References:\n[Negative UNL and validation quorum on xrpl.org](https://xrpl.org/docs/concepts/consensus-protocol/negative-unl) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#quorum-shortfall)", + "description": "###### What this is:\n*Number of times the pre-accept gate refused to declare a candidate ledger validated because its trusted validations were below quorum.*\n\n###### How it's computed:\n*Count of ledger_quorum_shortfall_total by stage. One increment per rejected gate evaluation, emitted from the early return that was trace-log-only before, so a node that peers and receives validations yet never validates is no longer indistinguishable from an idle one. The gate is re-entered on every fresh trusted validation for the candidate ledger, so one ledger that eventually validates can contribute several increments on its way there.*\n\n###### Reading it:\n*A steady low count is NORMAL and is not a fault. The gate is evaluated the instant this node finishes building a ledger, before its peers' validations for that ledger have arrived, so the first evaluation of each round routinely tallies short and is retried as validations come in. Read this against the count of ledger closes and against Trusted Validations vs Quorum Target — it is the ratio and the accompanying tally that carry the signal, never the bare presence of a count.*\n\n###### Healthy range:\n*A low steady count, on the order of one per ledger close or less, on a node that is validating.*\n\n###### Watch for:\n*A count that climbs well above one per ledger close while Publish Lag grows and Time to First Validated Ledger stays at zero — that combination is the retry loop never converging, so the tally is not merely early, it never reaches the target. Confirm on Trusted Validations vs Quorum Target: a climbing tally is slow and will finish, a flat tally below the target is stuck (too few trusted validators reachable, or a UNL / negative-UNL misconfiguration). Check UNL Quorum Headroom in the Bootstrap row before anything else in this row, because a trusted list that cannot satisfy quorum makes every panel below it look starved.*\n\n###### Keywords:\n- **Quorum shortfall** *(per node)* — trusted validations for a candidate ledger falling short of the quorum needed to declare it validated, so the node holds the ledger and still cannot call it validated.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (call-site metric macro, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[LedgerMaster.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/LedgerMaster.cpp)\n\n###### Function:\n`LedgerMaster::checkAccept`\n\n###### References:\n[Negative UNL and validation quorum on xrpl.org](https://xrpl.org/docs/concepts/consensus-protocol/negative-unl) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#quorum-shortfall)", "fieldConfig": { "defaults": { "color": { @@ -2800,7 +2828,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "label_replace(label_join(label_replace(sum by (stage, service_instance_id, xrpl_branch, xrpl_work_item) (round(increase(ledger_quorum_shortfall_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", stage=~\"$shortfall_stage\"}[$__range]))), \"series\", \"$1\", \"stage\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "expr": "label_replace(label_join(label_replace(sum by (stage, service_instance_id, xrpl_branch, xrpl_work_item) (last_over_time(ledger_quorum_shortfall_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", stage=~\"$shortfall_stage\"}[$__range])), \"series\", \"$1\", \"stage\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", "refId": "A", "instant": true } @@ -3413,7 +3441,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "label_replace(label_join(label_join(sum by (reason, request, service_instance_id, xrpl_branch, xrpl_work_item) (round(increase(serve_refused_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", reason=~\"$serve_reason\", request=~\"$serve_request\"}[$__range]))), \"series\", \" \", \"reason\", \"request\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "expr": "label_replace(label_join(label_join(sum by (reason, request, service_instance_id, xrpl_branch, xrpl_work_item) (last_over_time(serve_refused_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", reason=~\"$serve_reason\", request=~\"$serve_request\"}[$__range])), \"series\", \" \", \"reason\", \"request\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", "refId": "A", "instant": true } @@ -3426,7 +3454,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*Rate at which this node abandoned the chain tip it had built and switched to a different last-closed ledger reported by the network.*\n\n###### How it's computed:\n*Count over the selected range of last-closed-ledger switches where the ledger the network reported was not the one this node built on, per node. The counter carries no labels, so the sum is over one series per node.*\n\n###### Reading it:\n*Any non-zero value means this node was told the network's last-closed ledger is not the one it built on, and it discarded its own chain tip in response. A single jump around startup or a restart is ordinary recovery.*\n\n###### Healthy range:\n*0.*\n\n###### Watch for:\n*Repeated jumps — that is wrong-chain thrash, not one-off recovery. Check the peer set and the configured network id: a node peered to the wrong network, or into a minority partition, keeps being overruled and keeps throwing away work. Pair with the Bootstrap row and with Ledgers Behind Network.*\n\n###### Keywords:\n- **Byzantine ledger jump** *(per node)* — the node replaced its own last-closed ledger with a different one reported by the network, discarding the chain tip it had built.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`NetworkOPsImp::switchLastClosedLedger`\n\n###### References:\n[Fork](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#fork) · [Ledger history mismatch](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledger-history-mismatch)", + "description": "###### What this is:\n*Number of times this node abandoned the chain tip it had built and switched to a different last-closed ledger reported by the network.*\n\n###### How it's computed:\n*Count over the selected range of last-closed-ledger switches where the ledger the network reported was not the one this node built on, per node. The counter carries no labels, so the sum is over one series per node.*\n\n###### Reading it:\n*Any non-zero value means this node was told the network's last-closed ledger is not the one it built on, and it discarded its own chain tip in response. A single jump around startup or a restart is ordinary recovery.*\n\n###### Healthy range:\n*0.*\n\n###### Watch for:\n*Repeated jumps — that is wrong-chain thrash, not one-off recovery. Check the peer set and the configured network id: a node peered to the wrong network, or into a minority partition, keeps being overruled and keeps throwing away work. Pair with the Bootstrap row and with Ledgers Behind Network.*\n\n###### Keywords:\n- **Byzantine ledger jump** *(per node)* — the node replaced its own last-closed ledger with a different one reported by the network, discarding the chain tip it had built.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`NetworkOPsImp::switchLastClosedLedger`\n\n###### References:\n[Fork](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#fork) · [Ledger history mismatch](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledger-history-mismatch)", "fieldConfig": { "defaults": { "color": { @@ -3483,7 +3511,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_work_item) (round(increase(ledger_jump_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__range]))), \"series\", \"Ledger Jumps\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_work_item) (last_over_time(ledger_jump_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__range])), \"series\", \"Ledger Jumps\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", "refId": "A", "instant": true } @@ -3509,7 +3537,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*Node-store write latency next to read latency, in microseconds per operation. The write side is the signal: a node with a large existing database back-fills slower than a fresh one, and back-fill is write-bound, so no read-side metric can show it.*\n\n###### How it's computed:\n*nodestore_state series node_writes_duration_us and node_reads_duration_us, each divided by its own count series (node_writes, node_reads_total) so the reading is the latency during the selected interval rather than the average since boot. All three concrete store paths time themselves, so the write side is live on an ordinary node.*\n\n###### Reading it:\n*Compare the two lines. Reads far above writes points at the read path or a cold cache; writes far above reads points at backend write pressure, which is the large-existing-database case.*\n\n###### Healthy range:\n*Both well under a few hundred microseconds on healthy local storage.*\n\n###### Watch for:\n*A rising write line during history back-fill: the backend cannot absorb writes fast enough and sync will stay slow no matter how many peers are available. Read with Fetch-Pack Peer Starvation to tell a data-supply problem from a disk problem. This is a mean, not a percentile — a tail that matters will move it, but p99 is not available from this signal.*\n\n###### Keywords:\n- **Node-store write latency** *(per node)* — how long the node store takes to persist one object.\n- **Node-store read latency** *(per node)* — how long the node store takes to retrieve one object.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerNodeStoreGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#node-store-write-latency)", + "description": "###### What this is:\n*Node-store write latency next to read latency, in microseconds per operation. The write side is the signal: a node with a large existing database back-fills slower than a fresh one, and back-fill is write-bound, so no read-side metric can show it.*\n\n###### How it's computed:\n*nodestore_state series node_writes_duration_us and node_reads_duration_us, each divided by its own count series (node_writes, node_reads_total) so the reading is the latency during the selected interval rather than the average since boot. All three concrete store paths time themselves, so the write side is live on an ordinary node.*\n\n###### Reading it:\n*Compare the two lines. Reads far above writes points at the read path or a cold cache; writes far above reads points at backend write pressure, which is the large-existing-database case.*\n\n###### Healthy range:\n*Both well under a few hundred microseconds on healthy local storage.*\n\n###### Watch for:\n*A rising write line during history back-fill: the backend cannot absorb writes fast enough and sync will stay slow no matter how many peers are available. Read with Peers Able to Serve Needed Sequence to tell a data-supply problem from a disk problem. This is a mean, not a percentile — a tail that matters will move it, but p99 is not available from this signal.*\n\n###### Keywords:\n- **Node-store write latency** *(per node)* — how long the node store takes to persist one object.\n- **Node-store read latency** *(per node)* — how long the node store takes to retrieve one object.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerNodeStoreGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#node-store-write-latency)", "fieldConfig": { "defaults": { "color": { @@ -3723,569 +3751,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*Rate at which ledger-replay sub-tasks give up and fall back to acquiring a whole ledger, split by which sub-task gave up.*\n\n###### How it's computed:\n*Count of ledger_replay_fallback_total by stage. The skip-list stage and the delta stage each emit once, on the transition into fallback, from the branch that was debug-log-only before.*\n\n###### Reading it:\n*Zero when replay-capable peers are available. Any sustained rate means the replay optimisation is being defeated and back-fill has reverted to the slower full-acquire path.*\n\n###### Healthy range:\n*Zero, or brief spikes while the peer set changes.*\n\n###### Watch for:\n*A persistent rate on either stage: too few connected peers support the ledger-replay feature, so every historical ledger is fetched in full instead of as a delta. Read with Replay Outcomes.*\n\n###### Keywords:\n- **Replay fallback** *(per node)* — a replay sub-task abandoning the delta shortcut and acquiring the entire ledger instead.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[SkipListAcquire.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/SkipListAcquire.cpp)\n\n###### Function:\n`SkipListAcquire::trigger` / `LedgerDeltaAcquire::trigger`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#replay-fallback)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "displayName": "Replay Fallback [${__field.labels.series}] ${__field.labels.xrpl_ident}", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short", - "decimals": 0 - }, - "overrides": [] - }, - "gridPos": { - "h": 12, - "w": 12, - "x": 0, - "y": 260 - }, - "id": 40, - "options": { - "displayMode": "gradient", - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "maxVizHeight": 300, - "minVizHeight": 16, - "minVizWidth": 8, - "namePlacement": "left", - "orientation": "horizontal", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showUnfilled": true, - "sizing": "manual", - "valueMode": "color" - }, - "pluginVersion": "13.2.0-28926505616", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(sum by (stage, service_instance_id, xrpl_branch, xrpl_work_item) (round(increase(ledger_replay_fallback_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", stage=~\"$replay_stage\"}[$__range]))), \"series\", \"$1\", \"stage\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", - "refId": "A", - "instant": true - } - ], - "title": "Replay Fallback To Full Acquire (Count By Stage)", - "type": "bargauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "###### What this is:\n*Count of ledger-replay tasks reaching each terminal state: success, timeout, build failure or parameter failure.*\n\n###### How it's computed:\n*Count of ledger_replay_outcome_total by outcome. Every terminal path in the replay task emits exactly once; before this, all four only set an internal flag and wrote a log line.*\n\n###### Reading it:\n*Successes only is healthy. Timeouts mean deltas never arrived; build failures mean a delta would not apply to its parent; parameter failures mean a peer served an inconsistent skip list.*\n\n###### Healthy range:\n*Successes non-zero while replaying, all failure outcomes at zero.*\n\n###### Watch for:\n*Any failure outcome climbing while successes stay flat: replay is running but never completing, so history back-fill is silently falling back to the slower path. The outcome value tells you which layer to look at — timeouts point at peers, build and parameter failures point at the data those peers served.*\n\n###### Keywords:\n- **Ledger replay** *(per node)* — rebuilding a range of historical ledgers from a start ledger plus per-ledger deltas instead of downloading each one whole.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[LedgerReplayTask.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/LedgerReplayTask.cpp)\n\n###### Function:\n`LedgerReplayTask::recordOutcome`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledger-replay)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "displayName": "Replay Outcome [${__field.labels.series}] ${__field.labels.xrpl_ident}", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short", - "decimals": 0 - }, - "overrides": [] - }, - "gridPos": { - "h": 12, - "w": 12, - "x": 12, - "y": 260 - }, - "id": 41, - "options": { - "displayMode": "gradient", - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "maxVizHeight": 300, - "minVizHeight": 16, - "minVizWidth": 8, - "namePlacement": "left", - "orientation": "horizontal", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showUnfilled": true, - "sizing": "manual", - "valueMode": "color" - }, - "pluginVersion": "13.2.0-28926505616", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(sum by (outcome, service_instance_id, xrpl_branch, xrpl_work_item) (round(increase(ledger_replay_outcome_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", outcome=~\"$replay_outcome\"}[$__range]))), \"series\", \"$1\", \"outcome\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", - "refId": "A", - "instant": true - } - ], - "title": "Replay Outcomes (Count By Terminal State)", - "type": "bargauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "###### What this is:\n*How long the heap trim that ends every cache sweep takes. The trim returns free heap pages to the kernel, and its cost scales with the resident heap — so this is the panel that shows a node with a large existing database paying a per-sweep penalty a fresh node never pays.*\n\n###### How it's computed:\n*p50 and p95 of the sweep_malloc_trim_us histogram. Recorded once per sweep, so the sample rate is one per sweep interval (10 s on a tiny node through 120 s on a huge one). The measurement used to run only at debug log level, so on an ordinary node there was no value at all.*\n\n###### Reading it:\n*Read the two lines together against the sweep interval. Sub-millisecond is free. Tens of milliseconds against a 10 s interval is still a small duty cycle but means the trim is walking a large heap, and it runs on the sweep job — so the cost lands on the job queue, not in the background.*\n\n###### Healthy range:\n*Under a millisecond on a warm node with a modest heap.*\n\n###### Watch for:\n*p95 climbing as the database grows, especially alongside a rising sweep-job queue wait. IMPORTANT LIMITATION: the fault counter beside this panel covers the trim call only. It proves the trim itself faults; it does NOT prove the trim causes the later faults taken as the caches refill. Correlate this duration against sweep-job queueing rather than concluding the trim caused a slow sync.*\n\n###### Keywords:\n- **Heap trim** *(per node)* — returning free heap pages from the allocator's arenas back to the kernel.\n- **Sweep interval** *(per node)* — how often the periodic cache sweep runs, set by node size.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[Application.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/main/Application.cpp)\n\n###### Function:\n`ApplicationImp::trimHeapAndRecord`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#heap-trim)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Trim Duration", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": 1800000, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "µs" - } - }, - "gridPos": { - "h": 12, - "w": 12, - "x": 0, - "y": 272 - }, - "id": 64, - "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, - "legend": { - "calcs": [], - "displayMode": "list", - "enableFacetedFilter": false, - "overflow": "ellipsis", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "13.2.0-28926505616", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(histogram_quantile(0.5, sum by (le, service_instance_id, xrpl_branch, xrpl_work_item) (rate(sweep_malloc_trim_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"P50 Trim\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_work_item) (rate(sweep_malloc_trim_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"P95 Trim\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", - "refId": "B" - } - ], - "title": "Sweep Heap-Trim Duration (p50/p95)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "###### What this is:\n*Two rates side by side: minor page faults taken inside the heap trim, and the resident memory the trim actually returned to the kernel. Together they say whether the trim is buying anything for what it costs.*\n\n###### How it's computed:\n*Rate of sweep_malloc_trim_minor_faults_total and of sweep_malloc_trim_reclaimed_kb_total. Both are cumulative counters exported per sweep, so the panel rates them rather than plotting the totals. Reclaim is published only when resident memory actually fell; a sweep during which another thread grew the heap contributes nothing rather than a negative amount.*\n\n###### Reading it:\n*Reclaim rate with a near-zero fault rate is a cheap, useful trim. A fault rate that moves with the reclaim rate means the pages are being handed back and immediately taken again, which is churn rather than savings.*\n\n###### Healthy range:\n*A reclaim rate that tracks cache turnover, with faults near zero.*\n\n###### Watch for:\n*Reclaim near zero while the trim duration panel shows real time being spent: the trim is walking the heap and freeing nothing, which is pure cost. IMPORTANT LIMITATION: the fault delta is scoped to the trim call, so it cannot show the re-fault cost paid later as the caches refill and touch the returned pages. That later cost is real but is NOT measured here; do not read a low fault rate as proof the trim was free.*\n\n###### Keywords:\n- **Minor page fault** *(per node)* — a memory access satisfied without disk I/O, by mapping a page the kernel already holds.\n- **Reclaimed resident memory** *(per node)* — resident kilobytes the allocator handed back to the kernel.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[Application.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/main/Application.cpp)\n\n###### Function:\n`ApplicationImp::trimHeapAndRecord`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#minor-page-fault)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short", - "decimals": 0 - }, - "overrides": [] - }, - "gridPos": { - "h": 12, - "w": 12, - "x": 12, - "y": 272 - }, - "id": 65, - "options": { - "displayMode": "gradient", - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "maxVizHeight": 300, - "minVizHeight": 16, - "minVizWidth": 8, - "namePlacement": "left", - "orientation": "horizontal", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showUnfilled": true, - "sizing": "manual", - "valueMode": "color" - }, - "pluginVersion": "13.2.0-28926505616", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_work_item) (round(increase(sweep_malloc_trim_minor_faults_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__range]))), \"series\", \"Minor Faults\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", - "refId": "A", - "instant": true - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_work_item) (round(increase(sweep_malloc_trim_reclaimed_kb_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__range]))), \"series\", \"Reclaimed KB\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", - "refId": "B", - "instant": true - } - ], - "title": "Sweep Heap-Trim Faults & Reclaim (Count)", - "type": "bargauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "###### What this is:\n*When an online-delete rotation is running, and the rate of the extra writes it forces. A rotation rewrites into the new backend any node body the doomed archive serves, which is I/O an ordinary fetch would never perform — and it exists only on a populated, already-rotated database, which is why it never appears on a fresh node.*\n\n###### How it's computed:\n*rotation_state with metric=in_flight plotted raw (it is a 0/1 state flag), and metric=copy_forward — a cumulative write total — plotted as a rate. Both are read from the node store on each collection tick. The copy-forward count existed before as a log-only per-rotation tally that reset on every swap; the total behind this panel never resets, so it can be rated.*\n\n###### Reading it:\n*The two must move together: copy-forward writes should only appear while the window flag is 1. Read the write rate against the node-store write latency panel above — that is what tells extra rotation writes from a slow backend.*\n\n###### Healthy range:\n*Flag at 0 most of the time, rising to 1 briefly once per delete interval, with the write rate non-zero only inside those windows.*\n\n###### Watch for:\n*A copy-forward rate that is large enough to move node-store write latency: rotation is competing with sync I/O, which is the whole hypothesis this panel tests. Copy-forward writes while the flag reads 0 would mean the window flag leaked, not that rotation is cheap. NO SERIES AT ALL on either query means online_delete is not configured on this node, which is different from a rotation that costs nothing.*\n\n###### Keywords:\n- **Rotation window** *(per node)* — the interval during which an online-delete backend swap is in progress.\n- **Copy-forward write** *(per node)* — rewriting a node body from the backend about to be deleted into the one replacing it.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerRotationStateGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rotation-window)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "State & Writes / Sec", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": 1800000, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*Rotation In Flight.*" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "left" - }, - { - "id": "max", - "value": 1 - }, - { - "id": "min", - "value": 0 - }, - { - "id": "custom.axisLabel", - "value": "In Flight (0/1)" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*Copy-Forward.*" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "unit", - "value": "cps" - }, - { - "id": "custom.axisLabel", - "value": "Writes / Sec" - } - ] - } - ] - }, - "gridPos": { - "h": 12, - "w": 12, - "x": 0, - "y": 284 - }, - "id": 66, - "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, - "legend": { - "calcs": [], - "displayMode": "list", - "enableFacetedFilter": false, - "overflow": "ellipsis", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "13.2.0-28926505616", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_work_item) (rotation_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"in_flight\"}), \"series\", \"Rotation In Flight (0/1)\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_work_item) (rate(rotation_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"copy_forward\"}[$__rate_interval])), \"series\", \"Copy-Forward Writes / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", - "refId": "B" - } - ], - "title": "Online-Delete Rotation Window & Copy-Forward Writes", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "###### What this is:\n*Rate at which a rotation has to rescue a tree node that was missing from BOTH backends and rewrite it from memory. Each one is an extra write on top of the whole-state-map walk the rotation already performs.*\n\n###### How it's computed:\n*Count of rotation_copy_node_restore_total. Incremented once per rescued node in the rotation's state-map walk, from a branch that was warn-log-only before, so the volume was invisible unless someone was reading logs.*\n\n###### Reading it:\n*Zero on a healthy node. Any sustained rate means clean nodes reachable from the validated state map have no on-disk copy left, because the backend holding them was removed by an earlier rotation and they were never rewritten.*\n\n###### Healthy range:\n*Flat at zero.*\n\n###### Watch for:\n*A non-zero rate is the signal that earlier rotations dropped data the current state map still needs: each rescue is a write that competes with sync I/O, and without the rescue the node would later surface as an unresolvable missing-node error. Read with the copy-forward panel — both are rotation-time writes, but this one also indicates prior data loss rather than merely cost. The node hash is deliberately not a label (unbounded cardinality); get it from the copyNode warning in Loki, correlated by node and time.*\n\n###### Keywords:\n- **Node re-store** *(per node)* — rewriting an in-memory tree node whose only on-disk copy was removed by an earlier rotation.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[SHAMapStoreImp.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/SHAMapStoreImp.cpp)\n\n###### Function:\n`SHAMapStoreImp::copyNode`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#node-re-store)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short", - "decimals": 0 - }, - "overrides": [] - }, - "gridPos": { - "h": 12, - "w": 12, - "x": 12, - "y": 284 - }, - "id": 67, - "options": { - "displayMode": "gradient", - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "maxVizHeight": 300, - "minVizHeight": 16, - "minVizWidth": 8, - "namePlacement": "left", - "orientation": "horizontal", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showUnfilled": true, - "sizing": "manual", - "valueMode": "color" - }, - "pluginVersion": "13.2.0-28926505616", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_work_item) (round(increase(rotation_copy_node_restore_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__range]))), \"series\", \"Nodes Re-Stored\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", - "refId": "A", - "instant": true - } - ], - "title": "Rotation Node Re-Stores (Count)", - "type": "bargauge" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 248 - }, - "id": 63, - "panels": [], - "title": "Spans & traces", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "###### What this is:\n*95th-percentile duration of each of the three phases of one ledger acquisition: the header wait, the account-state tree fetch, and the transaction tree fetch.*\n\n###### How it's computed:\n*95th percentile of the span-derived duration histogram for the three ledger.acquire.header / .astree / .txtree child spans, one series per phase. The parent ledger.acquire span is flat and cannot separate them, which matters because the account-state tree is nearly all of the work in a real fresh sync.*\n\n###### Reading it:\n*The astree series dominating is expected and healthy. The value of the split is the comparison: a header series that is large means the node is waiting to be told what to fetch, which is a peer-supply problem upstream of either tree.*\n\n###### Healthy range:\n*The astree series largest, the txtree series small, the header series near zero.*\n\n###### Watch for:\n*A hot astree band that keeps growing means account-state nodes are not being served — check Ledger Acquire Phase Outcomes beside it for timed_out, and Outbound Dial Outcome Rate in the Bootstrap row for refused or timed-out dials. A large header series instead means no peer is answering the header request at all.*\n\n###### Keywords:\n- **Acquire phase** *(per node)* — one of the three sequential fetches a ledger acquisition is made of; the header gates both trees, because it is what names their root hashes.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Derived by the OTel Collector's spanmetrics connector from spans emitted by xrpld code; the collector counts the spans and their durations, and the Grafana query selects and aggregates the resulting series.*\n\n###### Source:\n[InboundLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/InboundLedger.cpp)\n\n###### Function:\n`InboundLedger::syncPhaseSpans`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#acquire-phase)", + "description": "###### What this is:\n*95th-percentile duration of each of the three phases of one ledger acquisition: the header wait, the account-state tree fetch, and the transaction tree fetch.*\n\n###### How it's computed:\n*95th percentile of the span-derived duration histogram for the three ledger.acquire.header / .astree / .txtree child spans, one series per phase. The parent ledger.acquire span is flat and cannot separate them, which matters because the account-state tree is nearly all of the work in a real fresh sync.*\n\n###### Reading it:\n*The astree series dominating is expected and healthy. The value of the split is the comparison: a header series that is large means the node is waiting to be told what to fetch, which is a peer-supply problem upstream of either tree.*\n\n###### Healthy range:\n*The astree series largest, the txtree series small, the header series near zero.*\n\n###### Watch for:\n*A hot astree band that keeps growing means account-state nodes are not being served — check Ledger Acquire Phase Outcomes beside it for timed_out, and Outbound Dial Outcomes (Count By Outcome) in the Bootstrap row for refused or timed-out dials. A large header series instead means no peer is answering the header request at all.*\n\n###### Keywords:\n- **Acquire phase** *(per node)* — one of the three sequential fetches a ledger acquisition is made of; the header gates both trees, because it is what names their root hashes.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Derived by the OTel Collector's spanmetrics connector from spans emitted by xrpld code; the collector counts the spans and their durations, and the Grafana query selects and aggregates the resulting series.*\n\n###### Source:\n[InboundLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/InboundLedger.cpp)\n\n###### Function:\n`InboundLedger::syncPhaseSpans`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#acquire-phase)", "fieldConfig": { "defaults": { "color": { @@ -4400,7 +3866,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*How many ledger-acquire phases finished each way over the selected range, as whole counts.*\n\n###### How it's computed:\n*`increase(span_calls_total[$__range])` per `span_name` and `outcome`, so the number shown is the count of phases that ended that way inside the dashboard time range.*\n\n###### Reading it:\n*Counts, not rates. These are discrete, low-frequency events: a rate of 0.14/s is 278 phases over 33 minutes, which no reader can infer from the rate. Compare the Complete and Abandoned bars for the same phase. Header needs one round trip; both tree phases need many.*\n\n###### Healthy range:\n*Abandoned near zero for every phase, and Complete rising with ledger progress.*\n\n###### Watch for:\n*Abandoned dominating the tree phases while Header still completes. That means requests reach peers and headers arrive, but the many-round-trip phases never finish. `Abandoned` here means the acquire object was destroyed while still fetching, not that it timed out — see Acquire Stall Rate and the runbook for the sweep mechanism.*\n\n###### Keywords:\n- **Acquire phase** *(per node)* — one of the three sequential fetches of a ledger: header, account-state tree, transaction tree.\n- **Abandoned** *(per node)* — the acquire was destroyed with no result, i.e. dropped while still fetching rather than failing or timing out.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Spans are emitted by xrpld code; the collector's spanmetrics connector turns them into `span_calls_total`; the Grafana query counts the increase over the range.*\n\n###### Source:\n[InboundLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/InboundLedger.cpp)\n\n###### Function:\n`InboundLedger::endPhaseSpan`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#fresh-node-sync-diagnostics)", + "description": "###### What this is:\n*How many ledger-acquire phases finished each way over the selected range, as whole counts.*\n\n###### How it's computed:\n*`increase(span_calls_total[$__range])` per `span_name` and `outcome`, so the number shown is the count of phases that ended that way inside the dashboard time range.*\n\n###### Reading it:\n*Counts, not rates. These are discrete, low-frequency events: a count of 0.14 is 278 phases over 33 minutes, which no reader can infer from the rate. Compare the Complete and Abandoned bars for the same phase. Header needs one round trip; both tree phases need many.*\n\n###### Healthy range:\n*Abandoned near zero for every phase, and Complete rising with ledger progress.*\n\n###### Watch for:\n*Abandoned dominating the tree phases while Header still completes. That means requests reach peers and headers arrive, but the many-round-trip phases never finish. `Abandoned` here means the acquire object was destroyed while still fetching, not that it timed out — see Acquire Stalls — No Progress (Count) and the runbook for the sweep mechanism.*\n\n###### Keywords:\n- **Acquire phase** *(per node)* — one of the three sequential fetches of a ledger: header, account-state tree, transaction tree.\n- **Abandoned** *(per node)* — the acquire was destroyed with no result, i.e. dropped while still fetching rather than failing or timing out.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Spans are emitted by xrpld code; the collector's spanmetrics connector turns them into `span_calls_total`; the Grafana query counts the increase over the range.*\n\n###### Source:\n[InboundLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/InboundLedger.cpp)\n\n###### Function:\n`InboundLedger::endPhaseSpan`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#fresh-node-sync-diagnostics)", "fieldConfig": { "defaults": { "color": { @@ -4465,6 +3931,146 @@ "title": "Ledger Acquire Phase Outcomes (Count By Phase & Outcome)", "type": "bargauge" }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Number of times ledger-replay sub-tasks give up and fall back to acquiring a whole ledger, split by which sub-task gave up.*\n\n###### How it's computed:\n*Count of ledger_replay_fallback_total by stage. The skip-list stage and the delta stage each emit once, on the transition into fallback, from the branch that was debug-log-only before.*\n\n###### Reading it:\n*Zero when replay-capable peers are available. Any sustained count means the replay optimisation is being defeated and back-fill has reverted to the slower full-acquire path.*\n\n###### Healthy range:\n*Zero, or brief spikes while the peer set changes.*\n\n###### Watch for:\n*A persistent count on either stage: too few connected peers support the ledger-replay feature, so every historical ledger is fetched in full instead of as a delta. Read with Replay Outcomes.*\n\n###### Keywords:\n- **Replay fallback** *(per node)* — a replay sub-task abandoning the delta shortcut and acquiring the entire ledger instead.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[SkipListAcquire.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/SkipListAcquire.cpp)\n\n###### Function:\n`SkipListAcquire::trigger` / `LedgerDeltaAcquire::trigger`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#replay-fallback)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "displayName": "Replay Fallback [${__field.labels.series}] ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short", + "decimals": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 260 + }, + "id": 40, + "options": { + "displayMode": "gradient", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "left", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "manual", + "valueMode": "color" + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (stage, service_instance_id, xrpl_branch, xrpl_work_item) (last_over_time(ledger_replay_fallback_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", stage=~\"$replay_stage\"}[$__range])), \"series\", \"$1\", \"stage\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A", + "instant": true + } + ], + "title": "Replay Fallback To Full Acquire (Count By Stage)", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Count of ledger-replay tasks reaching each terminal state: success, timeout, build failure or parameter failure.*\n\n###### How it's computed:\n*Count of ledger_replay_outcome_total by outcome. Every terminal path in the replay task emits exactly once; before this, all four only set an internal flag and wrote a log line.*\n\n###### Reading it:\n*Successes only is healthy. Timeouts mean deltas never arrived; build failures mean a delta would not apply to its parent; parameter failures mean a peer served an inconsistent skip list.*\n\n###### Healthy range:\n*Successes non-zero while replaying, all failure outcomes at zero.*\n\n###### Watch for:\n*Any failure outcome climbing while successes stay flat: replay is running but never completing, so history back-fill is silently falling back to the slower path. The outcome value tells you which layer to look at — timeouts point at peers, build and parameter failures point at the data those peers served.*\n\n###### Keywords:\n- **Ledger replay** *(per node)* — rebuilding a range of historical ledgers from a start ledger plus per-ledger deltas instead of downloading each one whole.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[LedgerReplayTask.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/LedgerReplayTask.cpp)\n\n###### Function:\n`LedgerReplayTask::recordOutcome`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledger-replay)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "displayName": "Replay Outcome [${__field.labels.series}] ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short", + "decimals": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 260 + }, + "id": 41, + "options": { + "displayMode": "gradient", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "left", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "manual", + "valueMode": "color" + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (outcome, service_instance_id, xrpl_branch, xrpl_work_item) (last_over_time(ledger_replay_outcome_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", outcome=~\"$replay_outcome\"}[$__range])), \"series\", \"$1\", \"outcome\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A", + "instant": true + } + ], + "title": "Replay Outcomes (Count By Terminal State)", + "type": "bargauge" + }, { "datasource": { "type": "prometheus", @@ -4668,7 +4274,193 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*Rate of outbound peer dials reaching each terminal outcome, derived from the per-attempt peer.dial span.*\n\n###### How it's computed:\n*Rate of span-derived call counts for the peer.dial span, split by outcome. The same five outcome values the overlay_connect_total counter carries, set from the same funnel in the dial state machine so the two cannot disagree.*\n\n###### Reading it:\n*Read alongside Outbound Dial Outcome Rate in the Bootstrap row, which is the native counter for the same events. This panel exists for what the counter cannot do: each point here is backed by traces, so clicking through gives the individual attempt and the peer address it was dialling, which is never a metric label because one series per peer address would be unbounded.*\n\n###### Healthy range:\n*The connected series non-zero, failure series at or near zero.*\n\n###### Watch for:\n*A failure series dominating while connected stays at zero means the node has no outbound peers and cannot sync at all. Use the trace drill-down to find which endpoint keeps failing — the aggregate rate cannot tell one bad peer from a broken local network.*\n\n###### Keywords:\n- **Outbound dial** *(per node)* — one attempt by this node to open a peer connection, spanning the TCP connect, the TLS handshake and the protocol upgrade.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Derived by the OTel Collector's spanmetrics connector from spans emitted by xrpld code; the collector counts the spans and their durations, and the Grafana query selects and aggregates the resulting series.*\n\n###### Source:\n[ConnectAttempt.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/ConnectAttempt.cpp)\n\n###### Function:\n`ConnectAttempt::reportOutcome`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#outbound-dial-latency)", + "description": "###### What this is:\n*How long the heap trim that ends every cache sweep takes. The trim returns free heap pages to the kernel, and its cost scales with the resident heap — so this is the panel that shows a node with a large existing database paying a per-sweep penalty a fresh node never pays.*\n\n###### How it's computed:\n*p50 and p95 of the sweep_malloc_trim_us histogram. Recorded once per sweep, so the sample rate is one per sweep interval (10 s on a tiny node through 120 s on a huge one). The measurement used to run only at debug log level, so on an ordinary node there was no value at all.*\n\n###### Reading it:\n*Read the two lines together against the sweep interval. Sub-millisecond is free. Tens of milliseconds against a 10 s interval is still a small duty cycle but means the trim is walking a large heap, and it runs on the sweep job — so the cost lands on the job queue, not in the background.*\n\n###### Healthy range:\n*Under a millisecond on a warm node with a modest heap.*\n\n###### Watch for:\n*p95 climbing as the database grows, especially alongside a rising sweep-job queue wait. IMPORTANT LIMITATION: the fault counter beside this panel covers the trim call only. It proves the trim itself faults; it does NOT prove the trim causes the later faults taken as the caches refill. Correlate this duration against sweep-job queueing rather than concluding the trim caused a slow sync.*\n\n###### Keywords:\n- **Heap trim** *(per node)* — returning free heap pages from the allocator's arenas back to the kernel.\n- **Sweep interval** *(per node)* — how often the periodic cache sweep runs, set by node size.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[Application.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/main/Application.cpp)\n\n###### Function:\n`ApplicationImp::trimHeapAndRecord`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#heap-trim)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Trim Duration", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "µs" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 272 + }, + "id": 64, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.5, sum by (le, service_instance_id, xrpl_branch, xrpl_work_item) (rate(sweep_malloc_trim_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"P50 Trim\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_work_item) (rate(sweep_malloc_trim_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"P95 Trim\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "B" + } + ], + "title": "Sweep Heap-Trim Duration (p50/p95)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Two rates side by side: minor page faults taken inside the heap trim, and the resident memory the trim actually returned to the kernel. Together they say whether the trim is buying anything for what it costs.*\n\n###### How it's computed:\n*Count of sweep_malloc_trim_minor_faults_total and of sweep_malloc_trim_reclaimed_kb_total. Both are cumulative counters exported per sweep, so the panel rates them rather than plotting the totals. Reclaim is published only when resident memory actually fell; a sweep during which another thread grew the heap contributes nothing rather than a negative amount.*\n\n###### Reading it:\n*Reclaimed volume with a near-zero fault count is a cheap, useful trim. A fault count that moves with the reclaimed volume means the pages are being handed back and immediately taken again, which is churn rather than savings.*\n\n###### Healthy range:\n*A reclaimed volume that tracks cache turnover, with faults near zero.*\n\n###### Watch for:\n*Reclaim near zero while the trim duration panel shows real time being spent: the trim is walking the heap and freeing nothing, which is pure cost. IMPORTANT LIMITATION: the fault delta is scoped to the trim call, so it cannot show the re-fault cost paid later as the caches refill and touch the returned pages. That later cost is real but is NOT measured here; do not read a low fault count as proof the trim was free.*\n\n###### Keywords:\n- **Minor page fault** *(per node)* — a memory access satisfied without disk I/O, by mapping a page the kernel already holds.\n- **Reclaimed resident memory** *(per node)* — resident kilobytes the allocator handed back to the kernel.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[Application.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/main/Application.cpp)\n\n###### Function:\n`ApplicationImp::trimHeapAndRecord`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#minor-page-fault)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short", + "decimals": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 272 + }, + "id": 65, + "options": { + "displayMode": "gradient", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "left", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "manual", + "valueMode": "color" + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_work_item) (last_over_time(sweep_malloc_trim_minor_faults_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__range])), \"series\", \"Minor Faults\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A", + "instant": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_work_item) (last_over_time(sweep_malloc_trim_reclaimed_kb_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__range])), \"series\", \"Reclaimed KB\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "B", + "instant": true + } + ], + "title": "Sweep Heap-Trim Faults & Reclaim (Count)", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Rate of outbound peer dials reaching each terminal outcome, derived from the per-attempt peer.dial span.*\n\n###### How it's computed:\n*Rate of span-derived call counts for the peer.dial span, split by outcome. The same five outcome values the overlay_connect_total counter carries, set from the same funnel in the dial state machine so the two cannot disagree.*\n\n###### Reading it:\n*Read alongside Outbound Dial Outcomes (Count By Outcome) in the Bootstrap row, which is the native counter for the same events. This panel exists for what the counter cannot do: each point here is backed by traces, so clicking through gives the individual attempt and the peer address it was dialling, which is never a metric label because one series per peer address would be unbounded.*\n\n###### Healthy range:\n*The connected series non-zero, failure series at or near zero.*\n\n###### Watch for:\n*A failure series dominating while connected stays at zero means the node has no outbound peers and cannot sync at all. Use the trace drill-down to find which endpoint keeps failing — the aggregate rate cannot tell one bad peer from a broken local network.*\n\n###### Keywords:\n- **Outbound dial** *(per node)* — one attempt by this node to open a peer connection, spanning the TCP connect, the TLS handshake and the protocol upgrade.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Derived by the OTel Collector's spanmetrics connector from spans emitted by xrpld code; the collector counts the spans and their durations, and the Grafana query selects and aggregates the resulting series.*\n\n###### Source:\n[ConnectAttempt.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/ConnectAttempt.cpp)\n\n###### Function:\n`ConnectAttempt::reportOutcome`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#outbound-dial-latency)", "fieldConfig": { "defaults": { "color": { @@ -4767,7 +4559,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*Rate at which this node answers peers' ledger-data requests, split by what was asked for and how the reply ended.*\n\n###### How it's computed:\n*Rate of span-derived call counts for the ledger.serve span, split by object_type (header, transaction tree, account-state tree, or a proposed transaction set) and by outcome. The outcome is derived from the reply itself, so it is refused whenever nothing was sent.*\n\n###### Reading it:\n*This is the supply side — what this node does for its peers, not what it receives. The account-state series is the one that matters to a syncing peer, since that tree is the bulk of a fresh sync.*\n\n###### Healthy range:\n*Non-zero complete series on the types peers ask for, refused near zero.*\n\n###### Watch for:\n*A refused series climbing means this node is declining to serve; the paired serve_refused_total counter on Ledger/Object Serve Refusals gives the specific cause. A partial series means replies keep hitting the size cap, so peers must make repeated round trips for one tree. Note this panel does not explain this node's own sync — it explains its peers'.*\n\n###### Keywords:\n- **Ledger serve** *(per node)* — this node answering a peer's request for ledger data, as opposed to requesting data for itself.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Derived by the OTel Collector's spanmetrics connector from spans emitted by xrpld code; the collector counts the spans and their durations, and the Grafana query selects and aggregates the resulting series.*\n\n###### Source:\n[PeerImp.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/PeerImp.cpp)\n\n###### Function:\n`PeerImp::processLedgerRequest`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledger-serve)", + "description": "###### What this is:\n*Rate at which this node answers peers' ledger-data requests, split by what was asked for and how the reply ended.*\n\n###### How it's computed:\n*Rate of span-derived call counts for the ledger.serve span, split by object_type (header, transaction tree, account-state tree, or a proposed transaction set) and by outcome. The outcome is derived from the reply itself, so it is refused whenever nothing was sent.*\n\n###### Reading it:\n*This is the supply side — what this node does for its peers, not what it receives. The account-state series is the one that matters to a syncing peer, since that tree is the bulk of a fresh sync.*\n\n###### Healthy range:\n*Non-zero complete series on the types peers ask for, refused near zero.*\n\n###### Watch for:\n*A refused series climbing means this node is declining to serve; the paired serve_refused_total counter on Ledger/Object Serve Refusals (Count By Reason & Request) gives the specific cause. A partial series means replies keep hitting the size cap, so peers must make repeated round trips for one tree. Note this panel does not explain this node's own sync — it explains its peers'.*\n\n###### Keywords:\n- **Ledger serve** *(per node)* — this node answering a peer's request for ledger data, as opposed to requesting data for itself.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Derived by the OTel Collector's spanmetrics connector from spans emitted by xrpld code; the collector counts the spans and their durations, and the Grafana query selects and aggregates the resulting series.*\n\n###### Source:\n[PeerImp.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/PeerImp.cpp)\n\n###### Function:\n`PeerImp::processLedgerRequest`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledger-serve)", "fieldConfig": { "defaults": { "color": { @@ -4866,7 +4658,230 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*Rate of trusted validations reaching the ledger-acceptance gate, split by what the validation store did with each one and by whether it actually reached the gate.*\n\n###### How it's computed:\n*Rate of span-derived calls for the consensus.validation.accept span, by validation_status and accept_gated. The span is emitted once per trusted validation and carries the trace id of the ledger it validates, so any point here can be opened as the full trace for that ledger — the validation, the acceptance decision it drove, and the acquire and store spans for the same ledger.*\n\n###### Reading it:\n*Nearly all of the rate should be validation_status=current, which is the only status that continues to the gate. accept_gated=true means another thread was already accepting that ledger, so no acceptance followed this validation; a modest share is normal when validations for one ledger arrive together.*\n\n###### Healthy range:\n*Dominated by current, at roughly the trusted-validator count per ledger close.*\n\n###### Watch for:\n*Rate concentrated in stale, bad_seq, multiple or conflicting: validations are arriving and being counted for nothing, which is the difference between a node that is slow to validate and one that never will — from the outside the two look identical. Confirm on Trusted Validations vs Quorum Target and Pre-Accept Quorum Shortfall Rate, and check UNL Quorum Headroom in the Bootstrap row first, since a trusted list that cannot satisfy quorum starves everything below it. A flat zero on a peered node means no trusted validations are arriving at all.*\n\n###### Keywords:\n- **Validation status** *(per node)* — what this node's validation store did with an arriving validation; only current counts toward accepting a ledger.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Derived by the OTel Collector's spanmetrics connector from spans emitted by xrpld code; the collector counts the spans, and the Grafana query selects and aggregates the resulting series.*\n\n###### Source:\n[RCLValidations.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/RCLValidations.cpp)\n\n###### Function:\n`handleNewValidation`\n\n###### References:\n[Negative UNL and validation quorum on xrpl.org](https://xrpl.org/docs/concepts/consensus-protocol/negative-unl) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#validation-status)", + "description": "###### What this is:\n*When an online-delete rotation is running, and the rate of the extra writes it forces. A rotation rewrites into the new backend any node body the doomed archive serves, which is I/O an ordinary fetch would never perform — and it exists only on a populated, already-rotated database, which is why it never appears on a fresh node.*\n\n###### How it's computed:\n*rotation_state with metric=in_flight plotted raw (it is a 0/1 state flag), and metric=copy_forward — a cumulative write total — plotted as a rate. Both are read from the node store on each collection tick. The copy-forward count existed before as a log-only per-rotation tally that reset on every swap; the total behind this panel never resets, so it can be rated.*\n\n###### Reading it:\n*The two must move together: copy-forward writes should only appear while the window flag is 1. Read the write rate against the node-store write latency panel above — that is what tells extra rotation writes from a slow backend.*\n\n###### Healthy range:\n*Flag at 0 most of the time, rising to 1 briefly once per delete interval, with the write rate non-zero only inside those windows.*\n\n###### Watch for:\n*A copy-forward rate that is large enough to move node-store write latency: rotation is competing with sync I/O, which is the whole hypothesis this panel tests. Copy-forward writes while the flag reads 0 would mean the window flag leaked, not that rotation is cheap. NO SERIES AT ALL on either query means online_delete is not configured on this node, which is different from a rotation that costs nothing.*\n\n###### Keywords:\n- **Rotation window** *(per node)* — the interval during which an online-delete backend swap is in progress.\n- **Copy-forward write** *(per node)* — rewriting a node body from the backend about to be deleted into the one replacing it.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerRotationStateGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rotation-window)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "State & Writes / Sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*Rotation In Flight.*" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "left" + }, + { + "id": "max", + "value": 1 + }, + { + "id": "min", + "value": 0 + }, + { + "id": "custom.axisLabel", + "value": "In Flight (0/1)" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*Copy-Forward.*" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit", + "value": "cps" + }, + { + "id": "custom.axisLabel", + "value": "Writes / Sec" + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 284 + }, + "id": 66, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_work_item) (rotation_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"in_flight\"}), \"series\", \"Rotation In Flight (0/1)\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_work_item) (rate(rotation_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"copy_forward\"}[$__rate_interval])), \"series\", \"Copy-Forward Writes / Sec\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "B" + } + ], + "title": "Online-Delete Rotation Window & Copy-Forward Writes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Number of times a rotation has to rescue a tree node that was missing from BOTH backends and rewrite it from memory. Each one is an extra write on top of the whole-state-map walk the rotation already performs.*\n\n###### How it's computed:\n*Count of rotation_copy_node_restore_total. Incremented once per rescued node in the rotation's state-map walk, from a branch that was warn-log-only before, so the volume was invisible unless someone was reading logs.*\n\n###### Reading it:\n*Zero on a healthy node. Any sustained count means clean nodes reachable from the validated state map have no on-disk copy left, because the backend holding them was removed by an earlier rotation and they were never rewritten.*\n\n###### Healthy range:\n*Flat at zero.*\n\n###### Watch for:\n*A non-zero count is the signal that earlier rotations dropped data the current state map still needs: each rescue is a write that competes with sync I/O, and without the rescue the node would later surface as an unresolvable missing-node error. Read with the copy-forward panel — both are rotation-time writes, but this one also indicates prior data loss rather than merely cost. The node hash is deliberately not a label (unbounded cardinality); get it from the copyNode warning in Loki, correlated by node and time.*\n\n###### Keywords:\n- **Node re-store** *(per node)* — rewriting an in-memory tree node whose only on-disk copy was removed by an earlier rotation.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[SHAMapStoreImp.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/SHAMapStoreImp.cpp)\n\n###### Function:\n`SHAMapStoreImp::copyNode`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#node-re-store)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short", + "decimals": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 284 + }, + "id": 67, + "options": { + "displayMode": "gradient", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "left", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "manual", + "valueMode": "color" + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_work_item) (last_over_time(rotation_copy_node_restore_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__range])), \"series\", \"Nodes Re-Stored\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A", + "instant": true + } + ], + "title": "Rotation Node Re-Stores (Count)", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Rate of trusted validations reaching the ledger-acceptance gate, split by what the validation store did with each one and by whether it actually reached the gate.*\n\n###### How it's computed:\n*Rate of span-derived calls for the consensus.validation.accept span, by validation_status and accept_gated. The span is emitted once per trusted validation and carries the trace id of the ledger it validates, so any point here can be opened as the full trace for that ledger — the validation, the acceptance decision it drove, and the acquire and store spans for the same ledger.*\n\n###### Reading it:\n*Nearly all of the rate should be validation_status=current, which is the only status that continues to the gate. accept_gated=true means another thread was already accepting that ledger, so no acceptance followed this validation; a modest share is normal when validations for one ledger arrive together.*\n\n###### Healthy range:\n*Dominated by current, at roughly the trusted-validator count per ledger close.*\n\n###### Watch for:\n*Rate concentrated in stale, bad_seq, multiple or conflicting: validations are arriving and being counted for nothing, which is the difference between a node that is slow to validate and one that never will — from the outside the two look identical. Confirm on Trusted Validations vs Quorum Target and Pre-Accept Quorum Shortfalls (Count By Stage), and check UNL Quorum Headroom in the Bootstrap row first, since a trusted list that cannot satisfy quorum starves everything below it. A flat zero on a peered node means no trusted validations are arriving at all.*\n\n###### Keywords:\n- **Validation status** *(per node)* — what this node's validation store did with an arriving validation; only current counts toward accepting a ledger.\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Derived by the OTel Collector's spanmetrics connector from spans emitted by xrpld code; the collector counts the spans, and the Grafana query selects and aggregates the resulting series.*\n\n###### Source:\n[RCLValidations.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/RCLValidations.cpp)\n\n###### Function:\n`handleNewValidation`\n\n###### References:\n[Negative UNL and validation quorum on xrpl.org](https://xrpl.org/docs/concepts/consensus-protocol/negative-unl) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#validation-status)", "fieldConfig": { "defaults": { "color": { @@ -4960,6 +4975,19 @@ ], "title": "Trusted Validation Accept Rate by Status", "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 296 + }, + "id": 63, + "panels": [], + "title": "Spans & traces", + "type": "row" } ], "schemaVersion": 39,