diff --git a/docker/telemetry/grafana/dashboards/ledger-sync-health.json b/docker/telemetry/grafana/dashboards/ledger-sync-health.json index c614a972d8..7e868e4eb2 100644 --- a/docker/telemetry/grafana/dashboards/ledger-sync-health.json +++ b/docker/telemetry/grafana/dashboards/ledger-sync-health.json @@ -125,109 +125,6 @@ "repeatDirection": "h", "maxPerRow": 2 }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "###### What this is:\n*Time taken to resolve a configured peer hostname, at the 95th percentile.*\n\n###### How it's computed:\n*Resolution duration samples aggregated to their 95th percentile per node.*\n\n###### Reading it:\n*Lower is better; it is the delay before the node can start dialling peers.*\n\n###### Healthy range:\n*Tens of milliseconds against a healthy resolver.*\n\n###### Watch for:\n*Seconds-scale latency means the resolver is timing out and every bootstrap attempt is paying that delay before the first dial.*\n\n###### Keywords:\n- **DNS resolve** *(per node)* \u2014 turning a configured peer hostname into IP addresses before any dial is attempted; slow resolution delays the whole bootstrap.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl::reportDnsResolve`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#dns-resolve)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Latency (ms)", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": 1800000, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - } - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 1 - }, - "id": 3, - "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, - "legend": { - "calcs": [], - "displayMode": "list", - "enableFacetedFilter": false, - "overflow": "ellipsis", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "13.2.0-28926505616", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_work_item) (dns_resolve_latency_ms_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"})), \"series\", \"P95 DNS Resolve\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", - "refId": "A" - } - ], - "title": "DNS Resolve Latency (p95)", - "type": "timeseries" - }, { "datasource": { "type": "prometheus", @@ -260,7 +157,7 @@ "x": 0, "y": 11 }, - "id": 4, + "id": 3, "options": { "displayMode": "gradient", "legend": { @@ -301,109 +198,6 @@ "repeatDirection": "h", "maxPerRow": 2 }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "###### What this is:\n*Time from starting an outbound peer dial to its terminal outcome, at the 95th percentile.*\n\n###### How it's computed:\n*Dial duration samples aggregated to their 95th percentile per node.*\n\n###### Reading it:\n*Lower is better. The series covers successes and failures together, so a rising p95 usually means attempts are ending in timeout rather than being refused fast.*\n\n###### Healthy range:\n*Tens to low hundreds of milliseconds on a local or same-region peer.*\n\n###### Watch for:\n*A p95 pinned near the dial timeout, which means peers accept the TCP connection but never complete the handshake.*\n\n###### Keywords:\n- **Outbound dial latency** *(per node)* \u2014 elapsed time of an outbound peer connection attempt, measured to whichever outcome ends it.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[ConnectAttempt.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/ConnectAttempt.cpp)\n\n###### Function:\n`ConnectAttempt::reportOutcome`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#outbound-dial-latency)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Latency (ms)", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": 1800000, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - } - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 11 - }, - "id": 5, - "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, - "legend": { - "calcs": [], - "displayMode": "list", - "enableFacetedFilter": false, - "overflow": "ellipsis", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "13.2.0-28926505616", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_work_item) (overlay_dial_latency_ms_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"})), \"series\", \"P95 Outbound Dial\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", - "refId": "A" - } - ], - "title": "Outbound Dial Latency (p95)", - "type": "timeseries" - }, { "datasource": { "type": "prometheus", @@ -436,7 +230,7 @@ "x": 0, "y": 21 }, - "id": 6, + "id": 4, "options": { "displayMode": "gradient", "legend": { @@ -506,10 +300,10 @@ "gridPos": { "h": 10, "w": 12, - "x": 12, - "y": 21 + "x": 0, + "y": 31 }, - "id": 7, + "id": 5, "options": { "displayMode": "gradient", "legend": { @@ -550,6 +344,283 @@ "repeatDirection": "h", "maxPerRow": 2 }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Spare trusted validator keys above the required quorum \u2014 the single number that says whether this node can ever validate.*\n\n###### How it's computed:\n*Trusted key count minus the required quorum, matched per node.*\n\n###### Reading it:\n*Positive is healthy. Zero or negative (red) means the trusted UNL is too small to ever satisfy quorum, so the node will stay short of a validated ledger.*\n\n###### Healthy range:\n*Positive; the exact figure depends on UNL size and the configured quorum.*\n\n###### Watch for:\n*Zero or below. Pair it with UNL Fetch Outcomes (Count By Site & Outcome): a site stuck on fetch_error or expired is the usual cause of a UNL too small to meet quorum.*\n\n###### Keywords:\n- **UNL quorum headroom** *(per node)* \u2014 trusted UNL key count minus the required quorum; at or below zero the node can never declare a ledger validated.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerUnlQuorumGauge`\n\n###### References:\n[Validation quorum on xrpl.org](https://xrpl.org/docs/concepts/consensus-protocol/negative-unl) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#unl-quorum-headroom)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "green", + "value": 2 + } + ] + }, + "unit": "short", + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}" + } + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 41 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(unl_quorum{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"trusted_keys\"} - ignoring(metric) unl_quorum{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"quorum\"}, \"series\", \"Quorum Headroom\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "UNL Quorum Headroom [$xrpl_network_type]", + "type": "stat", + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Time taken to resolve a configured peer hostname, at the 95th percentile.*\n\n###### How it's computed:\n*Resolution duration samples aggregated to their 95th percentile per node.*\n\n###### Reading it:\n*Lower is better; it is the delay before the node can start dialling peers.*\n\n###### Healthy range:\n*Tens of milliseconds against a healthy resolver.*\n\n###### Watch for:\n*Seconds-scale latency means the resolver is timing out and every bootstrap attempt is paying that delay before the first dial.*\n\n###### Keywords:\n- **DNS resolve** *(per node)* \u2014 turning a configured peer hostname into IP addresses before any dial is attempted; slow resolution delays the whole bootstrap.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl::reportDnsResolve`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#dns-resolve)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Latency (ms)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + } + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 51 + }, + "id": 7, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_work_item) (dns_resolve_latency_ms_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"})), \"series\", \"P95 DNS Resolve\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "DNS Resolve Latency (p95)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Time from starting an outbound peer dial to its terminal outcome, at the 95th percentile.*\n\n###### How it's computed:\n*Dial duration samples aggregated to their 95th percentile per node.*\n\n###### Reading it:\n*Lower is better. The series covers successes and failures together, so a rising p95 usually means attempts are ending in timeout rather than being refused fast.*\n\n###### Healthy range:\n*Tens to low hundreds of milliseconds on a local or same-region peer.*\n\n###### Watch for:\n*A p95 pinned near the dial timeout, which means peers accept the TCP connection but never complete the handshake.*\n\n###### Keywords:\n- **Outbound dial latency** *(per node)* \u2014 elapsed time of an outbound peer connection attempt, measured to whichever outcome ends it.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[ConnectAttempt.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/ConnectAttempt.cpp)\n\n###### Function:\n`ConnectAttempt::reportOutcome`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#outbound-dial-latency)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Latency (ms)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + } + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 51 + }, + "id": 8, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_work_item) (overlay_dial_latency_ms_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"})), \"series\", \"P95 Outbound Dial\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Outbound Dial Latency (p95)", + "type": "timeseries" + }, { "datasource": { "type": "prometheus", @@ -616,9 +687,9 @@ "h": 10, "w": 12, "x": 0, - "y": 31 + "y": 61 }, - "id": 8, + "id": 9, "options": { "annotations": { "clustering": -1, @@ -661,77 +732,6 @@ "title": "UNL Trusted Keys vs Quorum", "type": "timeseries" }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "###### What this is:\n*Spare trusted validator keys above the required quorum \u2014 the single number that says whether this node can ever validate.*\n\n###### How it's computed:\n*Trusted key count minus the required quorum, matched per node.*\n\n###### Reading it:\n*Positive is healthy. Zero or negative (red) means the trusted UNL is too small to ever satisfy quorum, so the node will stay short of a validated ledger.*\n\n###### Healthy range:\n*Positive; the exact figure depends on UNL size and the configured quorum.*\n\n###### Watch for:\n*Zero or below. Pair it with UNL Fetch Outcomes (Count By Site & Outcome): a site stuck on fetch_error or expired is the usual cause of a UNL too small to meet quorum.*\n\n###### Keywords:\n- **UNL quorum headroom** *(per node)* \u2014 trusted UNL key count minus the required quorum; at or below zero the node can never declare a ledger validated.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerUnlQuorumGauge`\n\n###### References:\n[Validation quorum on xrpl.org](https://xrpl.org/docs/concepts/consensus-protocol/negative-unl) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#unl-quorum-headroom)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "yellow", - "value": 1 - }, - { - "color": "green", - "value": 2 - } - ] - }, - "unit": "short", - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}" - } - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 31 - }, - "id": 9, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "center", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "value_and_name", - "wideLayout": true - }, - "pluginVersion": "13.2.0-28926505616", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(unl_quorum{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"trusted_keys\"} - ignoring(metric) unl_quorum{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"quorum\"}, \"series\", \"Quorum Headroom\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", - "refId": "A" - } - ], - "title": "UNL Quorum Headroom [$xrpl_network_type]", - "type": "stat", - "repeat": "xrpl_network_type", - "repeatDirection": "h", - "maxPerRow": 2 - }, { "datasource": { "type": "prometheus", @@ -801,8 +801,8 @@ "gridPos": { "h": 10, "w": 12, - "x": 0, - "y": 41 + "x": 12, + "y": 61 }, "id": 10, "options": { @@ -853,13 +853,159 @@ "h": 1, "w": 24, "x": 0, - "y": 51 + "y": 71 }, "id": 11, "panels": [], "title": "Peer supply", "type": "row" }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Count of inbound peer connection handoffs, split by terminal outcome.*\n\n###### How it's computed:\n*Count over the selected range of finished inbound handoffs grouped by outcome, per node. Filter the outcome set with the Accept Outcome variable.*\n\n###### Reading it:\n*accepted should dominate. The failure outcomes name what rejected the connection: resource_limit and no_slot are this node's own capacity, protocol_mismatch and bad_cookie are the peer or the network identity, handshake_error and local_endpoint_fail are the transport.*\n\n###### Healthy range:\n*accepted dominant, failures near zero.*\n\n###### Watch for:\n*Read this together with Outbound Dial Outcomes (Count By Outcome) above, which is the outbound twin on overlay_connect_total{outcome}. The two share the same one-outcome-per-attempt shape, so together they give the full in/out split: a node that accepts nothing but dials successfully has a very different fault from one that can neither dial nor accept. A steady no_slot or resource_limit is this node refusing peers it has no room for, which is capacity rather than a fault.*\n\n###### Keywords:\n- **Inbound peer accept outcome** *(per node)* \u2014 the terminal result of an inbound peer connection handoff; each handoff ends in exactly one outcome.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl::onHandoff`\n\n###### References:\n[Overlay](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#overlay) \u00b7 [Outbound dial latency](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#outbound-dial-latency)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "displayName": "Peer Accept [${__field.labels.series}] ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short", + "decimals": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 72 + }, + "id": 12, + "options": { + "displayMode": "gradient", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "left", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "manual", + "valueMode": "color" + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (outcome, service_instance_id, xrpl_branch, xrpl_work_item) (increase(peer_accept_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", outcome=~\"$accept_outcome\"}[$__range])), \"series\", \"$1\", \"outcome\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A", + "instant": true + } + ], + "title": "Inbound Peer Accept Outcomes (Count By Outcome) [$xrpl_network_type]", + "type": "bargauge", + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Number of times peer connections are closed, split by why they closed and by which side opened them.*\n\n###### How it's computed:\n*Count over the selected range of peer closes grouped by reason and direction (inbound or outbound), per node. Filter with the Disconnect Reason and Disconnect Direction variables.*\n\n###### Reading it:\n*The split separates our-fault backpressure from topology and network faults. large_sendq and charge_resources are this node shedding a peer it cannot keep up with. not_useful and ping_timeout are topology and liveness. read_error and write_error are the transport. graceful, stopping and shutdown are ordinary lifecycle, not faults.*\n\n###### Healthy range:\n*Mostly graceful; the fault reasons near zero.*\n\n###### Watch for:\n*A sustained large_sendq or charge_resources means this node is the bottleneck and is dropping peers \u2014 which removes the very peers it needs to sync from, so a slow node makes itself slower. A rising not_useful or ping_timeout on outbound points at the peer set instead. Distinct from the single unlabelled peer-disconnect total on the Node Health dashboard, which cannot say which of these is happening.*\n\n###### Keywords:\n- **Peer disconnect reason** *(per node)* \u2014 the cause recorded when a peer connection is closed, paired with whether that connection was inbound or outbound.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[PeerImp.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/PeerImp.cpp)\n\n###### Function:\n`PeerImp::close`\n\n###### References:\n[Resource disconnect](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#resource-disconnect) \u00b7 [Insane / diverged peers](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#insane-diverged-peers)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "displayName": "Peer Disconnect [${__field.labels.series}] ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short", + "decimals": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 82 + }, + "id": 13, + "options": { + "displayMode": "gradient", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "left", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "manual", + "valueMode": "color" + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_join(sum by (reason, direction, service_instance_id, xrpl_branch, xrpl_work_item) (increase(peer_disconnect_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", reason=~\"$disconnect_reason\", direction=~\"$disconnect_direction\"}[$__range])), \"series\", \" \", \"reason\", \"direction\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A", + "instant": true + } + ], + "title": "Peer Disconnects (Count By Reason & Direction) [$xrpl_network_type]", + "type": "bargauge", + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 + }, { "datasource": { "type": "prometheus", @@ -922,9 +1068,9 @@ "h": 10, "w": 12, "x": 0, - "y": 52 + "y": 92 }, - "id": 12, + "id": 14, "options": { "annotations": { "clustering": -1, @@ -1052,9 +1198,9 @@ "h": 10, "w": 12, "x": 12, - "y": 52 + "y": 92 }, - "id": 13, + "id": 15, "options": { "annotations": { "clustering": -1, @@ -1159,9 +1305,9 @@ "h": 10, "w": 12, "x": 0, - "y": 62 + "y": 102 }, - "id": 14, + "id": 16, "options": { "annotations": { "clustering": -1, @@ -1258,9 +1404,9 @@ "h": 10, "w": 12, "x": 12, - "y": 62 + "y": 102 }, - "id": 15, + "id": 17, "options": { "annotations": { "clustering": -1, @@ -1295,159 +1441,13 @@ "title": "PeerFinder Address Caches & Fixed Peers", "type": "timeseries" }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "###### What this is:\n*Count of inbound peer connection handoffs, split by terminal outcome.*\n\n###### How it's computed:\n*Count over the selected range of finished inbound handoffs grouped by outcome, per node. Filter the outcome set with the Accept Outcome variable.*\n\n###### Reading it:\n*accepted should dominate. The failure outcomes name what rejected the connection: resource_limit and no_slot are this node's own capacity, protocol_mismatch and bad_cookie are the peer or the network identity, handshake_error and local_endpoint_fail are the transport.*\n\n###### Healthy range:\n*accepted dominant, failures near zero.*\n\n###### Watch for:\n*Read this together with Outbound Dial Outcomes (Count By Outcome) above, which is the outbound twin on overlay_connect_total{outcome}. The two share the same one-outcome-per-attempt shape, so together they give the full in/out split: a node that accepts nothing but dials successfully has a very different fault from one that can neither dial nor accept. A steady no_slot or resource_limit is this node refusing peers it has no room for, which is capacity rather than a fault.*\n\n###### Keywords:\n- **Inbound peer accept outcome** *(per node)* \u2014 the terminal result of an inbound peer connection handoff; each handoff ends in exactly one outcome.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[OverlayImpl.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/OverlayImpl.cpp)\n\n###### Function:\n`OverlayImpl::onHandoff`\n\n###### References:\n[Overlay](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#overlay) \u00b7 [Outbound dial latency](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#outbound-dial-latency)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "displayName": "Peer Accept [${__field.labels.series}] ${__field.labels.xrpl_ident}", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short", - "decimals": 0 - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 72 - }, - "id": 16, - "options": { - "displayMode": "gradient", - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "maxVizHeight": 300, - "minVizHeight": 16, - "minVizWidth": 8, - "namePlacement": "left", - "orientation": "horizontal", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showUnfilled": true, - "sizing": "manual", - "valueMode": "color" - }, - "pluginVersion": "13.2.0-28926505616", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(sum by (outcome, service_instance_id, xrpl_branch, xrpl_work_item) (increase(peer_accept_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", outcome=~\"$accept_outcome\"}[$__range])), \"series\", \"$1\", \"outcome\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", - "refId": "A", - "instant": true - } - ], - "title": "Inbound Peer Accept Outcomes (Count By Outcome) [$xrpl_network_type]", - "type": "bargauge", - "repeat": "xrpl_network_type", - "repeatDirection": "h", - "maxPerRow": 2 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "###### What this is:\n*Number of times peer connections are closed, split by why they closed and by which side opened them.*\n\n###### How it's computed:\n*Count over the selected range of peer closes grouped by reason and direction (inbound or outbound), per node. Filter with the Disconnect Reason and Disconnect Direction variables.*\n\n###### Reading it:\n*The split separates our-fault backpressure from topology and network faults. large_sendq and charge_resources are this node shedding a peer it cannot keep up with. not_useful and ping_timeout are topology and liveness. read_error and write_error are the transport. graceful, stopping and shutdown are ordinary lifecycle, not faults.*\n\n###### Healthy range:\n*Mostly graceful; the fault reasons near zero.*\n\n###### Watch for:\n*A sustained large_sendq or charge_resources means this node is the bottleneck and is dropping peers \u2014 which removes the very peers it needs to sync from, so a slow node makes itself slower. A rising not_useful or ping_timeout on outbound points at the peer set instead. Distinct from the single unlabelled peer-disconnect total on the Node Health dashboard, which cannot say which of these is happening.*\n\n###### Keywords:\n- **Peer disconnect reason** *(per node)* \u2014 the cause recorded when a peer connection is closed, paired with whether that connection was inbound or outbound.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[PeerImp.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/overlay/detail/PeerImp.cpp)\n\n###### Function:\n`PeerImp::close`\n\n###### References:\n[Resource disconnect](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#resource-disconnect) \u00b7 [Insane / diverged peers](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#insane-diverged-peers)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "displayName": "Peer Disconnect [${__field.labels.series}] ${__field.labels.xrpl_ident}", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short", - "decimals": 0 - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 72 - }, - "id": 17, - "options": { - "displayMode": "gradient", - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "maxVizHeight": 300, - "minVizHeight": 16, - "minVizWidth": 8, - "namePlacement": "left", - "orientation": "horizontal", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showUnfilled": true, - "sizing": "manual", - "valueMode": "color" - }, - "pluginVersion": "13.2.0-28926505616", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_join(sum by (reason, direction, service_instance_id, xrpl_branch, xrpl_work_item) (increase(peer_disconnect_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", reason=~\"$disconnect_reason\", direction=~\"$disconnect_direction\"}[$__range])), \"series\", \" \", \"reason\", \"direction\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", - "refId": "A", - "instant": true - } - ], - "title": "Peer Disconnects (Count By Reason & Direction) [$xrpl_network_type]", - "type": "bargauge", - "repeat": "xrpl_network_type", - "repeatDirection": "h", - "maxPerRow": 2 - }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 82 + "y": 112 }, "id": 18, "panels": [], @@ -1490,7 +1490,7 @@ "h": 10, "w": 12, "x": 0, - "y": 83 + "y": 113 }, "id": 19, "options": { @@ -1556,8 +1556,8 @@ "gridPos": { "h": 10, "w": 12, - "x": 12, - "y": 83 + "x": 0, + "y": 123 }, "id": 20, "options": { @@ -1624,7 +1624,7 @@ "h": 10, "w": 12, "x": 0, - "y": 93 + "y": 133 }, "id": 21, "options": { @@ -1659,6 +1659,149 @@ "repeatDirection": "h", "maxPerRow": 2 }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*How often the server enters a NEW stall episode, as opposed to how long one stall lasts.*\n\n###### How it's computed:\n*Count over the selected range of server_stall_events_total, which counts once per stall episode rather than once per stalled second.*\n\n###### Reading it:\n*Flat at zero is healthy. Read it beside the Server Stall stat: a rising count means repeated fresh stalls, while a flat count with a large stall value means one long unresolved stall.*\n\n###### Healthy range:\n*0.*\n\n###### Watch for:\n*Any repeating count. Recurring short stalls and one long stall have different causes, and this panel is what separates them.*\n\n###### Keywords:\n- **Server stall** *(per node)* \u2014 the main loop failing to check in with the load monitor, measured in seconds of unresponsiveness.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[LoadManager.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/main/LoadManager.cpp)\n\n###### Function:\n`LoadManager::updateStallState`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#server-stall)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short", + "decimals": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 143 + }, + "id": 22, + "options": { + "displayMode": "gradient", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "left", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "manual", + "valueMode": "color" + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(increase(server_stall_events_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__range]), \"series\", \"Stall Episodes\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A", + "instant": true + } + ], + "title": "Server Stall Episodes (Count) [$xrpl_network_type]", + "type": "bargauge", + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Which edges of the sync state machine the node actually traversed over the dashboard window, counted per from-to pair.*\n\n###### How it's computed:\n*increase(state_changes_total) over the selected range, summed by the from and to labels.*\n\n###### Reading it:\n*A clean fresh sync shows single traversals along disconnected to connected to syncing to tracking to full. Repeated counts on the full-to-connected edge paired with connected-to-full is flapping.*\n\n###### Healthy range:\n*One traversal per climb edge and nothing on the reverse edges.*\n\n###### Watch for:\n*High counts on a reverse edge such as full to connected: the node reaches full and keeps losing it, which an unlabelled state-change total cannot distinguish from a clean climb.*\n\n###### Keywords:\n- **Operating mode / server state** *(per node)* \u2014 how fully the node is participating, in ascending order: disconnected, connected, syncing, tracking, full.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`NetworkOPsImp::setMode`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#operating-mode-server-state)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "displayName": "${__field.labels.series}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + } + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 153 + }, + "id": 23, + "options": { + "displayMode": "gradient", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "left", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "manual", + "valueMode": "color" + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_join(sum by (from, to, service_instance_id, xrpl_branch, xrpl_work_item) (increase(state_changes_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", from=~\"$mode_from\", to=~\"$mode_to\"}[$__range])), \"series\", \" \u2192 \", \"from\", \"to\")", + "refId": "A" + } + ], + "title": "Mode Transitions by Edge [$xrpl_network_type]", + "type": "bargauge", + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 + }, { "datasource": { "type": "prometheus", @@ -1728,10 +1871,10 @@ "gridPos": { "h": 10, "w": 12, - "x": 12, - "y": 93 + "x": 0, + "y": 163 }, - "id": 22, + "id": 24, "options": { "annotations": { "clustering": -1, @@ -1766,18 +1909,31 @@ "title": "Ledgers Behind Network", "type": "timeseries" }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 173 + }, + "id": 25, + "panels": [], + "title": "Ledger acquire & SHAMap fetch", + "type": "row" + }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*How often the server enters a NEW stall episode, as opposed to how long one stall lasts.*\n\n###### How it's computed:\n*Count over the selected range of server_stall_events_total, which counts once per stall episode rather than once per stalled second.*\n\n###### Reading it:\n*Flat at zero is healthy. Read it beside the Server Stall stat: a rising count means repeated fresh stalls, while a flat count with a large stall value means one long unresolved stall.*\n\n###### Healthy range:\n*0.*\n\n###### Watch for:\n*Any repeating count. Recurring short stalls and one long stall have different causes, and this panel is what separates them.*\n\n###### Keywords:\n- **Server stall** *(per node)* \u2014 the main loop failing to check in with the load monitor, measured in seconds of unresponsiveness.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[LoadManager.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/main/LoadManager.cpp)\n\n###### Function:\n`LoadManager::updateStallState`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#server-stall)", + "description": "###### What this is:\n*Count of ledger-acquire timeouts where not a single new node arrived since the previous timeout.*\n\n###### How it's computed:\n*count of sync_acquire_no_progress_total, incremented on each acquire timeout whose progress flag was false. The acquire timer fires every 3 seconds at most.*\n\n###### Reading it:\n*Zero means every timeout window saw at least some new data. Any sustained count means acquires are repeatedly timing out with nothing received.*\n\n###### Healthy range:\n*0 on a synced node; brief non-zero bursts during initial sync are normal.*\n\n###### Watch for:\n*A sustained count together with a flat Missing SHAMap Nodes panel: the node is asking and no peer is answering. Check peer count and whether any peer holds the ledger range being requested.*\n\n###### Keywords:\n- **Acquire stall** *(per node)* \u2014 an acquire timeout in which no new SHAMap node was received, so the acquire made no progress at all.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (call-site metric macro, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[InboundLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/InboundLedger.cpp)\n\n###### Function:\n`InboundLedger::onTimer`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#acquire-stall)", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "displayName": "Acquire Stalls ${__field.labels.xrpl_ident}", "thresholds": { "mode": "absolute", "steps": [ @@ -1796,9 +1952,9 @@ "h": 10, "w": 12, "x": 0, - "y": 103 + "y": 174 }, - "id": 23, + "id": 26, "options": { "displayMode": "gradient", "legend": { @@ -1828,12 +1984,12 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "label_replace(label_join(label_replace(increase(server_stall_events_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__range]), \"series\", \"Stall Episodes\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "expr": "label_replace(label_join(label_replace(increase(sync_acquire_no_progress_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__range]), \"series\", \"Stalled Timeouts\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", "refId": "A", "instant": true } ], - "title": "Server Stall Episodes (Count) [$xrpl_network_type]", + "title": "Acquire Stalls — No Progress (Count) [$xrpl_network_type]", "type": "bargauge", "repeat": "xrpl_network_type", "repeatDirection": "h", @@ -1844,13 +2000,13 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*Which edges of the sync state machine the node actually traversed over the dashboard window, counted per from-to pair.*\n\n###### How it's computed:\n*increase(state_changes_total) over the selected range, summed by the from and to labels.*\n\n###### Reading it:\n*A clean fresh sync shows single traversals along disconnected to connected to syncing to tracking to full. Repeated counts on the full-to-connected edge paired with connected-to-full is flapping.*\n\n###### Healthy range:\n*One traversal per climb edge and nothing on the reverse edges.*\n\n###### Watch for:\n*High counts on a reverse edge such as full to connected: the node reaches full and keeps losing it, which an unlabelled state-change total cannot distinguish from a clean climb.*\n\n###### Keywords:\n- **Operating mode / server state** *(per node)* \u2014 how fully the node is participating, in ascending order: disconnected, connected, syncing, tracking, full.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`NetworkOPsImp::setMode`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#operating-mode-server-state)", + "description": "###### What this is:\n*Count of ledger acquires split by whether the local node store already held the whole ledger or the data had to come from peers.*\n\n###### How it's computed:\n*count of sync_acquire_source_total by source, emitted once per new acquire right after the first local-store lookup.*\n\n###### Reading it:\n*network dominant during initial sync is expected \u2014 nothing is local yet. local dominant on a warm node means the store is serving requests without peer traffic.*\n\n###### Healthy range:\n*Mostly local on a warm node with complete history.*\n\n###### Watch for:\n*Sustained network on a node that should already hold the range: the local store is not retaining data, so sync is disk-bound rather than peer-bound. Read with the SHAMap cache hit-rate panel.*\n\n###### Keywords:\n- **Acquire source** *(per node)* \u2014 whether a ledger acquire was satisfied entirely from the local node store or required fetching from peers.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (call-site metric macro, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[InboundLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/InboundLedger.cpp)\n\n###### Function:\n`InboundLedger::init`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#acquire-source)", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, - "displayName": "${__field.labels.series}", + "displayName": "Acquire Source [${__field.labels.series}] ${__field.labels.xrpl_ident}", "thresholds": { "mode": "absolute", "steps": [ @@ -1860,16 +2016,18 @@ } ] }, - "unit": "short" - } + "unit": "short", + "decimals": 0 + }, + "overrides": [] }, "gridPos": { "h": 10, "w": 12, - "x": 12, - "y": 103 + "x": 0, + "y": 184 }, - "id": 24, + "id": 27, "options": { "displayMode": "gradient", "legend": { @@ -1899,29 +2057,17 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "label_join(sum by (from, to, service_instance_id, xrpl_branch, xrpl_work_item) (increase(state_changes_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", from=~\"$mode_from\", to=~\"$mode_to\"}[$__range])), \"series\", \" \u2192 \", \"from\", \"to\")", - "refId": "A" + "expr": "label_replace(label_join(label_replace(sum by (source, service_instance_id, xrpl_branch, xrpl_work_item) (increase(sync_acquire_source_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", source=~\"$acquire_source\"}[$__range])), \"series\", \"$1\", \"source\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A", + "instant": true } ], - "title": "Mode Transitions by Edge [$xrpl_network_type]", + "title": "Acquire Source (Count: Local vs Network) [$xrpl_network_type]", "type": "bargauge", "repeat": "xrpl_network_type", "repeatDirection": "h", "maxPerRow": 2 }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 113 - }, - "id": 25, - "panels": [], - "title": "Ledger acquire & SHAMap fetch", - "type": "row" - }, { "datasource": { "type": "prometheus", @@ -1992,9 +2138,9 @@ "h": 10, "w": 12, "x": 0, - "y": 114 + "y": 194 }, - "id": 26, + "id": 28, "options": { "annotations": { "clustering": -1, @@ -2029,79 +2175,6 @@ "title": "Missing SHAMap Nodes per Acquire (state/tx)", "type": "timeseries" }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "###### What this is:\n*Count of ledger-acquire timeouts where not a single new node arrived since the previous timeout.*\n\n###### How it's computed:\n*count of sync_acquire_no_progress_total, incremented on each acquire timeout whose progress flag was false. The acquire timer fires every 3 seconds at most.*\n\n###### Reading it:\n*Zero means every timeout window saw at least some new data. Any sustained count means acquires are repeatedly timing out with nothing received.*\n\n###### Healthy range:\n*0 on a synced node; brief non-zero bursts during initial sync are normal.*\n\n###### Watch for:\n*A sustained count together with a flat Missing SHAMap Nodes panel: the node is asking and no peer is answering. Check peer count and whether any peer holds the ledger range being requested.*\n\n###### Keywords:\n- **Acquire stall** *(per node)* \u2014 an acquire timeout in which no new SHAMap node was received, so the acquire made no progress at all.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (call-site metric macro, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[InboundLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/InboundLedger.cpp)\n\n###### Function:\n`InboundLedger::onTimer`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#acquire-stall)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "displayName": "Acquire Stalls ${__field.labels.xrpl_ident}", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short", - "decimals": 0 - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 114 - }, - "id": 27, - "options": { - "displayMode": "gradient", - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "maxVizHeight": 300, - "minVizHeight": 16, - "minVizWidth": 8, - "namePlacement": "left", - "orientation": "horizontal", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showUnfilled": true, - "sizing": "manual", - "valueMode": "color" - }, - "pluginVersion": "13.2.0-28926505616", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(increase(sync_acquire_no_progress_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__range]), \"series\", \"Stalled Timeouts\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", - "refId": "A", - "instant": true - } - ], - "title": "Acquire Stalls — No Progress (Count) [$xrpl_network_type]", - "type": "bargauge", - "repeat": "xrpl_network_type", - "repeatDirection": "h", - "maxPerRow": 2 - }, { "datasource": { "type": "prometheus", @@ -2163,10 +2236,10 @@ "gridPos": { "h": 10, "w": 12, - "x": 0, - "y": 124 + "x": 12, + "y": 194 }, - "id": 28, + "id": 29, "options": { "annotations": { "clustering": -1, @@ -2294,10 +2367,10 @@ "gridPos": { "h": 10, "w": 12, - "x": 12, - "y": 124 + "x": 0, + "y": 204 }, - "id": 29, + "id": 30, "options": { "annotations": { "clustering": -1, @@ -2332,79 +2405,6 @@ "title": "Received-Data Stash Depth & In-Flight Acquires", "type": "timeseries" }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "###### What this is:\n*Count of ledger acquires split by whether the local node store already held the whole ledger or the data had to come from peers.*\n\n###### How it's computed:\n*count of sync_acquire_source_total by source, emitted once per new acquire right after the first local-store lookup.*\n\n###### Reading it:\n*network dominant during initial sync is expected \u2014 nothing is local yet. local dominant on a warm node means the store is serving requests without peer traffic.*\n\n###### Healthy range:\n*Mostly local on a warm node with complete history.*\n\n###### Watch for:\n*Sustained network on a node that should already hold the range: the local store is not retaining data, so sync is disk-bound rather than peer-bound. Read with the SHAMap cache hit-rate panel.*\n\n###### Keywords:\n- **Acquire source** *(per node)* \u2014 whether a ledger acquire was satisfied entirely from the local node store or required fetching from peers.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (call-site metric macro, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[InboundLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/InboundLedger.cpp)\n\n###### Function:\n`InboundLedger::init`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#acquire-source)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "displayName": "Acquire Source [${__field.labels.series}] ${__field.labels.xrpl_ident}", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short", - "decimals": 0 - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 134 - }, - "id": 30, - "options": { - "displayMode": "gradient", - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "maxVizHeight": 300, - "minVizHeight": 16, - "minVizWidth": 8, - "namePlacement": "left", - "orientation": "horizontal", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showUnfilled": true, - "sizing": "manual", - "valueMode": "color" - }, - "pluginVersion": "13.2.0-28926505616", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(sum by (source, service_instance_id, xrpl_branch, xrpl_work_item) (increase(sync_acquire_source_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", source=~\"$acquire_source\"}[$__range])), \"series\", \"$1\", \"source\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", - "refId": "A", - "instant": true - } - ], - "title": "Acquire Source (Count: Local vs Network) [$xrpl_network_type]", - "type": "bargauge", - "repeat": "xrpl_network_type", - "repeatDirection": "h", - "maxPerRow": 2 - }, { "datasource": { "type": "prometheus", @@ -2475,7 +2475,7 @@ "h": 10, "w": 12, "x": 12, - "y": 134 + "y": 204 }, "id": 31, "options": { @@ -2518,7 +2518,7 @@ "h": 1, "w": 24, "x": 0, - "y": 144 + "y": 214 }, "id": 32, "panels": [], @@ -2561,7 +2561,7 @@ "h": 10, "w": 12, "x": 0, - "y": 145 + "y": 215 }, "id": 33, "options": { @@ -2657,8 +2657,8 @@ "gridPos": { "h": 10, "w": 12, - "x": 12, - "y": 145 + "x": 0, + "y": 225 }, "id": 34, "options": { @@ -2701,120 +2701,13 @@ "h": 1, "w": 24, "x": 0, - "y": 155 + "y": 235 }, "id": 35, "panels": [], "title": "Quorum & publish", "type": "row" }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "###### What this is:\n*Trusted validations counted at the most recent pre-accept gate, plotted against the number that gate required.*\n\n###### How it's computed:\n*Two series from the same gauge: trusted_validation_tally (agreeing trusted validations seen for the candidate ledger, after the negative-UNL filter) and quorum_target (what the gate demanded). Both are snapshotted on every gate evaluation, whether it passed or failed, so a node that keeps failing still reports both numbers.*\n\n###### Reading it:\n*Read the shape of the tally, not any single value. A tally climbing toward the target is a slow sync that will finish, so keep waiting. A tally flat below the target is stuck: it will never reach quorum on its own, and nothing in the acquire pipeline can fix it. Expect a sawtooth on a healthy node: each series is a snapshot of the most recent gate evaluation, and the first evaluation of every round runs before peer validations arrive, so a sampled low reading between higher ones is normal. Judge it over minutes, and only the sustained floor of the tally against the target carries the signal.*\n\n###### Healthy range:\n*Tally at or above Target, both flat, on a node that is validating.*\n\n###### Watch for:\n*A tally pinned below the target \u2014 too few trusted validators are reachable, or the UNL / negative-UNL configuration excludes the ones that are. Also watch the target jumping to about 9.2e18 (signed 64-bit maximum): that is the explicit quorum-disabled sentinel, meaning too many publishers are unavailable and the trusted list switched quorum off entirely, so the node can never validate however far the tally climbs. It is reported as that maximum rather than wrapping negative precisely so it cannot be misread as a tally that already exceeds its target. Both series flat at 0 means the gate has never been evaluated \u2014 nothing has been offered for validation yet, which sends you back to the Bootstrap row.*\n\n###### Keywords:\n- **Quorum shortfall** *(per node)* \u2014 trusted validations for a candidate ledger falling short of the quorum needed to declare it validated, so the node holds the ledger and still cannot call it validated.\n- **Validation quorum** *(per node)* \u2014 the number of agreeing trusted validations a ledger needs before this node treats it as validated.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerLedgerQuorumPublishGauge`\n\n###### References:\n[Negative UNL and validation quorum on xrpl.org](https://xrpl.org/docs/concepts/consensus-protocol/negative-unl) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#quorum-shortfall)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Validations", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": 1800000, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short" - } - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 156 - }, - "id": 36, - "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, - "legend": { - "calcs": [], - "displayMode": "list", - "enableFacetedFilter": false, - "overflow": "ellipsis", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "13.2.0-28926505616", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(ledger_quorum_publish{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"trusted_validation_tally\"}, \"series\", \"Trusted Tally\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(ledger_quorum_publish{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"quorum_target\"}, \"series\", \"Quorum Target\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", - "refId": "B" - } - ], - "title": "Trusted Validations vs Quorum Target", - "type": "timeseries" - }, { "datasource": { "type": "prometheus", @@ -2844,10 +2737,10 @@ "gridPos": { "h": 10, "w": 12, - "x": 12, - "y": 156 + "x": 0, + "y": 236 }, - "id": 37, + "id": 36, "options": { "displayMode": "gradient", "legend": { @@ -2924,9 +2817,9 @@ "h": 10, "w": 12, "x": 0, - "y": 166 + "y": 246 }, - "id": 38, + "id": 37, "options": { "colorMode": "value", "graphMode": "none", @@ -2959,6 +2852,113 @@ "repeatDirection": "h", "maxPerRow": 2 }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Trusted validations counted at the most recent pre-accept gate, plotted against the number that gate required.*\n\n###### How it's computed:\n*Two series from the same gauge: trusted_validation_tally (agreeing trusted validations seen for the candidate ledger, after the negative-UNL filter) and quorum_target (what the gate demanded). Both are snapshotted on every gate evaluation, whether it passed or failed, so a node that keeps failing still reports both numbers.*\n\n###### Reading it:\n*Read the shape of the tally, not any single value. A tally climbing toward the target is a slow sync that will finish, so keep waiting. A tally flat below the target is stuck: it will never reach quorum on its own, and nothing in the acquire pipeline can fix it. Expect a sawtooth on a healthy node: each series is a snapshot of the most recent gate evaluation, and the first evaluation of every round runs before peer validations arrive, so a sampled low reading between higher ones is normal. Judge it over minutes, and only the sustained floor of the tally against the target carries the signal.*\n\n###### Healthy range:\n*Tally at or above Target, both flat, on a node that is validating.*\n\n###### Watch for:\n*A tally pinned below the target \u2014 too few trusted validators are reachable, or the UNL / negative-UNL configuration excludes the ones that are. Also watch the target jumping to about 9.2e18 (signed 64-bit maximum): that is the explicit quorum-disabled sentinel, meaning too many publishers are unavailable and the trusted list switched quorum off entirely, so the node can never validate however far the tally climbs. It is reported as that maximum rather than wrapping negative precisely so it cannot be misread as a tally that already exceeds its target. Both series flat at 0 means the gate has never been evaluated \u2014 nothing has been offered for validation yet, which sends you back to the Bootstrap row.*\n\n###### Keywords:\n- **Quorum shortfall** *(per node)* \u2014 trusted validations for a candidate ledger falling short of the quorum needed to declare it validated, so the node holds the ledger and still cannot call it validated.\n- **Validation quorum** *(per node)* \u2014 the number of agreeing trusted validations a ledger needs before this node treats it as validated.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerLedgerQuorumPublishGauge`\n\n###### References:\n[Negative UNL and validation quorum on xrpl.org](https://xrpl.org/docs/concepts/consensus-protocol/negative-unl) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#quorum-shortfall)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Validations", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + } + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 256 + }, + "id": 38, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(ledger_quorum_publish{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"trusted_validation_tally\"}, \"series\", \"Trusted Tally\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(ledger_quorum_publish{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"quorum_target\"}, \"series\", \"Quorum Target\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "B" + } + ], + "title": "Trusted Validations vs Quorum Target", + "type": "timeseries" + }, { "datasource": { "type": "prometheus", @@ -3029,7 +3029,7 @@ "h": 10, "w": 12, "x": 12, - "y": 166 + "y": 256 }, "id": 39, "options": { @@ -3090,7 +3090,7 @@ "h": 10, "w": 12, "x": 0, - "y": 176 + "y": 266 }, "id": 40, "options": { @@ -3194,7 +3194,7 @@ "h": 10, "w": 12, "x": 12, - "y": 176 + "y": 266 }, "id": 41, "options": { @@ -3245,7 +3245,7 @@ "h": 1, "w": 24, "x": 0, - "y": 186 + "y": 276 }, "id": 42, "panels": [], @@ -3304,7 +3304,7 @@ "h": 10, "w": 12, "x": 0, - "y": 187 + "y": 277 }, "id": 43, "options": { @@ -3339,109 +3339,6 @@ "repeatDirection": "h", "maxPerRow": 2 }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "###### What this is:\n*Whether an amendment this build does not support has reached majority on the network.*\n\n###### How it's computed:\n*amendment_block series warned: 1 while an unsupported amendment holds majority, 0 otherwise.*\n\n###### Reading it:\n*0 is healthy. A 1 is the first warning that an upgrade is required, and it is raised before the amendment activates rather than after.*\n\n###### Healthy range:\n*0.*\n\n###### Watch for:\n*The transition from 0 to 1 \u2014 that is the moment the upgrade clock starts. Read Amendment Block Countdown next for how long is left, and the Amendment Blocked stat on the Validator Health dashboard for whether the block has already happened; that one is the terminal state, this one is the warning. Which amendment is blocking is not a label (an arbitrary 256-bit amendment id would be unbounded cardinality); the hash is logged by AmendmentTableImpl::doValidatedLedger and is available via Loki.*\n\n###### Keywords:\n- **Amendment warning** *(per node)* \u2014 an unsupported amendment has reached majority; the node still validates, but will stop when that amendment activates.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerAmendmentBlockGauge`\n\n###### References:\n[Amendments on xrpl.org](https://xrpl.org/docs/concepts/networks-and-servers/amendments) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#amendment-blocked)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Warned", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": 1800000, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1 - } - ] - }, - "unit": "short" - } - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 187 - }, - "id": 44, - "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, - "legend": { - "calcs": [], - "displayMode": "list", - "enableFacetedFilter": false, - "overflow": "ellipsis", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "13.2.0-28926505616", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(amendment_block{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"warned\"}, \"series\", \"Warned\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", - "refId": "A" - } - ], - "title": "Amendment Warned", - "type": "timeseries" - }, { "datasource": { "type": "prometheus", @@ -3472,9 +3369,9 @@ "h": 10, "w": 12, "x": 0, - "y": 197 + "y": 287 }, - "id": 45, + "id": 44, "options": { "displayMode": "gradient", "legend": { @@ -3544,10 +3441,10 @@ "gridPos": { "h": 10, "w": 12, - "x": 12, - "y": 197 + "x": 0, + "y": 297 }, - "id": 46, + "id": 45, "options": { "displayMode": "gradient", "legend": { @@ -3588,19 +3485,423 @@ "repeatDirection": "h", "maxPerRow": 2 }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Whether an amendment this build does not support has reached majority on the network.*\n\n###### How it's computed:\n*amendment_block series warned: 1 while an unsupported amendment holds majority, 0 otherwise.*\n\n###### Reading it:\n*0 is healthy. A 1 is the first warning that an upgrade is required, and it is raised before the amendment activates rather than after.*\n\n###### Healthy range:\n*0.*\n\n###### Watch for:\n*The transition from 0 to 1 \u2014 that is the moment the upgrade clock starts. Read Amendment Block Countdown next for how long is left, and the Amendment Blocked stat on the Validator Health dashboard for whether the block has already happened; that one is the terminal state, this one is the warning. Which amendment is blocking is not a label (an arbitrary 256-bit amendment id would be unbounded cardinality); the hash is logged by AmendmentTableImpl::doValidatedLedger and is available via Loki.*\n\n###### Keywords:\n- **Amendment warning** *(per node)* \u2014 an unsupported amendment has reached majority; the node still validates, but will stop when that amendment activates.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerAmendmentBlockGauge`\n\n###### References:\n[Amendments on xrpl.org](https://xrpl.org/docs/concepts/networks-and-servers/amendments) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#amendment-blocked)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Warned", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "short" + } + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 307 + }, + "id": 46, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(amendment_block{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"warned\"}, \"series\", \"Warned\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + } + ], + "title": "Amendment Warned", + "type": "timeseries" + }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 207 + "y": 317 }, "id": 47, "panels": [], "title": "Back-fill & persistence", "type": "row" }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Number of times ledger-replay sub-tasks give up and fall back to acquiring a whole ledger, split by which sub-task gave up.*\n\n###### How it's computed:\n*Count of ledger_replay_fallback_total by stage. The skip-list stage and the delta stage each emit once, on the transition into fallback, from the branch that was debug-log-only before.*\n\n###### Reading it:\n*Zero when replay-capable peers are available. Any sustained count means the replay optimisation is being defeated and back-fill has reverted to the slower full-acquire path.*\n\n###### Healthy range:\n*Zero, or brief spikes while the peer set changes.*\n\n###### Watch for:\n*A persistent count on either stage: too few connected peers support the ledger-replay feature, so every historical ledger is fetched in full instead of as a delta. Read with Replay Outcomes.*\n\n###### Keywords:\n- **Replay fallback** *(per node)* \u2014 a replay sub-task abandoning the delta shortcut and acquiring the entire ledger instead.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[SkipListAcquire.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/SkipListAcquire.cpp)\n\n###### Function:\n`SkipListAcquire::trigger` / `LedgerDeltaAcquire::trigger`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#replay-fallback)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "displayName": "Replay Fallback [${__field.labels.series}] ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short", + "decimals": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 318 + }, + "id": 48, + "options": { + "displayMode": "gradient", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "left", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "manual", + "valueMode": "color" + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (stage, service_instance_id, xrpl_branch, xrpl_work_item) (increase(ledger_replay_fallback_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", stage=~\"$replay_stage\"}[$__range])), \"series\", \"$1\", \"stage\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A", + "instant": true + } + ], + "title": "Replay Fallback To Full Acquire (Count By Stage) [$xrpl_network_type]", + "type": "bargauge", + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Count of ledger-replay tasks reaching each terminal state: success, timeout, build failure or parameter failure.*\n\n###### How it's computed:\n*Count of ledger_replay_outcome_total by outcome. Every terminal path in the replay task emits exactly once; before this, all four only set an internal flag and wrote a log line.*\n\n###### Reading it:\n*Successes only is healthy. Timeouts mean deltas never arrived; build failures mean a delta would not apply to its parent; parameter failures mean a peer served an inconsistent skip list.*\n\n###### Healthy range:\n*Successes non-zero while replaying, all failure outcomes at zero.*\n\n###### Watch for:\n*Any failure outcome climbing while successes stay flat: replay is running but never completing, so history back-fill is silently falling back to the slower path. The outcome value tells you which layer to look at \u2014 timeouts point at peers, build and parameter failures point at the data those peers served.*\n\n###### Keywords:\n- **Ledger replay** *(per node)* \u2014 rebuilding a range of historical ledgers from a start ledger plus per-ledger deltas instead of downloading each one whole.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[LedgerReplayTask.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/LedgerReplayTask.cpp)\n\n###### Function:\n`LedgerReplayTask::recordOutcome`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledger-replay)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "displayName": "Replay Outcome [${__field.labels.series}] ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short", + "decimals": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 328 + }, + "id": 49, + "options": { + "displayMode": "gradient", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "left", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "manual", + "valueMode": "color" + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (outcome, service_instance_id, xrpl_branch, xrpl_work_item) (increase(ledger_replay_outcome_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", outcome=~\"$replay_outcome\"}[$__range])), \"series\", \"$1\", \"outcome\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A", + "instant": true + } + ], + "title": "Replay Outcomes (Count By Terminal State) [$xrpl_network_type]", + "type": "bargauge", + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Two rates side by side: minor page faults taken inside the heap trim, and the resident memory the trim actually returned to the kernel. Together they say whether the trim is buying anything for what it costs.*\n\n###### How it's computed:\n*Count of sweep_malloc_trim_minor_faults_total and of sweep_malloc_trim_reclaimed_kb_total. Both are cumulative counters exported per sweep, so the panel rates them rather than plotting the totals. Reclaim is published only when resident memory actually fell; a sweep during which another thread grew the heap contributes nothing rather than a negative amount.*\n\n###### Reading it:\n*Reclaimed volume with a near-zero fault count is a cheap, useful trim. A fault count that moves with the reclaimed volume means the pages are being handed back and immediately taken again, which is churn rather than savings.*\n\n###### Healthy range:\n*A reclaimed volume that tracks cache turnover, with faults near zero.*\n\n###### Watch for:\n*Reclaim near zero while the trim duration panel shows real time being spent: the trim is walking the heap and freeing nothing, which is pure cost. IMPORTANT LIMITATION: the fault delta is scoped to the trim call, so it cannot show the re-fault cost paid later as the caches refill and touch the returned pages. That later cost is real but is NOT measured here; do not read a low fault count as proof the trim was free.*\n\n###### Keywords:\n- **Minor page fault** *(per node)* \u2014 a memory access satisfied without disk I/O, by mapping a page the kernel already holds.\n- **Reclaimed resident memory** *(per node)* \u2014 resident kilobytes the allocator handed back to the kernel.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[Application.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/main/Application.cpp)\n\n###### Function:\n`ApplicationImp::trimHeapAndRecord`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#minor-page-fault)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short", + "decimals": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 338 + }, + "id": 50, + "options": { + "displayMode": "gradient", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "left", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "manual", + "valueMode": "color" + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_work_item) (increase(sweep_malloc_trim_minor_faults_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__range])), \"series\", \"Minor Faults\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A", + "instant": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_work_item) (increase(sweep_malloc_trim_reclaimed_kb_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__range])), \"series\", \"Reclaimed KB\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "B", + "instant": true + } + ], + "title": "Sweep Heap-Trim Faults & Reclaim (Count) [$xrpl_network_type]", + "type": "bargauge", + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*Number of times a rotation has to rescue a tree node that was missing from BOTH backends and rewrite it from memory. Each one is an extra write on top of the whole-state-map walk the rotation already performs.*\n\n###### How it's computed:\n*Count of rotation_copy_node_restore_total. Incremented once per rescued node in the rotation's state-map walk, from a branch that was warn-log-only before, so the volume was invisible unless someone was reading logs.*\n\n###### Reading it:\n*Zero on a healthy node. Any sustained count means clean nodes reachable from the validated state map have no on-disk copy left, because the backend holding them was removed by an earlier rotation and they were never rewritten.*\n\n###### Healthy range:\n*Flat at zero.*\n\n###### Watch for:\n*A non-zero count is the signal that earlier rotations dropped data the current state map still needs: each rescue is a write that competes with sync I/O, and without the rescue the node would later surface as an unresolvable missing-node error. Read with the copy-forward panel \u2014 both are rotation-time writes, but this one also indicates prior data loss rather than merely cost. The node hash is deliberately not a label (unbounded cardinality); get it from the copyNode warning in Loki, correlated by node and time.*\n\n###### Keywords:\n- **Node re-store** *(per node)* \u2014 rewriting an in-memory tree node whose only on-disk copy was removed by an earlier rotation.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[SHAMapStoreImp.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/SHAMapStoreImp.cpp)\n\n###### Function:\n`SHAMapStoreImp::copyNode`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#node-re-store)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short", + "decimals": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 348 + }, + "id": 51, + "options": { + "displayMode": "gradient", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "left", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "manual", + "valueMode": "color" + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_work_item) (increase(rotation_copy_node_restore_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__range])), \"series\", \"Nodes Re-Stored\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A", + "instant": true + } + ], + "title": "Rotation Node Re-Stores (Count) [$xrpl_network_type]", + "type": "bargauge", + "repeat": "xrpl_network_type", + "repeatDirection": "h", + "maxPerRow": 2 + }, { "datasource": { "type": "prometheus", @@ -3663,9 +3964,9 @@ "h": 10, "w": 12, "x": 0, - "y": 208 + "y": 358 }, - "id": 48, + "id": 52, "options": { "annotations": { "clustering": -1, @@ -3770,9 +4071,9 @@ "h": 10, "w": 12, "x": 12, - "y": 208 + "y": 358 }, - "id": 49, + "id": 53, "options": { "annotations": { "clustering": -1, @@ -3815,152 +4116,6 @@ "title": "NodeStore Operation Rate (writes vs reads)", "type": "timeseries" }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "###### What this is:\n*Number of times ledger-replay sub-tasks give up and fall back to acquiring a whole ledger, split by which sub-task gave up.*\n\n###### How it's computed:\n*Count of ledger_replay_fallback_total by stage. The skip-list stage and the delta stage each emit once, on the transition into fallback, from the branch that was debug-log-only before.*\n\n###### Reading it:\n*Zero when replay-capable peers are available. Any sustained count means the replay optimisation is being defeated and back-fill has reverted to the slower full-acquire path.*\n\n###### Healthy range:\n*Zero, or brief spikes while the peer set changes.*\n\n###### Watch for:\n*A persistent count on either stage: too few connected peers support the ledger-replay feature, so every historical ledger is fetched in full instead of as a delta. Read with Replay Outcomes.*\n\n###### Keywords:\n- **Replay fallback** *(per node)* \u2014 a replay sub-task abandoning the delta shortcut and acquiring the entire ledger instead.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[SkipListAcquire.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/SkipListAcquire.cpp)\n\n###### Function:\n`SkipListAcquire::trigger` / `LedgerDeltaAcquire::trigger`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#replay-fallback)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "displayName": "Replay Fallback [${__field.labels.series}] ${__field.labels.xrpl_ident}", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short", - "decimals": 0 - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 218 - }, - "id": 50, - "options": { - "displayMode": "gradient", - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "maxVizHeight": 300, - "minVizHeight": 16, - "minVizWidth": 8, - "namePlacement": "left", - "orientation": "horizontal", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showUnfilled": true, - "sizing": "manual", - "valueMode": "color" - }, - "pluginVersion": "13.2.0-28926505616", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(sum by (stage, service_instance_id, xrpl_branch, xrpl_work_item) (increase(ledger_replay_fallback_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", stage=~\"$replay_stage\"}[$__range])), \"series\", \"$1\", \"stage\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", - "refId": "A", - "instant": true - } - ], - "title": "Replay Fallback To Full Acquire (Count By Stage) [$xrpl_network_type]", - "type": "bargauge", - "repeat": "xrpl_network_type", - "repeatDirection": "h", - "maxPerRow": 2 - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "###### What this is:\n*Count of ledger-replay tasks reaching each terminal state: success, timeout, build failure or parameter failure.*\n\n###### How it's computed:\n*Count of ledger_replay_outcome_total by outcome. Every terminal path in the replay task emits exactly once; before this, all four only set an internal flag and wrote a log line.*\n\n###### Reading it:\n*Successes only is healthy. Timeouts mean deltas never arrived; build failures mean a delta would not apply to its parent; parameter failures mean a peer served an inconsistent skip list.*\n\n###### Healthy range:\n*Successes non-zero while replaying, all failure outcomes at zero.*\n\n###### Watch for:\n*Any failure outcome climbing while successes stay flat: replay is running but never completing, so history back-fill is silently falling back to the slower path. The outcome value tells you which layer to look at \u2014 timeouts point at peers, build and parameter failures point at the data those peers served.*\n\n###### Keywords:\n- **Ledger replay** *(per node)* \u2014 rebuilding a range of historical ledgers from a start ledger plus per-ledger deltas instead of downloading each one whole.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[LedgerReplayTask.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/LedgerReplayTask.cpp)\n\n###### Function:\n`LedgerReplayTask::recordOutcome`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#ledger-replay)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "displayName": "Replay Outcome [${__field.labels.series}] ${__field.labels.xrpl_ident}", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short", - "decimals": 0 - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 218 - }, - "id": 51, - "options": { - "displayMode": "gradient", - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "maxVizHeight": 300, - "minVizHeight": 16, - "minVizWidth": 8, - "namePlacement": "left", - "orientation": "horizontal", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showUnfilled": true, - "sizing": "manual", - "valueMode": "color" - }, - "pluginVersion": "13.2.0-28926505616", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(sum by (outcome, service_instance_id, xrpl_branch, xrpl_work_item) (increase(ledger_replay_outcome_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", outcome=~\"$replay_outcome\"}[$__range])), \"series\", \"$1\", \"outcome\", \"(.*)\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", - "refId": "A", - "instant": true - } - ], - "title": "Replay Outcomes (Count By Terminal State) [$xrpl_network_type]", - "type": "bargauge", - "repeat": "xrpl_network_type", - "repeatDirection": "h", - "maxPerRow": 2 - }, { "datasource": { "type": "prometheus", @@ -4023,9 +4178,9 @@ "h": 10, "w": 12, "x": 0, - "y": 228 + "y": 368 }, - "id": 52, + "id": 54, "options": { "annotations": { "clustering": -1, @@ -4068,88 +4223,6 @@ "title": "Sweep Heap-Trim Duration (p50/p95)", "type": "timeseries" }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "###### What this is:\n*Two rates side by side: minor page faults taken inside the heap trim, and the resident memory the trim actually returned to the kernel. Together they say whether the trim is buying anything for what it costs.*\n\n###### How it's computed:\n*Count of sweep_malloc_trim_minor_faults_total and of sweep_malloc_trim_reclaimed_kb_total. Both are cumulative counters exported per sweep, so the panel rates them rather than plotting the totals. Reclaim is published only when resident memory actually fell; a sweep during which another thread grew the heap contributes nothing rather than a negative amount.*\n\n###### Reading it:\n*Reclaimed volume with a near-zero fault count is a cheap, useful trim. A fault count that moves with the reclaimed volume means the pages are being handed back and immediately taken again, which is churn rather than savings.*\n\n###### Healthy range:\n*A reclaimed volume that tracks cache turnover, with faults near zero.*\n\n###### Watch for:\n*Reclaim near zero while the trim duration panel shows real time being spent: the trim is walking the heap and freeing nothing, which is pure cost. IMPORTANT LIMITATION: the fault delta is scoped to the trim call, so it cannot show the re-fault cost paid later as the caches refill and touch the returned pages. That later cost is real but is NOT measured here; do not read a low fault count as proof the trim was free.*\n\n###### Keywords:\n- **Minor page fault** *(per node)* \u2014 a memory access satisfied without disk I/O, by mapping a page the kernel already holds.\n- **Reclaimed resident memory** *(per node)* \u2014 resident kilobytes the allocator handed back to the kernel.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[Application.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/main/Application.cpp)\n\n###### Function:\n`ApplicationImp::trimHeapAndRecord`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#minor-page-fault)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short", - "decimals": 0 - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 228 - }, - "id": 53, - "options": { - "displayMode": "gradient", - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "maxVizHeight": 300, - "minVizHeight": 16, - "minVizWidth": 8, - "namePlacement": "left", - "orientation": "horizontal", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showUnfilled": true, - "sizing": "manual", - "valueMode": "color" - }, - "pluginVersion": "13.2.0-28926505616", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_work_item) (increase(sweep_malloc_trim_minor_faults_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__range])), \"series\", \"Minor Faults\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", - "refId": "A", - "instant": true - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_work_item) (increase(sweep_malloc_trim_reclaimed_kb_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__range])), \"series\", \"Reclaimed KB\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", - "refId": "B", - "instant": true - } - ], - "title": "Sweep Heap-Trim Faults & Reclaim (Count) [$xrpl_network_type]", - "type": "bargauge", - "repeat": "xrpl_network_type", - "repeatDirection": "h", - "maxPerRow": 2 - }, { "datasource": { "type": "prometheus", @@ -4261,10 +4334,10 @@ "gridPos": { "h": 10, "w": 12, - "x": 0, - "y": 238 + "x": 12, + "y": 368 }, - "id": 54, + "id": 55, "options": { "annotations": { "clustering": -1, @@ -4307,18 +4380,31 @@ "title": "Online-Delete Rotation Window & Copy-Forward Writes", "type": "timeseries" }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 378 + }, + "id": 56, + "panels": [], + "title": "Spans & traces", + "type": "row" + }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*Number of times a rotation has to rescue a tree node that was missing from BOTH backends and rewrite it from memory. Each one is an extra write on top of the whole-state-map walk the rotation already performs.*\n\n###### How it's computed:\n*Count of rotation_copy_node_restore_total. Incremented once per rescued node in the rotation's state-map walk, from a branch that was warn-log-only before, so the volume was invisible unless someone was reading logs.*\n\n###### Reading it:\n*Zero on a healthy node. Any sustained count means clean nodes reachable from the validated state map have no on-disk copy left, because the backend holding them was removed by an earlier rotation and they were never rewritten.*\n\n###### Healthy range:\n*Flat at zero.*\n\n###### Watch for:\n*A non-zero count is the signal that earlier rotations dropped data the current state map still needs: each rescue is a write that competes with sync I/O, and without the rescue the node would later surface as an unresolvable missing-node error. Read with the copy-forward panel \u2014 both are rotation-time writes, but this one also indicates prior data loss rather than merely cost. The node hash is deliberately not a label (unbounded cardinality); get it from the copyNode warning in Loki, correlated by node and time.*\n\n###### Keywords:\n- **Node re-store** *(per node)* \u2014 rewriting an in-memory tree node whose only on-disk copy was removed by an earlier rotation.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[SHAMapStoreImp.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/SHAMapStoreImp.cpp)\n\n###### Function:\n`SHAMapStoreImp::copyNode`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#node-re-store)", + "description": "###### What this is:\n*How many ledger-acquire phases finished each way over the selected range, as whole counts.*\n\n###### How it's computed:\n*`increase(span_calls_total[$__range])` per `span_name` and `outcome`, so the number shown is the count of phases that ended that way inside the dashboard time range.*\n\n###### Reading it:\n*Counts, not rates. These are discrete, low-frequency events: a count of 0.14 is 278 phases over 33 minutes, which no reader can infer from the rate. Compare the Complete and Abandoned bars for the same phase. Header needs one round trip; both tree phases need many.*\n\n###### Healthy range:\n*Abandoned near zero for every phase, and Complete rising with ledger progress.*\n\n###### Watch for:\n*Abandoned dominating the tree phases while Header still completes. That means requests reach peers and headers arrive, but the many-round-trip phases never finish. `Abandoned` here means the acquire object was destroyed while still fetching, not that it timed out \u2014 see Acquire Stalls \u2014 No Progress (Count) and the runbook for the sweep mechanism.*\n\n###### Keywords:\n- **Acquire phase** *(per node)* \u2014 one of the three sequential fetches of a ledger: header, account-state tree, transaction tree.\n- **Abandoned** *(per node)* \u2014 the acquire was destroyed with no result, i.e. dropped while still fetching rather than failing or timing out.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Spans are emitted by xrpld code; the collector's spanmetrics connector turns them into `span_calls_total`; the Grafana query counts the increase over the range.*\n\n###### Source:\n[InboundLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/InboundLedger.cpp)\n\n###### Function:\n`InboundLedger::endPhaseSpan`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#fresh-node-sync-diagnostics)", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "displayName": "Acquire Phase [${__field.labels.series}] ${__field.labels.xrpl_ident}", "thresholds": { "mode": "absolute", "steps": [ @@ -4336,10 +4422,10 @@ "gridPos": { "h": 10, "w": 12, - "x": 12, - "y": 238 + "x": 0, + "y": 379 }, - "id": 55, + "id": 57, "options": { "displayMode": "gradient", "legend": { @@ -4369,30 +4455,17 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_work_item) (increase(rotation_copy_node_restore_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__range])), \"series\", \"Nodes Re-Stored\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "expr": "label_replace(label_join(label_join(label_replace(label_replace(label_replace(label_replace(label_replace(label_replace(label_replace(sum by (span_name, outcome, service_instance_id, xrpl_branch, xrpl_work_item) (round(increase(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", outcome=~\"$span_outcome\", span_name=~\"ledger\\\\.acquire\\\\..*\"}[$__range]))), \"phase\", \"Header\", \"span_name\", \"ledger\\\\.acquire\\\\.header\"), \"phase\", \"Account State Tree\", \"span_name\", \"ledger\\\\.acquire\\\\.astree\"), \"phase\", \"Transaction Tree\", \"span_name\", \"ledger\\\\.acquire\\\\.txtree\"), \"outcome_t\", \"Complete\", \"outcome\", \"complete\"), \"outcome_t\", \"Abandoned\", \"outcome\", \"abandoned\"), \"outcome_t\", \"Timeout\", \"outcome\", \"timeout\"), \"outcome_t\", \"Failed\", \"outcome\", \"failed\"), \"series\", \" \", \"phase\", \"outcome_t\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", "refId": "A", "instant": true } ], - "title": "Rotation Node Re-Stores (Count) [$xrpl_network_type]", + "title": "Ledger Acquire Phase Outcomes (Count By Phase & Outcome) [$xrpl_network_type]", "type": "bargauge", "repeat": "xrpl_network_type", "repeatDirection": "h", "maxPerRow": 2 }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 248 - }, - "id": 56, - "panels": [], - "title": "Spans & traces", - "type": "row" - }, { "datasource": { "type": "prometheus", @@ -4455,9 +4528,9 @@ "h": 10, "w": 12, "x": 0, - "y": 249 + "y": 389 }, - "id": 57, + "id": 58, "options": { "annotations": { "clustering": -1, @@ -4508,79 +4581,6 @@ "title": "Ledger Acquire Phase Duration (p95 by phase)", "type": "timeseries" }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "###### What this is:\n*How many ledger-acquire phases finished each way over the selected range, as whole counts.*\n\n###### How it's computed:\n*`increase(span_calls_total[$__range])` per `span_name` and `outcome`, so the number shown is the count of phases that ended that way inside the dashboard time range.*\n\n###### Reading it:\n*Counts, not rates. These are discrete, low-frequency events: a count of 0.14 is 278 phases over 33 minutes, which no reader can infer from the rate. Compare the Complete and Abandoned bars for the same phase. Header needs one round trip; both tree phases need many.*\n\n###### Healthy range:\n*Abandoned near zero for every phase, and Complete rising with ledger progress.*\n\n###### Watch for:\n*Abandoned dominating the tree phases while Header still completes. That means requests reach peers and headers arrive, but the many-round-trip phases never finish. `Abandoned` here means the acquire object was destroyed while still fetching, not that it timed out \u2014 see Acquire Stalls \u2014 No Progress (Count) and the runbook for the sweep mechanism.*\n\n###### Keywords:\n- **Acquire phase** *(per node)* \u2014 one of the three sequential fetches of a ledger: header, account-state tree, transaction tree.\n- **Abandoned** *(per node)* \u2014 the acquire was destroyed with no result, i.e. dropped while still fetching rather than failing or timing out.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Spans are emitted by xrpld code; the collector's spanmetrics connector turns them into `span_calls_total`; the Grafana query counts the increase over the range.*\n\n###### Source:\n[InboundLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/InboundLedger.cpp)\n\n###### Function:\n`InboundLedger::endPhaseSpan`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#fresh-node-sync-diagnostics)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "displayName": "Acquire Phase [${__field.labels.series}] ${__field.labels.xrpl_ident}", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short", - "decimals": 0 - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 249 - }, - "id": 58, - "options": { - "displayMode": "gradient", - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "maxVizHeight": 300, - "minVizHeight": 16, - "minVizWidth": 8, - "namePlacement": "left", - "orientation": "horizontal", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showUnfilled": true, - "sizing": "manual", - "valueMode": "color" - }, - "pluginVersion": "13.2.0-28926505616", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_join(label_replace(label_replace(label_replace(label_replace(label_replace(label_replace(label_replace(sum by (span_name, outcome, service_instance_id, xrpl_branch, xrpl_work_item) (round(increase(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", outcome=~\"$span_outcome\", span_name=~\"ledger\\\\.acquire\\\\..*\"}[$__range]))), \"phase\", \"Header\", \"span_name\", \"ledger\\\\.acquire\\\\.header\"), \"phase\", \"Account State Tree\", \"span_name\", \"ledger\\\\.acquire\\\\.astree\"), \"phase\", \"Transaction Tree\", \"span_name\", \"ledger\\\\.acquire\\\\.txtree\"), \"outcome_t\", \"Complete\", \"outcome\", \"complete\"), \"outcome_t\", \"Abandoned\", \"outcome\", \"abandoned\"), \"outcome_t\", \"Timeout\", \"outcome\", \"timeout\"), \"outcome_t\", \"Failed\", \"outcome\", \"failed\"), \"series\", \" \", \"phase\", \"outcome_t\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", - "refId": "A", - "instant": true - } - ], - "title": "Ledger Acquire Phase Outcomes (Count By Phase & Outcome) [$xrpl_network_type]", - "type": "bargauge", - "repeat": "xrpl_network_type", - "repeatDirection": "h", - "maxPerRow": 2 - }, { "datasource": { "type": "prometheus", @@ -4642,8 +4642,8 @@ "gridPos": { "h": 10, "w": 12, - "x": 0, - "y": 259 + "x": 12, + "y": 389 }, "id": 59, "options": { @@ -4741,8 +4741,8 @@ "gridPos": { "h": 10, "w": 12, - "x": 12, - "y": 259 + "x": 0, + "y": 399 }, "id": 60, "options": { @@ -4840,8 +4840,8 @@ "gridPos": { "h": 10, "w": 12, - "x": 0, - "y": 269 + "x": 12, + "y": 399 }, "id": 61, "options": { @@ -4939,8 +4939,8 @@ "gridPos": { "h": 10, "w": 12, - "x": 12, - "y": 269 + "x": 0, + "y": 409 }, "id": 62, "options": { @@ -5039,8 +5039,8 @@ "gridPos": { "h": 10, "w": 12, - "x": 0, - "y": 279 + "x": 12, + "y": 409 }, "id": 63, "options": {