From d45bdeee9d79d154e23d3b95c393a49a65e73bd7 Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Thu, 6 Aug 2026 18:35:51 +0100 Subject: [PATCH] added core filters to queries and legends Signed-off-by: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> --- .../grafana/dashboards/consensus-health.json | 18 ++--- .../dashboards/log-derived-insights.json | 72 +++++++++---------- .../grafana/dashboards/node-health.json | 26 +++---- .../dashboards/transaction-overview.json | 4 +- docs/telemetry-runbook.md | 53 +++++++------- 5 files changed, 87 insertions(+), 86 deletions(-) diff --git a/docker/telemetry/grafana/dashboards/consensus-health.json b/docker/telemetry/grafana/dashboards/consensus-health.json index cd1e796a33..e77e207c10 100644 --- a/docker/telemetry/grafana/dashboards/consensus-health.json +++ b/docker/telemetry/grafana/dashboards/consensus-health.json @@ -279,7 +279,7 @@ "uid": "${DS_TEMPO}" }, "queryType": "traceql", - "query": "{name=\"consensus.accept\" && resource.service.instance.id=~\"$node\"} | quantile_over_time(span.round_time_ms, .5)", + "query": "{name=\"consensus.accept\" && resource.service.instance.id=~\"$node\" && resource.service.name=~\"$service_name\" && resource.deployment.environment=~\"$deployment_environment\" && resource.xrpl.network.type=~\"$xrpl_network_type\"} | quantile_over_time(span.round_time_ms, .5)", "legendFormat": "P50 Time to Consensus [{{resource.service.instance.id}}]", "refId": "A" }, @@ -290,7 +290,7 @@ }, "queryType": "traceql", "refId": "B", - "query": "{name=\"consensus.accept\" && resource.service.instance.id=~\"$node\"} | quantile_over_time(span.round_time_ms, .95)", + "query": "{name=\"consensus.accept\" && resource.service.instance.id=~\"$node\" && resource.service.name=~\"$service_name\" && resource.deployment.environment=~\"$deployment_environment\" && resource.xrpl.network.type=~\"$xrpl_network_type\"} | quantile_over_time(span.round_time_ms, .95)", "legendFormat": "P95 Time to Consensus [{{resource.service.instance.id}}]" } ], @@ -333,7 +333,7 @@ "uid": "${DS_TEMPO}" }, "queryType": "traceql", - "query": "{name=\"consensus.accept\" && resource.service.instance.id=~\"$node\"} | avg_over_time(span.round_time_ms)", + "query": "{name=\"consensus.accept\" && resource.service.instance.id=~\"$node\" && resource.service.name=~\"$service_name\" && resource.deployment.environment=~\"$deployment_environment\" && resource.xrpl.network.type=~\"$xrpl_network_type\"} | avg_over_time(span.round_time_ms)", "legendFormat": "Avg Time to Consensus [{{resource.service.instance.id}}]", "refId": "A" } @@ -383,7 +383,7 @@ }, "queryType": "traceql", "refId": "A", - "query": "{name=\"consensus.establish\" && resource.service.instance.id=~\"$node\"} | count_over_time() by (span.establish_count)", + "query": "{name=\"consensus.establish\" && resource.service.instance.id=~\"$node\" && resource.service.name=~\"$service_name\" && resource.deployment.environment=~\"$deployment_environment\" && resource.xrpl.network.type=~\"$xrpl_network_type\"} | count_over_time() by (span.establish_count)", "legendFormat": "{{span.establish_count}} Rounds" } ], @@ -427,7 +427,7 @@ "uid": "${DS_TEMPO}" }, "queryType": "traceql", - "query": "{name=\"consensus.round\" && resource.service.instance.id=~\"$node\"} | quantile_over_time(span.previous_round_time_ms, .95)", + "query": "{name=\"consensus.round\" && resource.service.instance.id=~\"$node\" && resource.service.name=~\"$service_name\" && resource.deployment.environment=~\"$deployment_environment\" && resource.xrpl.network.type=~\"$xrpl_network_type\"} | quantile_over_time(span.previous_round_time_ms, .95)", "legendFormat": "P95 Previous Round [{{resource.service.instance.id}}]", "refId": "A" } @@ -730,7 +730,7 @@ "uid": "${DS_TEMPO}" }, "queryType": "traceql", - "query": "{name=\"consensus.accept.apply\" && resource.service.instance.id=~\"$node\"} | avg_over_time(span.close_time_vote_bins)", + "query": "{name=\"consensus.accept.apply\" && resource.service.instance.id=~\"$node\" && resource.service.name=~\"$service_name\" && resource.deployment.environment=~\"$deployment_environment\" && resource.xrpl.network.type=~\"$xrpl_network_type\"} | avg_over_time(span.close_time_vote_bins)", "legendFormat": "Avg Vote Bins", "refId": "A" }, @@ -740,7 +740,7 @@ "uid": "${DS_TEMPO}" }, "queryType": "traceql", - "query": "{name=\"consensus.accept.apply\" && resource.service.instance.id=~\"$node\"} | avg_over_time(span.close_resolution_ms)", + "query": "{name=\"consensus.accept.apply\" && resource.service.instance.id=~\"$node\" && resource.service.name=~\"$service_name\" && resource.deployment.environment=~\"$deployment_environment\" && resource.xrpl.network.type=~\"$xrpl_network_type\"} | avg_over_time(span.close_resolution_ms)", "legendFormat": "Avg Resolution (ms)", "refId": "B" } @@ -789,7 +789,7 @@ "uid": "${DS_TEMPO}" }, "queryType": "traceql", - "query": "{name=\"consensus.accept.apply\" && resource.service.instance.id=~\"$node\" && span.resolution_direction=~\"$resolution_direction\"} | count_over_time() by (span.resolution_direction)", + "query": "{name=\"consensus.accept.apply\" && resource.service.instance.id=~\"$node\" && resource.service.name=~\"$service_name\" && resource.deployment.environment=~\"$deployment_environment\" && resource.xrpl.network.type=~\"$xrpl_network_type\" && span.resolution_direction=~\"$resolution_direction\"} | count_over_time() by (span.resolution_direction)", "legendFormat": "{{span.resolution_direction}}", "refId": "A" } @@ -861,7 +861,7 @@ "uid": "${DS_TEMPO}" }, "queryType": "traceql", - "query": "{name=\"consensus.accept.apply\" && resource.service.instance.id=~\"$node\"} | count_over_time() by (span.close_time_vote_bins)", + "query": "{name=\"consensus.accept.apply\" && resource.service.instance.id=~\"$node\" && resource.service.name=~\"$service_name\" && resource.deployment.environment=~\"$deployment_environment\" && resource.xrpl.network.type=~\"$xrpl_network_type\"} | count_over_time() by (span.close_time_vote_bins)", "legendFormat": "{{span.close_time_vote_bins}} Distinct Position(s)", "refId": "A" } diff --git a/docker/telemetry/grafana/dashboards/log-derived-insights.json b/docker/telemetry/grafana/dashboards/log-derived-insights.json index 6937748bc8..8f7aedfbd5 100644 --- a/docker/telemetry/grafana/dashboards/log-derived-insights.json +++ b/docker/telemetry/grafana/dashboards/log-derived-insights.json @@ -312,10 +312,10 @@ "uid": "${DS_LOKI}" }, "refId": "A", - "expr": "topk($topn, sum by (service_instance_id) (count_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | severity =~ `ERR|FTL` [$__range])))", + "expr": "topk($topn, sum by (service_instance_id, xrpl_network_type) (count_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | severity =~ `ERR|FTL` [$__range])))", "instant": true, "queryType": "instant", - "legendFormat": "Errors [{{service_instance_id}}]" + "legendFormat": "Errors [{{service_instance_id}}] [{{xrpl_network_type}}]" } ], "id": 3 @@ -383,10 +383,10 @@ "uid": "${DS_LOKI}" }, "refId": "A", - "expr": "topk($topn, sum by (service_instance_id) (count_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `Resource` |~ `useless data|unwanted data|init drop` [$__range])))", + "expr": "topk($topn, sum by (service_instance_id, xrpl_network_type) (count_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `Resource` |~ `useless data|unwanted data|init drop` [$__range])))", "instant": true, "queryType": "instant", - "legendFormat": "Charges [{{service_instance_id}}]" + "legendFormat": "Charges [{{service_instance_id}}] [{{xrpl_network_type}}]" } ], "id": 4 @@ -454,10 +454,10 @@ "uid": "${DS_LOKI}" }, "refId": "A", - "expr": "topk($topn, sum by (service_instance_id) (sum_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `Resource` | regexp `\\(\\$(?P[0-9]+)\\)` | unwrap fee [$__range])))", + "expr": "topk($topn, sum by (service_instance_id, xrpl_network_type) (sum_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `Resource` | regexp `\\(\\$(?P[0-9]+)\\)` | unwrap fee [$__range])))", "instant": true, "queryType": "instant", - "legendFormat": "Fee Charged [{{service_instance_id}}]" + "legendFormat": "Fee Charged [{{service_instance_id}}] [{{xrpl_network_type}}]" } ], "id": 5 @@ -525,10 +525,10 @@ "uid": "${DS_LOKI}" }, "refId": "A", - "expr": "topk($topn, sum by (service_instance_id) (count_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `ManifestCache` |~ `Manifest: (Stale|Invalid|Revoked|UntrustedCapacity)` [$__range])))", + "expr": "topk($topn, sum by (service_instance_id, xrpl_network_type) (count_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `ManifestCache` |~ `Manifest: (Stale|Invalid|Revoked|UntrustedCapacity)` [$__range])))", "instant": true, "queryType": "instant", - "legendFormat": "Rejections [{{service_instance_id}}]" + "legendFormat": "Rejections [{{service_instance_id}}] [{{xrpl_network_type}}]" } ], "id": 6 @@ -596,10 +596,10 @@ "uid": "${DS_LOKI}" }, "refId": "A", - "expr": "topk($topn, sum by (service_instance_id) (count_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `LedgerConsensus` | severity =~ `WRN|ERR|FTL` [$__range])))", + "expr": "topk($topn, sum by (service_instance_id, xrpl_network_type) (count_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `LedgerConsensus` | severity =~ `WRN|ERR|FTL` [$__range])))", "instant": true, "queryType": "instant", - "legendFormat": "Consensus Events [{{service_instance_id}}]" + "legendFormat": "Consensus Events [{{service_instance_id}}] [{{xrpl_network_type}}]" } ], "id": 7 @@ -667,10 +667,10 @@ "uid": "${DS_LOKI}" }, "refId": "A", - "expr": "topk($topn, sum by (service_instance_id) (count_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `LoadMonitor` |~ `Job: .* run: [0-9]+ms wait: [0-9]+ms` [$__range])))", + "expr": "topk($topn, sum by (service_instance_id, xrpl_network_type) (count_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `LoadMonitor` |~ `Job: .* run: [0-9]+ms wait: [0-9]+ms` [$__range])))", "instant": true, "queryType": "instant", - "legendFormat": "Breaches [{{service_instance_id}}]" + "legendFormat": "Breaches [{{service_instance_id}}] [{{xrpl_network_type}}]" } ], "id": 8 @@ -738,10 +738,10 @@ "uid": "${DS_LOKI}" }, "refId": "A", - "expr": "topk($topn, sum by (service_instance_id) (count_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" |~ `STATE->(syncing|connected|tracking|disconnected)` [$__range])))", + "expr": "topk($topn, sum by (service_instance_id, xrpl_network_type) (count_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" |~ `STATE->(syncing|connected|tracking|disconnected)` [$__range])))", "instant": true, "queryType": "instant", - "legendFormat": "Transitions [{{service_instance_id}}]" + "legendFormat": "Transitions [{{service_instance_id}}] [{{xrpl_network_type}}]" } ], "id": 9 @@ -809,10 +809,10 @@ "uid": "${DS_LOKI}" }, "refId": "A", - "expr": "topk($topn, sum by (service_instance_id) (sum_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `InboundLedger` | regexp `dupe:(?P[0-9]+)` | unwrap dupe [$__range])))", + "expr": "topk($topn, sum by (service_instance_id, xrpl_network_type) (sum_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `InboundLedger` | regexp `dupe:(?P[0-9]+)` | unwrap dupe [$__range])))", "instant": true, "queryType": "instant", - "legendFormat": "Duplicates [{{service_instance_id}}]" + "legendFormat": "Duplicates [{{service_instance_id}}] [{{xrpl_network_type}}]" } ], "id": 10 @@ -1199,8 +1199,8 @@ "uid": "${DS_LOKI}" }, "refId": "A", - "legendFormat": "Accepted [{{service_instance_id}}]", - "expr": "sum by (service_instance_id) (rate({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `ManifestCache` | severity =~ \"$severity\" |~ `Manifest: Accepted` [$__auto])) or vector(0)" + "legendFormat": "Accepted [{{service_instance_id}}] [{{xrpl_network_type}}]", + "expr": "sum by (service_instance_id, xrpl_network_type) (rate({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `ManifestCache` | severity =~ \"$severity\" |~ `Manifest: Accepted` [$__auto])) or vector(0)" }, { "datasource": { @@ -1208,8 +1208,8 @@ "uid": "${DS_LOKI}" }, "refId": "B", - "legendFormat": "Rejected [{{service_instance_id}}]", - "expr": "sum by (service_instance_id) (rate({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `ManifestCache` | severity =~ \"$severity\" |~ `Manifest: (Stale|Invalid|Revoked|UntrustedCapacity)` [$__auto])) or vector(0)" + "legendFormat": "Rejected [{{service_instance_id}}] [{{xrpl_network_type}}]", + "expr": "sum by (service_instance_id, xrpl_network_type) (rate({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `ManifestCache` | severity =~ \"$severity\" |~ `Manifest: (Stale|Invalid|Revoked|UntrustedCapacity)` [$__auto])) or vector(0)" } ], "id": 19, @@ -1716,8 +1716,8 @@ "uid": "${DS_LOKI}" }, "refId": "A", - "legendFormat": "Duplicate Ratio [{{service_instance_id}}]", - "expr": "sum by (service_instance_id) (sum_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `InboundLedger` | severity =~ \"$severity\" | regexp `dupe:(?P[0-9]+)` | unwrap dupe [$__auto])) / (sum by (service_instance_id) (sum_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `InboundLedger` | severity =~ \"$severity\" | regexp `dupe:(?P[0-9]+)` | unwrap dupe [$__auto])) + sum by (service_instance_id) (sum_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `InboundLedger` | severity =~ \"$severity\" | regexp `good:(?P[0-9]+)` | unwrap good [$__auto])))" + "legendFormat": "Duplicate Ratio [{{service_instance_id}}] [{{xrpl_network_type}}]", + "expr": "sum by (service_instance_id, xrpl_network_type) (sum_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `InboundLedger` | severity =~ \"$severity\" | regexp `dupe:(?P[0-9]+)` | unwrap dupe [$__auto])) / (sum by (service_instance_id, xrpl_network_type) (sum_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `InboundLedger` | severity =~ \"$severity\" | regexp `dupe:(?P[0-9]+)` | unwrap dupe [$__auto])) + sum by (service_instance_id, xrpl_network_type) (sum_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `InboundLedger` | severity =~ \"$severity\" | regexp `good:(?P[0-9]+)` | unwrap good [$__auto])))" } ], "id": 26, @@ -1775,8 +1775,8 @@ "uid": "${DS_LOKI}" }, "refId": "A", - "legendFormat": "Good [{{service_instance_id}}]", - "expr": "sum by (service_instance_id) (sum_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `InboundLedger` | severity =~ \"$severity\" | regexp `good:(?P[0-9]+)` | unwrap good [$__auto]))" + "legendFormat": "Good [{{service_instance_id}}] [{{xrpl_network_type}}]", + "expr": "sum by (service_instance_id, xrpl_network_type) (sum_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `InboundLedger` | severity =~ \"$severity\" | regexp `good:(?P[0-9]+)` | unwrap good [$__auto]))" }, { "datasource": { @@ -1784,8 +1784,8 @@ "uid": "${DS_LOKI}" }, "refId": "B", - "legendFormat": "Duplicate [{{service_instance_id}}]", - "expr": "sum by (service_instance_id) (sum_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `InboundLedger` | severity =~ \"$severity\" | regexp `dupe:(?P[0-9]+)` | unwrap dupe [$__auto]))" + "legendFormat": "Duplicate [{{service_instance_id}}] [{{xrpl_network_type}}]", + "expr": "sum by (service_instance_id, xrpl_network_type) (sum_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `InboundLedger` | severity =~ \"$severity\" | regexp `dupe:(?P[0-9]+)` | unwrap dupe [$__auto]))" }, { "datasource": { @@ -1793,8 +1793,8 @@ "uid": "${DS_LOKI}" }, "refId": "C", - "legendFormat": "Acquire Timeouts [{{service_instance_id}}]", - "expr": "sum by (service_instance_id) (sum_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `InboundLedger` | severity =~ \"$severity\" | regexp `Acquire \\S+ timeouts:(?P[0-9]+)` | unwrap timeouts [$__auto]))" + "legendFormat": "Acquire Timeouts [{{service_instance_id}}] [{{xrpl_network_type}}]", + "expr": "sum by (service_instance_id, xrpl_network_type) (sum_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `InboundLedger` | severity =~ \"$severity\" | regexp `Acquire \\S+ timeouts:(?P[0-9]+)` | unwrap timeouts [$__auto]))" } ], "id": 27, @@ -1924,8 +1924,8 @@ "uid": "${DS_LOKI}" }, "refId": "A", - "legendFormat": "Handshake [{{service_instance_id}}]", - "expr": "sum by (service_instance_id) (rate({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `PeerFinder` | severity =~ \"$severity\" |~ `Logic handshake` [$__auto]))" + "legendFormat": "Handshake [{{service_instance_id}}] [{{xrpl_network_type}}]", + "expr": "sum by (service_instance_id, xrpl_network_type) (rate({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `PeerFinder` | severity =~ \"$severity\" |~ `Logic handshake` [$__auto]))" }, { "datasource": { @@ -1933,8 +1933,8 @@ "uid": "${DS_LOKI}" }, "refId": "B", - "legendFormat": "Accept [{{service_instance_id}}]", - "expr": "sum by (service_instance_id) (rate({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `PeerFinder` | severity =~ \"$severity\" |~ `Logic accept` [$__auto]))" + "legendFormat": "Accept [{{service_instance_id}}] [{{xrpl_network_type}}]", + "expr": "sum by (service_instance_id, xrpl_network_type) (rate({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `PeerFinder` | severity =~ \"$severity\" |~ `Logic accept` [$__auto]))" } ], "id": 30, @@ -2124,8 +2124,8 @@ "uid": "${DS_LOKI}" }, "refId": "A", - "legendFormat": "Quorum [{{service_instance_id}}]", - "expr": "max by (service_instance_id) (max_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `ValidatorList` | severity =~ \"$severity\" | regexp `Using quorum of (?P[0-9]+) for new set of (?P[0-9]+) trusted` | unwrap quorum [$__auto]))" + "legendFormat": "Quorum [{{service_instance_id}}] [{{xrpl_network_type}}]", + "expr": "max by (service_instance_id, xrpl_network_type) (max_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `ValidatorList` | severity =~ \"$severity\" | regexp `Using quorum of (?P[0-9]+) for new set of (?P[0-9]+) trusted` | unwrap quorum [$__auto]))" }, { "datasource": { @@ -2133,8 +2133,8 @@ "uid": "${DS_LOKI}" }, "refId": "B", - "legendFormat": "Trusted Validators [{{service_instance_id}}]", - "expr": "max by (service_instance_id) (max_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `ValidatorList` | severity =~ \"$severity\" | regexp `Using quorum of (?P[0-9]+) for new set of (?P[0-9]+) trusted` | unwrap total [$__auto]))" + "legendFormat": "Trusted Validators [{{service_instance_id}}] [{{xrpl_network_type}}]", + "expr": "max by (service_instance_id, xrpl_network_type) (max_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `ValidatorList` | severity =~ \"$severity\" | regexp `Using quorum of (?P[0-9]+) for new set of (?P[0-9]+) trusted` | unwrap total [$__auto]))" } ], "id": 34, diff --git a/docker/telemetry/grafana/dashboards/node-health.json b/docker/telemetry/grafana/dashboards/node-health.json index 41597900e1..e73637bf26 100644 --- a/docker/telemetry/grafana/dashboards/node-health.json +++ b/docker/telemetry/grafana/dashboards/node-health.json @@ -1177,7 +1177,7 @@ "uid": "${DS_PROMETHEUS}" }, "expr": "server_info{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"server_state\"}", - "legendFormat": "State [{{service_instance_id}}]", + "legendFormat": "State [{{service_instance_id}}] [{{xrpl_network_type}}]", "refId": "A" } ], @@ -3177,7 +3177,7 @@ }, "expr": "server_info{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"server_state\"}", "instant": false, - "legendFormat": "{{service_instance_id}} [{{xrpl_branch}}]", + "legendFormat": "{{service_instance_id}} [{{xrpl_network_type}}] [{{xrpl_branch}}]", "queryType": "range", "range": true, "refId": "A" @@ -3241,7 +3241,7 @@ }, "expr": "server_info{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"uptime\"}", "instant": true, - "legendFormat": "{{service_instance_id}}", + "legendFormat": "{{service_instance_id}} [{{xrpl_network_type}}]", "refId": "A" } ], @@ -3311,7 +3311,7 @@ }, "expr": "server_info{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"peers\"}", "instant": true, - "legendFormat": "{{service_instance_id}}", + "legendFormat": "{{service_instance_id}} [{{xrpl_network_type}}]", "refId": "A" } ], @@ -3385,7 +3385,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*The spread in validated ledger sequence across all selected nodes.*\n\n###### How it's computed:\n*Highest validated ledger sequence minus the lowest, across the selected nodes.*\n\n###### Reading it:\n*0 means every node agrees on the same validated ledger; larger means they diverge.*\n\n###### Healthy range:\n*0 to 1 ledger in steady state.*\n\n###### Watch for:\n*A sustained spread above a few ledgers means some nodes are lagging or the fleet is diverging.*\n\n###### Keywords:\n- **Validated ledger** *(network-wide)* \u2014 a ledger confirmed final by the trusted validator quorum; its contents never change.\n\n###### Computation boundary:\n*Result: Across the selected nodes \u2014 the query aggregates instances into one series.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerServerInfoGauge`\n\n###### References:\n[Validated ledger](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#validated-ledger)", + "description": "###### What this is:\n*The spread in validated ledger sequence across all selected nodes.*\n\n###### How it's computed:\n*Highest validated ledger sequence minus the lowest, among the selected nodes on the same network.*\n\n###### Reading it:\n*0 means every node agrees on the same validated ledger; larger means they diverge.*\n\n###### Healthy range:\n*0 to 1 ledger in steady state.*\n\n###### Watch for:\n*A sustained spread above a few ledgers means some nodes are lagging or the fleet is diverging.*\n\n###### Keywords:\n- **Validated ledger** *(network-wide)* \u2014 a ledger confirmed final by the trusted validator quorum; its contents never change.\n\n###### Computation boundary:\n*Result: Per network \u2014 the query aggregates the selected nodes into one series for each `xrpl_network_type`, so mainnet and devnet sequences are never subtracted from each other.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerServerInfoGauge`\n\n###### References:\n[Validated ledger](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#validated-ledger)", "fieldConfig": { "defaults": { "color": { @@ -3467,15 +3467,15 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "max(server_info{metric=\"validated_ledger_seq\",service_name=~\"$service_name\",service_instance_id=~\"$node\",xrpl_network_type=~\"$xrpl_network_type\",xrpl_branch=~\"$xrpl_branch\",xrpl_node_role=~\"$xrpl_node_role\",deployment_environment=~\"$deployment_environment\",xrpl_work_item=~\"$xrpl_work_item\"}) - min(server_info{metric=\"validated_ledger_seq\",service_name=~\"$service_name\",service_instance_id=~\"$node\",xrpl_network_type=~\"$xrpl_network_type\",xrpl_branch=~\"$xrpl_branch\",xrpl_node_role=~\"$xrpl_node_role\",deployment_environment=~\"$deployment_environment\",xrpl_work_item=~\"$xrpl_work_item\"})", + "expr": "max by (xrpl_network_type) (server_info{metric=\"validated_ledger_seq\",service_name=~\"$service_name\",service_instance_id=~\"$node\",xrpl_network_type=~\"$xrpl_network_type\",xrpl_branch=~\"$xrpl_branch\",xrpl_node_role=~\"$xrpl_node_role\",deployment_environment=~\"$deployment_environment\",xrpl_work_item=~\"$xrpl_work_item\"}) - min by (xrpl_network_type) (server_info{metric=\"validated_ledger_seq\",service_name=~\"$service_name\",service_instance_id=~\"$node\",xrpl_network_type=~\"$xrpl_network_type\",xrpl_branch=~\"$xrpl_branch\",xrpl_node_role=~\"$xrpl_node_role\",deployment_environment=~\"$deployment_environment\",xrpl_work_item=~\"$xrpl_work_item\"})", "instant": true, - "legendFormat": "Spread", + "legendFormat": "Spread [{{xrpl_network_type}}]", "queryType": "instant", "range": false, "refId": "A" } ], - "title": "Validated Ledger Seq \u2014 Convergence (Max \u2212 Min)", + "title": "Validated Ledger Seq \u2014 Convergence (Max \u2212 Min, per network)", "type": "stat" }, { @@ -3483,7 +3483,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*How far each node's validated ledger lags behind the network tip, in ledgers.*\n\n###### How it's computed:\n*Highest validated ledger sequence across the selected nodes, minus each node's own sequence.*\n\n###### Reading it:\n*0 means the node is at the tip; larger values mean it trails further behind.*\n\n###### Healthy range:\n*0 to 1 ledger on a synced node.*\n\n###### Watch for:\n*A node stuck at a growing value is falling behind and not keeping up with consensus.*\n\n###### Keywords:\n- **Validated ledger** *(network-wide)* \u2014 a ledger confirmed final by the trusted validator quorum; its contents never change.\n- **Consensus** *(network event)* \u2014 the protocol by which validators agree on the next ledger's transaction set and close time.\n\n###### Computation boundary:\n*Result: Across the selected nodes \u2014 the query aggregates instances into one series.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerServerInfoGauge`\n\n###### References:\n[Validated ledger](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) \u00b7 [Consensus](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#validated-ledger)", + "description": "###### What this is:\n*How far each node's validated ledger lags behind the network tip, in ledgers.*\n\n###### How it's computed:\n*Highest validated ledger sequence within the node's own network, minus each node's own sequence.*\n\n###### Reading it:\n*0 means the node is at the tip; larger values mean it trails further behind.*\n\n###### Healthy range:\n*0 to 1 ledger on a synced node.*\n\n###### Watch for:\n*A node stuck at a growing value is falling behind and not keeping up with consensus.*\n\n###### Keywords:\n- **Validated ledger** *(network-wide)* \u2014 a ledger confirmed final by the trusted validator quorum; its contents never change.\n- **Consensus** *(network event)* \u2014 the protocol by which validators agree on the next ledger's transaction set and close time.\n\n###### Computation boundary:\n*Result: Per node \u2014 each node is compared against the highest sequence on its own `xrpl_network_type`, so mainnet and devnet sequences are never subtracted from each other.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerServerInfoGauge`\n\n###### References:\n[Validated ledger](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) \u00b7 [Consensus](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#validated-ledger)", "fieldConfig": { "defaults": { "color": { @@ -3533,15 +3533,15 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "sort_desc(scalar(max(server_info{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"validated_ledger_seq\"})) - server_info{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"validated_ledger_seq\"})", + "expr": "sort_desc(max by (xrpl_network_type) (server_info{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"validated_ledger_seq\"}) - on(xrpl_network_type) group_right() server_info{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"validated_ledger_seq\"})", "instant": true, - "legendFormat": "{{service_instance_id}}", + "legendFormat": "{{service_instance_id}} [{{xrpl_network_type}}]", "queryType": "instant", "range": false, "refId": "A" } ], - "title": "Validated Ledger Seq \u2014 Current (Stat)", + "title": "Validated Ledger Seq \u2014 Lag Behind Network Tip", "type": "stat" }, { @@ -4218,7 +4218,7 @@ "expr": "complete_ledgers{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}", "format": "table", "instant": true, - "legendFormat": "{{bound}} [range {{index}}] [{{service_instance_id}}]", + "legendFormat": "{{bound}} [range {{index}}] [{{service_instance_id}}] [{{xrpl_network_type}}]", "refId": "A" } ], diff --git a/docker/telemetry/grafana/dashboards/transaction-overview.json b/docker/telemetry/grafana/dashboards/transaction-overview.json index ecb9450172..f8aa73bad7 100644 --- a/docker/telemetry/grafana/dashboards/transaction-overview.json +++ b/docker/telemetry/grafana/dashboards/transaction-overview.json @@ -474,9 +474,9 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "sum by (service_instance_id) (increase(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"txq.accept_tx\", txq_status=\"applied\"}[$__rate_interval]))\n/\nsum by (service_instance_id) (increase(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"txq.accept_tx\", txq_status=~\"applied|failed\"}[$__rate_interval]))", + "expr": "sum by (service_instance_id, xrpl_network_type) (increase(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"txq.accept_tx\", txq_status=\"applied\"}[$__rate_interval]))\n/\nsum by (service_instance_id, xrpl_network_type) (increase(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"txq.accept_tx\", txq_status=~\"applied|failed\"}[$__rate_interval]))", "interval": "15s", - "legendFormat": "{{service_instance_id}}" + "legendFormat": "{{service_instance_id}} [{{xrpl_network_type}}]" } ], "fieldConfig": { diff --git a/docs/telemetry-runbook.md b/docs/telemetry-runbook.md index 49b2827a45..d1a8c25152 100644 --- a/docs/telemetry-runbook.md +++ b/docs/telemetry-runbook.md @@ -1929,32 +1929,33 @@ Requires `trace_peer=1` in the `[telemetry]` config section. ### Node Health -- System Metrics (`node-health`) -| Panel | Type | PromQL | Labels Used | -| -------------------------------------- | ---------- | ---------------------------------------------------------- | ---------------- | -| Validated Ledger Age | stat | `ledgermaster_validated_ledger_age` | — | -| Published Ledger Age | stat | `ledgermaster_published_ledger_age` | — | -| Operating Mode (Time Share) | timeseries | `rate(state_accounting_X_duration) / sum(rate(all modes))` | — | -| Operating Mode Transitions | timeseries | `state_accounting_*_transitions` | — | -| I/O Latency | timeseries | `histogram_quantile(0.95, ios_latency_bucket)` | — | -| Job Queue Depth | timeseries | `jobq_job_count` | — | -| Ledger Fetch Rate | stat | `rate(ledger_fetches[5m])` | — | -| Ledger History Mismatches | stat | `rate(ledger_history_mismatch[5m])` | — | -| Key Jobs Execution Time | timeseries | `acceptledger{quantile="$quantile"}` (+ 10 more key jobs) | `quantile` | -| Key Jobs Dequeue Wait Time | timeseries | `acceptledger_q{quantile="$quantile"}` (+ 10 more) | `quantile` | -| FullBelowCache Size | timeseries | `node_family_full_below_cache_size` | — | -| FullBelowCache Hit Rate | gauge | `node_family_full_below_cache_hit_rate` | — | -| Ledger Publish Gap | stat | `Published_Ledger_Age - Validated_Ledger_Age` | — | -| State Duration Rate (Full vs Tracking) | timeseries | `rate(state_accounting_full_duration[5m]) / 1000000` | — | -| All Jobs Execution Time (Detail) | timeseries | `{__name__=~"", quantile="$quantile"}` | `quantile` | -| All Jobs Dequeue Wait (Detail) | timeseries | `{__name__=~"_q", quantile="$quantile"}` | `quantile` | -| Server State | stat | `server_info{metric="server_state"}` | `metric` | -| Uptime | stat | `server_info{metric="uptime"}` | `metric` | -| Peer Count | stat | `server_info{metric="peers"}` | `metric` | -| Validated Ledger Seq | stat | `server_info{metric="validated_ledger_seq"}` | `metric` | -| Build Version | stat | `build_info` | `version` | -| Complete Ledger Ranges | table | `complete_ledgers` | `bound`, `index` | -| Database Sizes | timeseries | `db_metrics{metric=~"db_kb_.*"}` | `metric` | -| Historical Fetch Rate | stat | `db_metrics{metric="historical_perminute"}` | `metric` | +| Panel | Type | PromQL | Labels Used | +| ------------------------------------------------------------ | ---------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------- | +| Validated Ledger Age | stat | `ledgermaster_validated_ledger_age` | — | +| Published Ledger Age | stat | `ledgermaster_published_ledger_age` | — | +| Operating Mode (Time Share) | timeseries | `rate(state_accounting_X_duration) / sum(rate(all modes))` | — | +| Operating Mode Transitions | timeseries | `state_accounting_*_transitions` | — | +| I/O Latency | timeseries | `histogram_quantile(0.95, ios_latency_bucket)` | — | +| Job Queue Depth | timeseries | `jobq_job_count` | — | +| Ledger Fetch Rate | stat | `rate(ledger_fetches[5m])` | — | +| Ledger History Mismatches | stat | `rate(ledger_history_mismatch[5m])` | — | +| Key Jobs Execution Time | timeseries | `acceptledger{quantile="$quantile"}` (+ 10 more key jobs) | `quantile` | +| Key Jobs Dequeue Wait Time | timeseries | `acceptledger_q{quantile="$quantile"}` (+ 10 more) | `quantile` | +| FullBelowCache Size | timeseries | `node_family_full_below_cache_size` | — | +| FullBelowCache Hit Rate | gauge | `node_family_full_below_cache_hit_rate` | — | +| Ledger Publish Gap | stat | `Published_Ledger_Age - Validated_Ledger_Age` | — | +| State Duration Rate (Full vs Tracking) | timeseries | `rate(state_accounting_full_duration[5m]) / 1000000` | — | +| All Jobs Execution Time (Detail) | timeseries | `{__name__=~"", quantile="$quantile"}` | `quantile` | +| All Jobs Dequeue Wait (Detail) | timeseries | `{__name__=~"_q", quantile="$quantile"}` | `quantile` | +| Server State | stat | `server_info{metric="server_state"}` | `metric` | +| Uptime | stat | `server_info{metric="uptime"}` | `metric` | +| Peer Count | stat | `server_info{metric="peers"}` | `metric` | +| Validated Ledger Seq -- Lag Behind Network Tip | stat | `max by (xrpl_network_type) (server_info{metric="validated_ledger_seq"}) - on(xrpl_network_type) group_right() server_info{metric="validated_ledger_seq"}` | `metric` | +| Validated Ledger Seq -- Convergence (Max - Min, per network) | stat | `max by (xrpl_network_type) (server_info{metric="validated_ledger_seq"}) - min by (xrpl_network_type) (server_info{metric="validated_ledger_seq"})` | `metric` | +| Build Version | stat | `build_info` | `version` | +| Complete Ledger Ranges | table | `complete_ledgers` | `bound`, `index` | +| Database Sizes | timeseries | `db_metrics{metric=~"db_kb_.*"}` | `metric` | +| Historical Fetch Rate | stat | `db_metrics{metric="historical_perminute"}` | `metric` | ### Network Traffic -- System Metrics (`network-traffic`)