diff --git a/OpenTelemetryPlan/09-data-collection-reference.md b/OpenTelemetryPlan/09-data-collection-reference.md index 38e9356705..82afb25adb 100644 --- a/OpenTelemetryPlan/09-data-collection-reference.md +++ b/OpenTelemetryPlan/09-data-collection-reference.md @@ -912,7 +912,7 @@ Bidirectional linking between logs and traces is configured via Grafana datasour ### Loki Backend -Grafana Loki (v3.4.2) serves as the log storage backend. It receives log entries from the OTel Collector's `otlphttp/loki` exporter via the native OTLP endpoint at `http://loki:3100/otlp`. +Grafana Loki (v3.7.6) serves as the log storage backend. It receives log entries from the OTel Collector's `otlphttp/loki` exporter via the native OTLP endpoint at `http://loki:3100/otlp`. ### LogQL Query Examples diff --git a/OpenTelemetryPlan/Phase8_taskList.md b/OpenTelemetryPlan/Phase8_taskList.md index 7849990858..8c4d80e80b 100644 --- a/OpenTelemetryPlan/Phase8_taskList.md +++ b/OpenTelemetryPlan/Phase8_taskList.md @@ -76,7 +76,7 @@ - Add Loki service: ```yaml loki: - image: grafana/loki:3.4.2 + image: grafana/loki:3.7.6 ports: - "3100:3100" command: -config.file=/etc/loki/local-config.yaml diff --git a/docker/telemetry/docker-compose.yml b/docker/telemetry/docker-compose.yml index cffe3d139a..8aa2c3df64 100644 --- a/docker/telemetry/docker-compose.yml +++ b/docker/telemetry/docker-compose.yml @@ -24,7 +24,7 @@ services: # OpenTelemetry Collector: receives spans from xrpld via OTLP protocol, # batches them for efficiency, and forwards to Tempo for storage. otel-collector: - image: otel/opentelemetry-collector-contrib:0.121.0 + image: otel/opentelemetry-collector-contrib:0.158.0 command: ["--config=/etc/otel-collector-config.yaml"] ports: - "4317:4317" # OTLP gRPC @@ -52,7 +52,7 @@ services: # Grafana Tempo: distributed tracing backend that stores and indexes # spans. Queryable via TraceQL in Grafana Explore. tempo: - image: grafana/tempo:2.7.2 + image: grafana/tempo:2.9.4 command: ["-config.file=/etc/tempo.yaml"] ports: - "3200:3200" # Tempo HTTP API (health check, query) @@ -69,7 +69,7 @@ services: # Collector exports via otlphttp to Loki's /otlp endpoint. # Query logs via Grafana Explore -> Loki at http://localhost:3000. loki: - image: grafana/loki:3.4.2 + image: grafana/loki:3.7.6 ports: - "3100:3100" command: -config.file=/etc/loki/local-config.yaml @@ -79,8 +79,8 @@ services: - xrpld-telemetry prometheus: - # Pinned to the v2.53 LTS line for reproducible, config-stable runs. - image: prom/prometheus:v2.53.3 + # Pinned to an exact patch release for reproducible, config-stable runs. + image: prom/prometheus:v3.13.2 ports: - "9090:9090" volumes: @@ -94,7 +94,7 @@ services: # Grafana: visualization UI with Tempo pre-configured as a datasource. # Anonymous admin access enabled for local development convenience. grafana: - image: grafana/grafana:11.5.2 + image: grafana/grafana:13.1.2 # Alerting secrets/addresses (Slack webhook, alert email) come from this # gitignored file; Grafana expands the ${VARS} referenced in the alerting # provisioning YAML. Absent file = unset vars = receivers have no live @@ -110,6 +110,10 @@ services: # the callback URL is how the renderer fetches the panel from grafana. - GF_RENDERING_SERVER_URL=http://renderer:8081/render - GF_RENDERING_CALLBACK_URL=http://grafana:3000/ + # Grafana 13 turns the renderAuthJWT feature toggle on by default, which + # makes the renderer reject requests unless both sides share a token that + # is not the "-" default. Must match AUTH_TOKEN on the renderer service. + - GF_RENDERING_RENDERER_TOKEN=${GF_RENDERING_RENDERER_TOKEN:-xrpld-local-render} # SMTP for the critical-tier email receiver and the Slack webhook / email # address for the contact points all come from the env_file above, which # injects them straight into the container environment for Grafana to @@ -133,7 +137,14 @@ services: # Grafana image renderer: a sidecar that renders panels/dashboards to PNG # for image export and alerting. Grafana calls it at http://renderer:8081. renderer: - image: grafana/grafana-image-renderer:latest + # Pinned like every other image in this stack so a stack that worked + # yesterday still works today. Upstream maintains only the newest release, + # which is expected to serve all supported Grafana versions. + image: grafana/grafana-image-renderer:v5.12.0 + environment: + # Shared secret for the JWT-authenticated render requests Grafana 13 + # sends. Must match GF_RENDERING_RENDERER_TOKEN on the grafana service. + - AUTH_TOKEN=${GF_RENDERING_RENDERER_TOKEN:-xrpld-local-render} ports: - "8081:8081" # Renderer HTTP endpoint (called by grafana) networks: diff --git a/docker/telemetry/grafana/dashboards/consensus-health.json b/docker/telemetry/grafana/dashboards/consensus-health.json index cd1e796a33..e77e207c10 100644 --- a/docker/telemetry/grafana/dashboards/consensus-health.json +++ b/docker/telemetry/grafana/dashboards/consensus-health.json @@ -279,7 +279,7 @@ "uid": "${DS_TEMPO}" }, "queryType": "traceql", - "query": "{name=\"consensus.accept\" && resource.service.instance.id=~\"$node\"} | quantile_over_time(span.round_time_ms, .5)", + "query": "{name=\"consensus.accept\" && resource.service.instance.id=~\"$node\" && resource.service.name=~\"$service_name\" && resource.deployment.environment=~\"$deployment_environment\" && resource.xrpl.network.type=~\"$xrpl_network_type\"} | quantile_over_time(span.round_time_ms, .5)", "legendFormat": "P50 Time to Consensus [{{resource.service.instance.id}}]", "refId": "A" }, @@ -290,7 +290,7 @@ }, "queryType": "traceql", "refId": "B", - "query": "{name=\"consensus.accept\" && resource.service.instance.id=~\"$node\"} | quantile_over_time(span.round_time_ms, .95)", + "query": "{name=\"consensus.accept\" && resource.service.instance.id=~\"$node\" && resource.service.name=~\"$service_name\" && resource.deployment.environment=~\"$deployment_environment\" && resource.xrpl.network.type=~\"$xrpl_network_type\"} | quantile_over_time(span.round_time_ms, .95)", "legendFormat": "P95 Time to Consensus [{{resource.service.instance.id}}]" } ], @@ -333,7 +333,7 @@ "uid": "${DS_TEMPO}" }, "queryType": "traceql", - "query": "{name=\"consensus.accept\" && resource.service.instance.id=~\"$node\"} | avg_over_time(span.round_time_ms)", + "query": "{name=\"consensus.accept\" && resource.service.instance.id=~\"$node\" && resource.service.name=~\"$service_name\" && resource.deployment.environment=~\"$deployment_environment\" && resource.xrpl.network.type=~\"$xrpl_network_type\"} | avg_over_time(span.round_time_ms)", "legendFormat": "Avg Time to Consensus [{{resource.service.instance.id}}]", "refId": "A" } @@ -383,7 +383,7 @@ }, "queryType": "traceql", "refId": "A", - "query": "{name=\"consensus.establish\" && resource.service.instance.id=~\"$node\"} | count_over_time() by (span.establish_count)", + "query": "{name=\"consensus.establish\" && resource.service.instance.id=~\"$node\" && resource.service.name=~\"$service_name\" && resource.deployment.environment=~\"$deployment_environment\" && resource.xrpl.network.type=~\"$xrpl_network_type\"} | count_over_time() by (span.establish_count)", "legendFormat": "{{span.establish_count}} Rounds" } ], @@ -427,7 +427,7 @@ "uid": "${DS_TEMPO}" }, "queryType": "traceql", - "query": "{name=\"consensus.round\" && resource.service.instance.id=~\"$node\"} | quantile_over_time(span.previous_round_time_ms, .95)", + "query": "{name=\"consensus.round\" && resource.service.instance.id=~\"$node\" && resource.service.name=~\"$service_name\" && resource.deployment.environment=~\"$deployment_environment\" && resource.xrpl.network.type=~\"$xrpl_network_type\"} | quantile_over_time(span.previous_round_time_ms, .95)", "legendFormat": "P95 Previous Round [{{resource.service.instance.id}}]", "refId": "A" } @@ -730,7 +730,7 @@ "uid": "${DS_TEMPO}" }, "queryType": "traceql", - "query": "{name=\"consensus.accept.apply\" && resource.service.instance.id=~\"$node\"} | avg_over_time(span.close_time_vote_bins)", + "query": "{name=\"consensus.accept.apply\" && resource.service.instance.id=~\"$node\" && resource.service.name=~\"$service_name\" && resource.deployment.environment=~\"$deployment_environment\" && resource.xrpl.network.type=~\"$xrpl_network_type\"} | avg_over_time(span.close_time_vote_bins)", "legendFormat": "Avg Vote Bins", "refId": "A" }, @@ -740,7 +740,7 @@ "uid": "${DS_TEMPO}" }, "queryType": "traceql", - "query": "{name=\"consensus.accept.apply\" && resource.service.instance.id=~\"$node\"} | avg_over_time(span.close_resolution_ms)", + "query": "{name=\"consensus.accept.apply\" && resource.service.instance.id=~\"$node\" && resource.service.name=~\"$service_name\" && resource.deployment.environment=~\"$deployment_environment\" && resource.xrpl.network.type=~\"$xrpl_network_type\"} | avg_over_time(span.close_resolution_ms)", "legendFormat": "Avg Resolution (ms)", "refId": "B" } @@ -789,7 +789,7 @@ "uid": "${DS_TEMPO}" }, "queryType": "traceql", - "query": "{name=\"consensus.accept.apply\" && resource.service.instance.id=~\"$node\" && span.resolution_direction=~\"$resolution_direction\"} | count_over_time() by (span.resolution_direction)", + "query": "{name=\"consensus.accept.apply\" && resource.service.instance.id=~\"$node\" && resource.service.name=~\"$service_name\" && resource.deployment.environment=~\"$deployment_environment\" && resource.xrpl.network.type=~\"$xrpl_network_type\" && span.resolution_direction=~\"$resolution_direction\"} | count_over_time() by (span.resolution_direction)", "legendFormat": "{{span.resolution_direction}}", "refId": "A" } @@ -861,7 +861,7 @@ "uid": "${DS_TEMPO}" }, "queryType": "traceql", - "query": "{name=\"consensus.accept.apply\" && resource.service.instance.id=~\"$node\"} | count_over_time() by (span.close_time_vote_bins)", + "query": "{name=\"consensus.accept.apply\" && resource.service.instance.id=~\"$node\" && resource.service.name=~\"$service_name\" && resource.deployment.environment=~\"$deployment_environment\" && resource.xrpl.network.type=~\"$xrpl_network_type\"} | count_over_time() by (span.close_time_vote_bins)", "legendFormat": "{{span.close_time_vote_bins}} Distinct Position(s)", "refId": "A" } diff --git a/docker/telemetry/grafana/dashboards/job-queue.json b/docker/telemetry/grafana/dashboards/job-queue.json index f9b5376ece..c7e745e7e9 100644 --- a/docker/telemetry/grafana/dashboards/job-queue.json +++ b/docker/telemetry/grafana/dashboards/job-queue.json @@ -34,13 +34,13 @@ "datasource": { "type": "prometheus" }, - "expr": "label_replace(histogram_quantile(0.99, sum by (le, xrpl_branch, xrpl_node_role) (rate(job_queued_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\", handler=~\"$handler\"}[5m]))), \"series\", \"p99 Wait\", \"\", \"\")" + "expr": "label_replace(histogram_quantile(0.99, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(job_queued_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\", handler=~\"$handler\"}[5m]))), \"series\", \"p99 Wait\", \"\", \"\")" }, { "datasource": { "type": "prometheus" }, - "expr": "label_replace(histogram_quantile(0.99, sum by (le, xrpl_branch, xrpl_node_role) (rate(job_running_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\", handler=~\"$handler\"}[5m]))), \"series\", \"p99 Exec\", \"\", \"\")" + "expr": "label_replace(histogram_quantile(0.99, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(job_running_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\", handler=~\"$handler\"}[5m]))), \"series\", \"p99 Exec\", \"\", \"\")" } ], "fieldConfig": { diff --git a/docker/telemetry/grafana/dashboards/ledger-data-sync.json b/docker/telemetry/grafana/dashboards/ledger-data-sync.json index 640b5b464c..a31c2e87b8 100644 --- a/docker/telemetry/grafana/dashboards/ledger-data-sync.json +++ b/docker/telemetry/grafana/dashboards/ledger-data-sync.json @@ -1248,7 +1248,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_duration_us\"}[$__rate_interval])) / clamp_min(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_total\"}[$__rate_interval])), 1), \"series\", \"NuDB us/read\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_duration_us\"}[$__rate_interval])) / (sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_total\"}[$__rate_interval])) > 0), \"series\", \"NuDB us/read\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" } ], "fieldConfig": { @@ -1332,7 +1332,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_hit\"}[$__rate_interval])) / clamp_min(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_total\"}[$__rate_interval])), 1), \"series\", \"NuDB Found Ratio\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_hit\"}[$__rate_interval])) / (sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_total\"}[$__rate_interval])) > 0), \"series\", \"NuDB Found Ratio\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" } ], "fieldConfig": { @@ -1685,7 +1685,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_duration_us\"}[$__rate_interval])) / clamp_min(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_total\"}[$__rate_interval])), 1), \"series\", \"Read Mean (Windowed)\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_duration_us\"}[$__rate_interval])) / (sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(nodestore_state{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"node_reads_total\"}[$__rate_interval])) > 0), \"series\", \"Read Mean (Windowed)\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" } ], "fieldConfig": { diff --git a/docker/telemetry/grafana/dashboards/log-derived-insights.json b/docker/telemetry/grafana/dashboards/log-derived-insights.json index 6937748bc8..8f7aedfbd5 100644 --- a/docker/telemetry/grafana/dashboards/log-derived-insights.json +++ b/docker/telemetry/grafana/dashboards/log-derived-insights.json @@ -312,10 +312,10 @@ "uid": "${DS_LOKI}" }, "refId": "A", - "expr": "topk($topn, sum by (service_instance_id) (count_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | severity =~ `ERR|FTL` [$__range])))", + "expr": "topk($topn, sum by (service_instance_id, xrpl_network_type) (count_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | severity =~ `ERR|FTL` [$__range])))", "instant": true, "queryType": "instant", - "legendFormat": "Errors [{{service_instance_id}}]" + "legendFormat": "Errors [{{service_instance_id}}] [{{xrpl_network_type}}]" } ], "id": 3 @@ -383,10 +383,10 @@ "uid": "${DS_LOKI}" }, "refId": "A", - "expr": "topk($topn, sum by (service_instance_id) (count_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `Resource` |~ `useless data|unwanted data|init drop` [$__range])))", + "expr": "topk($topn, sum by (service_instance_id, xrpl_network_type) (count_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `Resource` |~ `useless data|unwanted data|init drop` [$__range])))", "instant": true, "queryType": "instant", - "legendFormat": "Charges [{{service_instance_id}}]" + "legendFormat": "Charges [{{service_instance_id}}] [{{xrpl_network_type}}]" } ], "id": 4 @@ -454,10 +454,10 @@ "uid": "${DS_LOKI}" }, "refId": "A", - "expr": "topk($topn, sum by (service_instance_id) (sum_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `Resource` | regexp `\\(\\$(?P[0-9]+)\\)` | unwrap fee [$__range])))", + "expr": "topk($topn, sum by (service_instance_id, xrpl_network_type) (sum_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `Resource` | regexp `\\(\\$(?P[0-9]+)\\)` | unwrap fee [$__range])))", "instant": true, "queryType": "instant", - "legendFormat": "Fee Charged [{{service_instance_id}}]" + "legendFormat": "Fee Charged [{{service_instance_id}}] [{{xrpl_network_type}}]" } ], "id": 5 @@ -525,10 +525,10 @@ "uid": "${DS_LOKI}" }, "refId": "A", - "expr": "topk($topn, sum by (service_instance_id) (count_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `ManifestCache` |~ `Manifest: (Stale|Invalid|Revoked|UntrustedCapacity)` [$__range])))", + "expr": "topk($topn, sum by (service_instance_id, xrpl_network_type) (count_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `ManifestCache` |~ `Manifest: (Stale|Invalid|Revoked|UntrustedCapacity)` [$__range])))", "instant": true, "queryType": "instant", - "legendFormat": "Rejections [{{service_instance_id}}]" + "legendFormat": "Rejections [{{service_instance_id}}] [{{xrpl_network_type}}]" } ], "id": 6 @@ -596,10 +596,10 @@ "uid": "${DS_LOKI}" }, "refId": "A", - "expr": "topk($topn, sum by (service_instance_id) (count_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `LedgerConsensus` | severity =~ `WRN|ERR|FTL` [$__range])))", + "expr": "topk($topn, sum by (service_instance_id, xrpl_network_type) (count_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `LedgerConsensus` | severity =~ `WRN|ERR|FTL` [$__range])))", "instant": true, "queryType": "instant", - "legendFormat": "Consensus Events [{{service_instance_id}}]" + "legendFormat": "Consensus Events [{{service_instance_id}}] [{{xrpl_network_type}}]" } ], "id": 7 @@ -667,10 +667,10 @@ "uid": "${DS_LOKI}" }, "refId": "A", - "expr": "topk($topn, sum by (service_instance_id) (count_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `LoadMonitor` |~ `Job: .* run: [0-9]+ms wait: [0-9]+ms` [$__range])))", + "expr": "topk($topn, sum by (service_instance_id, xrpl_network_type) (count_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `LoadMonitor` |~ `Job: .* run: [0-9]+ms wait: [0-9]+ms` [$__range])))", "instant": true, "queryType": "instant", - "legendFormat": "Breaches [{{service_instance_id}}]" + "legendFormat": "Breaches [{{service_instance_id}}] [{{xrpl_network_type}}]" } ], "id": 8 @@ -738,10 +738,10 @@ "uid": "${DS_LOKI}" }, "refId": "A", - "expr": "topk($topn, sum by (service_instance_id) (count_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" |~ `STATE->(syncing|connected|tracking|disconnected)` [$__range])))", + "expr": "topk($topn, sum by (service_instance_id, xrpl_network_type) (count_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" |~ `STATE->(syncing|connected|tracking|disconnected)` [$__range])))", "instant": true, "queryType": "instant", - "legendFormat": "Transitions [{{service_instance_id}}]" + "legendFormat": "Transitions [{{service_instance_id}}] [{{xrpl_network_type}}]" } ], "id": 9 @@ -809,10 +809,10 @@ "uid": "${DS_LOKI}" }, "refId": "A", - "expr": "topk($topn, sum by (service_instance_id) (sum_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `InboundLedger` | regexp `dupe:(?P[0-9]+)` | unwrap dupe [$__range])))", + "expr": "topk($topn, sum by (service_instance_id, xrpl_network_type) (sum_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `InboundLedger` | regexp `dupe:(?P[0-9]+)` | unwrap dupe [$__range])))", "instant": true, "queryType": "instant", - "legendFormat": "Duplicates [{{service_instance_id}}]" + "legendFormat": "Duplicates [{{service_instance_id}}] [{{xrpl_network_type}}]" } ], "id": 10 @@ -1199,8 +1199,8 @@ "uid": "${DS_LOKI}" }, "refId": "A", - "legendFormat": "Accepted [{{service_instance_id}}]", - "expr": "sum by (service_instance_id) (rate({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `ManifestCache` | severity =~ \"$severity\" |~ `Manifest: Accepted` [$__auto])) or vector(0)" + "legendFormat": "Accepted [{{service_instance_id}}] [{{xrpl_network_type}}]", + "expr": "sum by (service_instance_id, xrpl_network_type) (rate({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `ManifestCache` | severity =~ \"$severity\" |~ `Manifest: Accepted` [$__auto])) or vector(0)" }, { "datasource": { @@ -1208,8 +1208,8 @@ "uid": "${DS_LOKI}" }, "refId": "B", - "legendFormat": "Rejected [{{service_instance_id}}]", - "expr": "sum by (service_instance_id) (rate({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `ManifestCache` | severity =~ \"$severity\" |~ `Manifest: (Stale|Invalid|Revoked|UntrustedCapacity)` [$__auto])) or vector(0)" + "legendFormat": "Rejected [{{service_instance_id}}] [{{xrpl_network_type}}]", + "expr": "sum by (service_instance_id, xrpl_network_type) (rate({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `ManifestCache` | severity =~ \"$severity\" |~ `Manifest: (Stale|Invalid|Revoked|UntrustedCapacity)` [$__auto])) or vector(0)" } ], "id": 19, @@ -1716,8 +1716,8 @@ "uid": "${DS_LOKI}" }, "refId": "A", - "legendFormat": "Duplicate Ratio [{{service_instance_id}}]", - "expr": "sum by (service_instance_id) (sum_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `InboundLedger` | severity =~ \"$severity\" | regexp `dupe:(?P[0-9]+)` | unwrap dupe [$__auto])) / (sum by (service_instance_id) (sum_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `InboundLedger` | severity =~ \"$severity\" | regexp `dupe:(?P[0-9]+)` | unwrap dupe [$__auto])) + sum by (service_instance_id) (sum_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `InboundLedger` | severity =~ \"$severity\" | regexp `good:(?P[0-9]+)` | unwrap good [$__auto])))" + "legendFormat": "Duplicate Ratio [{{service_instance_id}}] [{{xrpl_network_type}}]", + "expr": "sum by (service_instance_id, xrpl_network_type) (sum_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `InboundLedger` | severity =~ \"$severity\" | regexp `dupe:(?P[0-9]+)` | unwrap dupe [$__auto])) / (sum by (service_instance_id, xrpl_network_type) (sum_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `InboundLedger` | severity =~ \"$severity\" | regexp `dupe:(?P[0-9]+)` | unwrap dupe [$__auto])) + sum by (service_instance_id, xrpl_network_type) (sum_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `InboundLedger` | severity =~ \"$severity\" | regexp `good:(?P[0-9]+)` | unwrap good [$__auto])))" } ], "id": 26, @@ -1775,8 +1775,8 @@ "uid": "${DS_LOKI}" }, "refId": "A", - "legendFormat": "Good [{{service_instance_id}}]", - "expr": "sum by (service_instance_id) (sum_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `InboundLedger` | severity =~ \"$severity\" | regexp `good:(?P[0-9]+)` | unwrap good [$__auto]))" + "legendFormat": "Good [{{service_instance_id}}] [{{xrpl_network_type}}]", + "expr": "sum by (service_instance_id, xrpl_network_type) (sum_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `InboundLedger` | severity =~ \"$severity\" | regexp `good:(?P[0-9]+)` | unwrap good [$__auto]))" }, { "datasource": { @@ -1784,8 +1784,8 @@ "uid": "${DS_LOKI}" }, "refId": "B", - "legendFormat": "Duplicate [{{service_instance_id}}]", - "expr": "sum by (service_instance_id) (sum_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `InboundLedger` | severity =~ \"$severity\" | regexp `dupe:(?P[0-9]+)` | unwrap dupe [$__auto]))" + "legendFormat": "Duplicate [{{service_instance_id}}] [{{xrpl_network_type}}]", + "expr": "sum by (service_instance_id, xrpl_network_type) (sum_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `InboundLedger` | severity =~ \"$severity\" | regexp `dupe:(?P[0-9]+)` | unwrap dupe [$__auto]))" }, { "datasource": { @@ -1793,8 +1793,8 @@ "uid": "${DS_LOKI}" }, "refId": "C", - "legendFormat": "Acquire Timeouts [{{service_instance_id}}]", - "expr": "sum by (service_instance_id) (sum_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `InboundLedger` | severity =~ \"$severity\" | regexp `Acquire \\S+ timeouts:(?P[0-9]+)` | unwrap timeouts [$__auto]))" + "legendFormat": "Acquire Timeouts [{{service_instance_id}}] [{{xrpl_network_type}}]", + "expr": "sum by (service_instance_id, xrpl_network_type) (sum_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `InboundLedger` | severity =~ \"$severity\" | regexp `Acquire \\S+ timeouts:(?P[0-9]+)` | unwrap timeouts [$__auto]))" } ], "id": 27, @@ -1924,8 +1924,8 @@ "uid": "${DS_LOKI}" }, "refId": "A", - "legendFormat": "Handshake [{{service_instance_id}}]", - "expr": "sum by (service_instance_id) (rate({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `PeerFinder` | severity =~ \"$severity\" |~ `Logic handshake` [$__auto]))" + "legendFormat": "Handshake [{{service_instance_id}}] [{{xrpl_network_type}}]", + "expr": "sum by (service_instance_id, xrpl_network_type) (rate({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `PeerFinder` | severity =~ \"$severity\" |~ `Logic handshake` [$__auto]))" }, { "datasource": { @@ -1933,8 +1933,8 @@ "uid": "${DS_LOKI}" }, "refId": "B", - "legendFormat": "Accept [{{service_instance_id}}]", - "expr": "sum by (service_instance_id) (rate({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `PeerFinder` | severity =~ \"$severity\" |~ `Logic accept` [$__auto]))" + "legendFormat": "Accept [{{service_instance_id}}] [{{xrpl_network_type}}]", + "expr": "sum by (service_instance_id, xrpl_network_type) (rate({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `PeerFinder` | severity =~ \"$severity\" |~ `Logic accept` [$__auto]))" } ], "id": 30, @@ -2124,8 +2124,8 @@ "uid": "${DS_LOKI}" }, "refId": "A", - "legendFormat": "Quorum [{{service_instance_id}}]", - "expr": "max by (service_instance_id) (max_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `ValidatorList` | severity =~ \"$severity\" | regexp `Using quorum of (?P[0-9]+) for new set of (?P[0-9]+) trusted` | unwrap quorum [$__auto]))" + "legendFormat": "Quorum [{{service_instance_id}}] [{{xrpl_network_type}}]", + "expr": "max by (service_instance_id, xrpl_network_type) (max_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `ValidatorList` | severity =~ \"$severity\" | regexp `Using quorum of (?P[0-9]+) for new set of (?P[0-9]+) trusted` | unwrap quorum [$__auto]))" }, { "datasource": { @@ -2133,8 +2133,8 @@ "uid": "${DS_LOKI}" }, "refId": "B", - "legendFormat": "Trusted Validators [{{service_instance_id}}]", - "expr": "max by (service_instance_id) (max_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `ValidatorList` | severity =~ \"$severity\" | regexp `Using quorum of (?P[0-9]+) for new set of (?P[0-9]+) trusted` | unwrap total [$__auto]))" + "legendFormat": "Trusted Validators [{{service_instance_id}}] [{{xrpl_network_type}}]", + "expr": "max by (service_instance_id, xrpl_network_type) (max_over_time({service_name=~\"$service_name\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\"} | xrpl_network_type =~ \"$xrpl_network_type\" | partition = `ValidatorList` | severity =~ \"$severity\" | regexp `Using quorum of (?P[0-9]+) for new set of (?P[0-9]+) trusted` | unwrap total [$__auto]))" } ], "id": 34, diff --git a/docker/telemetry/grafana/dashboards/node-health.json b/docker/telemetry/grafana/dashboards/node-health.json index 41597900e1..518f7bd5ca 100644 --- a/docker/telemetry/grafana/dashboards/node-health.json +++ b/docker/telemetry/grafana/dashboards/node-health.json @@ -632,7 +632,7 @@ }, { "color": "red", - "value": 80 + "value": 0.8 } ] }, @@ -1177,7 +1177,7 @@ "uid": "${DS_PROMETHEUS}" }, "expr": "server_info{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"server_state\"}", - "legendFormat": "State [{{service_instance_id}}]", + "legendFormat": "State [{{service_instance_id}}] [{{xrpl_network_type}}]", "refId": "A" } ], @@ -1320,7 +1320,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*Object-store read, found and write operation rates. The found series is `node_reads_hit`, which counts fetches that returned an object whatever served them, so it is not a cache-hit count.*\n\n###### How it's computed:\n*Per-second rates of the read, found and write counters, plotted as lines.*\n\n###### Reading it:\n*Steadily rising lines are normal; the slope reflects store activity.*\n\n###### Healthy range:\n*Reads and writes rising smoothly, with found tracking almost all reads on a node that has the data.*\n\n###### Watch for:\n*A sudden surge in reads or writes signals heavy back-end I/O, from sync, replay, or query load.*\n\n###### Keywords:\n- **NodeStore** *(per node)* \u2014 the key-value object store holding ledger data (tree nodes), backed by NuDB.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerNodeStoreGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#nodestore)", + "description": "###### What this is:\n*Fraction of object-store reads that returned an object. `node_reads_hit` counts fetches that found the object whatever served them, so this is not a cache-hit rate.*\n\n###### How it's computed:\n*Per-second rate of `node_reads_hit` divided by the per-second rate of `node_reads_total`, as a single ratio per node.*\n\n###### Reading it:\n*A value near 1.0 means almost every read finds its object; dips mean reads are missing.*\n\n###### Healthy range:\n*Close to 1.0 on a node that holds the data it is being asked for.*\n\n###### Watch for:\n*A sustained drop means the node is repeatedly asked for objects it does not have, which usually accompanies backfill or a gap in history.*\n\n###### Keywords:\n- **NodeStore** *(per node)* \u2014 the key-value object store holding ledger data (tree nodes), backed by NuDB.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerNodeStoreGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#nodestore)", "fieldConfig": { "defaults": { "color": { @@ -1365,12 +1365,12 @@ "mode": "absolute", "steps": [ { - "color": "green", + "color": "red", "value": 0 }, { - "color": "red", - "value": 80 + "color": "green", + "value": 0.8 } ] }, @@ -1768,7 +1768,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*Read-thread utilization, bundled read count, and cumulative read time for the object store.*\n\n###### How it's computed:\n*Current values of the running/total read-thread gauges, read-bundle gauge, and cumulative read-duration counter, plotted as lines.*\n\n###### Reading it:\n*Running threads well below total means spare capacity; a rising duration line reflects time spent in read I/O.*\n\n###### Healthy range:\n*Running threads below the total count most of the time.*\n\n###### Watch for:\n*Running threads pinned at the total for long periods means read I/O is saturated.*\n\n###### Keywords:\n- **Read threads / read queue / write load** *(per node)* \u2014 NodeStore back-end I/O internals \u2014 worker threads reading, their queue depth, and write pressure.\n- **NodeStore** *(per node)* \u2014 the key-value object store holding ledger data (tree nodes), backed by NuDB.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerNodeStoreGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#read-threads-read-queue-write-load)", + "description": "###### What this is:\n*Share of wall-clock time the object store spent inside read I/O.*\n\n###### How it's computed:\n*Per-second rate of the cumulative `node_reads_duration_us` counter, converted from microseconds to seconds, as a single ratio per node.*\n\n###### Reading it:\n*1.0 means the store spent a full second per second in reads; well below 1.0 means spare read capacity.*\n\n###### Healthy range:\n*Below roughly 0.8 in steady state.*\n\n###### Watch for:\n*Sustained values at or above 1.0 mean read I/O is saturated and reads are queueing.*\n\n###### Keywords:\n- **Read threads / read queue / write load** *(per node)* \u2014 NodeStore back-end I/O internals \u2014 worker threads reading, their queue depth, and write pressure.\n- **NodeStore** *(per node)* \u2014 the key-value object store holding ledger data (tree nodes), backed by NuDB.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerNodeStoreGauge`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#read-threads-read-queue-write-load)", "fieldConfig": { "defaults": { "color": { @@ -1818,7 +1818,7 @@ }, { "color": "red", - "value": 80 + "value": 0.8 } ] }, @@ -3177,7 +3177,7 @@ }, "expr": "server_info{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"server_state\"}", "instant": false, - "legendFormat": "{{service_instance_id}} [{{xrpl_branch}}]", + "legendFormat": "{{service_instance_id}} [{{xrpl_network_type}}] [{{xrpl_branch}}]", "queryType": "range", "range": true, "refId": "A" @@ -3241,7 +3241,7 @@ }, "expr": "server_info{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"uptime\"}", "instant": true, - "legendFormat": "{{service_instance_id}}", + "legendFormat": "{{service_instance_id}} [{{xrpl_network_type}}]", "refId": "A" } ], @@ -3311,7 +3311,7 @@ }, "expr": "server_info{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"peers\"}", "instant": true, - "legendFormat": "{{service_instance_id}}", + "legendFormat": "{{service_instance_id}} [{{xrpl_network_type}}]", "refId": "A" } ], @@ -3385,7 +3385,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*The spread in validated ledger sequence across all selected nodes.*\n\n###### How it's computed:\n*Highest validated ledger sequence minus the lowest, across the selected nodes.*\n\n###### Reading it:\n*0 means every node agrees on the same validated ledger; larger means they diverge.*\n\n###### Healthy range:\n*0 to 1 ledger in steady state.*\n\n###### Watch for:\n*A sustained spread above a few ledgers means some nodes are lagging or the fleet is diverging.*\n\n###### Keywords:\n- **Validated ledger** *(network-wide)* \u2014 a ledger confirmed final by the trusted validator quorum; its contents never change.\n\n###### Computation boundary:\n*Result: Across the selected nodes \u2014 the query aggregates instances into one series.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerServerInfoGauge`\n\n###### References:\n[Validated ledger](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#validated-ledger)", + "description": "###### What this is:\n*The spread in validated ledger sequence across all selected nodes.*\n\n###### How it's computed:\n*Highest validated ledger sequence minus the lowest, among the selected nodes on the same network.*\n\n###### Reading it:\n*0 means every node agrees on the same validated ledger; larger means they diverge.*\n\n###### Healthy range:\n*0 to 1 ledger in steady state.*\n\n###### Watch for:\n*A sustained spread above a few ledgers means some nodes are lagging or the fleet is diverging.*\n\n###### Keywords:\n- **Validated ledger** *(network-wide)* \u2014 a ledger confirmed final by the trusted validator quorum; its contents never change.\n\n###### Computation boundary:\n*Result: Per network \u2014 the query aggregates the selected nodes into one series for each `xrpl_network_type`, so mainnet and devnet sequences are never subtracted from each other.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerServerInfoGauge`\n\n###### References:\n[Validated ledger](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#validated-ledger)", "fieldConfig": { "defaults": { "color": { @@ -3467,15 +3467,15 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "max(server_info{metric=\"validated_ledger_seq\",service_name=~\"$service_name\",service_instance_id=~\"$node\",xrpl_network_type=~\"$xrpl_network_type\",xrpl_branch=~\"$xrpl_branch\",xrpl_node_role=~\"$xrpl_node_role\",deployment_environment=~\"$deployment_environment\",xrpl_work_item=~\"$xrpl_work_item\"}) - min(server_info{metric=\"validated_ledger_seq\",service_name=~\"$service_name\",service_instance_id=~\"$node\",xrpl_network_type=~\"$xrpl_network_type\",xrpl_branch=~\"$xrpl_branch\",xrpl_node_role=~\"$xrpl_node_role\",deployment_environment=~\"$deployment_environment\",xrpl_work_item=~\"$xrpl_work_item\"})", + "expr": "max by (xrpl_network_type) (server_info{metric=\"validated_ledger_seq\",service_name=~\"$service_name\",service_instance_id=~\"$node\",xrpl_network_type=~\"$xrpl_network_type\",xrpl_branch=~\"$xrpl_branch\",xrpl_node_role=~\"$xrpl_node_role\",deployment_environment=~\"$deployment_environment\",xrpl_work_item=~\"$xrpl_work_item\"}) - min by (xrpl_network_type) (server_info{metric=\"validated_ledger_seq\",service_name=~\"$service_name\",service_instance_id=~\"$node\",xrpl_network_type=~\"$xrpl_network_type\",xrpl_branch=~\"$xrpl_branch\",xrpl_node_role=~\"$xrpl_node_role\",deployment_environment=~\"$deployment_environment\",xrpl_work_item=~\"$xrpl_work_item\"})", "instant": true, - "legendFormat": "Spread", + "legendFormat": "Spread [{{xrpl_network_type}}]", "queryType": "instant", "range": false, "refId": "A" } ], - "title": "Validated Ledger Seq \u2014 Convergence (Max \u2212 Min)", + "title": "Validated Ledger Seq \u2014 Convergence (Max \u2212 Min, per network)", "type": "stat" }, { @@ -3483,7 +3483,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*How far each node's validated ledger lags behind the network tip, in ledgers.*\n\n###### How it's computed:\n*Highest validated ledger sequence across the selected nodes, minus each node's own sequence.*\n\n###### Reading it:\n*0 means the node is at the tip; larger values mean it trails further behind.*\n\n###### Healthy range:\n*0 to 1 ledger on a synced node.*\n\n###### Watch for:\n*A node stuck at a growing value is falling behind and not keeping up with consensus.*\n\n###### Keywords:\n- **Validated ledger** *(network-wide)* \u2014 a ledger confirmed final by the trusted validator quorum; its contents never change.\n- **Consensus** *(network event)* \u2014 the protocol by which validators agree on the next ledger's transaction set and close time.\n\n###### Computation boundary:\n*Result: Across the selected nodes \u2014 the query aggregates instances into one series.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerServerInfoGauge`\n\n###### References:\n[Validated ledger](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) \u00b7 [Consensus](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#validated-ledger)", + "description": "###### What this is:\n*How far each node's validated ledger lags behind the network tip, in ledgers.*\n\n###### How it's computed:\n*Highest validated ledger sequence within the node's own network, minus each node's own sequence.*\n\n###### Reading it:\n*0 means the node is at the tip; larger values mean it trails further behind.*\n\n###### Healthy range:\n*0 to 1 ledger on a synced node.*\n\n###### Watch for:\n*A node stuck at a growing value is falling behind and not keeping up with consensus.*\n\n###### Keywords:\n- **Validated ledger** *(network-wide)* \u2014 a ledger confirmed final by the trusted validator quorum; its contents never change.\n- **Consensus** *(network event)* \u2014 the protocol by which validators agree on the next ledger's transaction set and close time.\n\n###### Computation boundary:\n*Result: Per node \u2014 each node is compared against the highest sequence on its own `xrpl_network_type`, so mainnet and devnet sequences are never subtracted from each other.*\n*Computed in xrpld code (MetricsRegistry, OpenTelemetry SDK) and exported as a metric; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`registerServerInfoGauge`\n\n###### References:\n[Validated ledger](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) \u00b7 [Consensus](https://xrpl.org/docs/concepts/consensus-protocol/consensus-structure) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#validated-ledger)", "fieldConfig": { "defaults": { "color": { @@ -3533,15 +3533,15 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "sort_desc(scalar(max(server_info{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"validated_ledger_seq\"})) - server_info{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"validated_ledger_seq\"})", + "expr": "sort_desc(max by (xrpl_network_type) (server_info{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"validated_ledger_seq\"}) - on(xrpl_network_type) group_right() server_info{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", metric=\"validated_ledger_seq\"})", "instant": true, - "legendFormat": "{{service_instance_id}}", + "legendFormat": "{{service_instance_id}} [{{xrpl_network_type}}]", "queryType": "instant", "range": false, "refId": "A" } ], - "title": "Validated Ledger Seq \u2014 Current (Stat)", + "title": "Validated Ledger Seq \u2014 Lag Behind Network Tip", "type": "stat" }, { @@ -4218,7 +4218,7 @@ "expr": "complete_ledgers{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}", "format": "table", "instant": true, - "legendFormat": "{{bound}} [range {{index}}] [{{service_instance_id}}]", + "legendFormat": "{{bound}} [range {{index}}] [{{service_instance_id}}] [{{xrpl_network_type}}]", "refId": "A" } ], diff --git a/docker/telemetry/grafana/dashboards/transaction-overview.json b/docker/telemetry/grafana/dashboards/transaction-overview.json index ecb9450172..f71b2ce73e 100644 --- a/docker/telemetry/grafana/dashboards/transaction-overview.json +++ b/docker/telemetry/grafana/dashboards/transaction-overview.json @@ -474,9 +474,9 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "sum by (service_instance_id) (increase(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"txq.accept_tx\", txq_status=\"applied\"}[$__rate_interval]))\n/\nsum by (service_instance_id) (increase(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"txq.accept_tx\", txq_status=~\"applied|failed\"}[$__rate_interval]))", + "expr": "sum by (service_instance_id, xrpl_network_type) (increase(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"txq.accept_tx\", txq_status=\"applied\"}[$__rate_interval]))\n/\nsum by (service_instance_id, xrpl_network_type) (increase(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"txq.accept_tx\", txq_status=~\"applied|failed\"}[$__rate_interval]))", "interval": "15s", - "legendFormat": "{{service_instance_id}}" + "legendFormat": "{{service_instance_id}} [{{xrpl_network_type}}]" } ], "fieldConfig": { @@ -739,56 +739,14 @@ }, "id": 16 }, - { - "title": "Queue Bypass Ratio (Direct Apply vs Enqueue)", - "description": "###### What this is:\n*The fraction of transactions that applied straight to the open ledger instead of being queued.*\n\n###### How it's computed:\n*Direct-apply rate divided by direct-apply plus enqueue rate over 5 minutes, per node.*\n\n###### Reading it:\n*Higher is better; near 1.0 means the network is not congested.*\n\n###### Healthy range:\n*Close to 1.0 when not congested; falls as demand exceeds capacity.*\n\n###### Watch for:\n*A falling ratio, the cleanest single signal the network has entered sustained fee escalation.*\n\n###### Keywords:\n- **Direct apply vs enqueue** *(per node)* \u2014 whether a transaction applied straight to the open ledger or was placed in the queue for a later ledger.\n- **Fee escalation** *(per node)* \u2014 the exponential rise in the open-ledger cost once the ledger exceeds its soft transaction target.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[TxQ.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/detail/TxQ.cpp)\n\n###### Function:\n`TxQ::tryDirectApply ; TxQ::apply`\n\n###### References:\n[Direct apply vs enqueue](https://xrpl.org/docs/concepts/transactions/transaction-queue) \u00b7 [Fee escalation](https://xrpl.org/docs/concepts/transactions/transaction-cost#open-ledger-cost) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#direct-apply-vs-enqueue)", - "type": "timeseries", - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 92 - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc", - "maxHeight": 600 - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "label_replace(label_join(label_replace(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"txq.apply_direct\"}[$__rate_interval])) / clamp_min(sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"txq.apply_direct\"}[$__rate_interval])) + sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_calls_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"txq.enqueue\"}[$__rate_interval])), 1), \"series\", \"Direct-Apply Fraction\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" - } - ], - "fieldConfig": { - "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "unit": "percentunit", - "custom": { - "axisLabel": "Bypass Fraction", - "spanNulls": 1800000, - "insertNulls": false, - "showPoints": "auto", - "pointSize": 3 - } - }, - "overrides": [] - }, - "id": 18 - }, { "title": "Queue Accept (Drain) Duration per Ledger", "description": "###### What this is:\n*The 95th-percentile time to drain queued transactions into a newly closed ledger.*\n\n###### How it's computed:\n*95th-percentile of queue-drain durations over 5 minutes, per node.*\n\n###### Reading it:\n*Lower is better; rising time signals queue pressure at ledger close.*\n\n###### Healthy range:\n*A few milliseconds when the queue is light; workload-dependent.*\n\n###### Watch for:\n*A sustained climb, indicating a large or contended queue slowing ledger close.*\n\n###### Keywords:\n- **Queue accept (drain)** *(per node)* \u2014 moving queued transactions into a newly closed ledger; the applied share shows drain health.\n- **Ledger close** *(network event)* \u2014 the current open ledger is closed and a new closed ledger is built from the agreed transaction set.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in code as a trace span, turned into a metric by the collector (SpanMetrics connector), then aggregated by the Grafana query.*\n\n###### Source:\n[TxQ.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/detail/TxQ.cpp)\n\n###### Function:\n`TxQ::accept`\n\n###### References:\n[Queue accept (drain)](https://xrpl.org/docs/concepts/transactions/transaction-queue) \u00b7 [Ledger close](https://xrpl.org/docs/concepts/ledgers/open-closed-validated-ledgers) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#queue-accept-drain)", "type": "timeseries", "gridPos": { "h": 8, - "w": 12, - "x": 12, + "w": 24, + "x": 0, "y": 92 }, "options": { diff --git a/docs/telemetry-runbook.md b/docs/telemetry-runbook.md index e09debdc82..b26aeb020b 100644 --- a/docs/telemetry-runbook.md +++ b/docs/telemetry-runbook.md @@ -1931,32 +1931,33 @@ Requires `trace_peer=1` in the `[telemetry]` config section. ### Node Health -- System Metrics (`node-health`) -| Panel | Type | PromQL | Labels Used | -| -------------------------------------- | ---------- | ---------------------------------------------------------- | ---------------- | -| Validated Ledger Age | stat | `ledgermaster_validated_ledger_age` | — | -| Published Ledger Age | stat | `ledgermaster_published_ledger_age` | — | -| Operating Mode (Time Share) | timeseries | `rate(state_accounting_X_duration) / sum(rate(all modes))` | — | -| Operating Mode Transitions | timeseries | `state_accounting_*_transitions` | — | -| I/O Latency | timeseries | `histogram_quantile(0.95, ios_latency_bucket)` | — | -| Job Queue Depth | timeseries | `jobq_job_count` | — | -| Ledger Fetch Rate | stat | `rate(ledger_fetches[5m])` | — | -| Ledger History Mismatches | stat | `rate(ledger_history_mismatch[5m])` | — | -| Key Jobs Execution Time | timeseries | `acceptledger{quantile="$quantile"}` (+ 10 more key jobs) | `quantile` | -| Key Jobs Dequeue Wait Time | timeseries | `acceptledger_q{quantile="$quantile"}` (+ 10 more) | `quantile` | -| FullBelowCache Size | timeseries | `node_family_full_below_cache_size` | — | -| FullBelowCache Hit Rate | gauge | `node_family_full_below_cache_hit_rate` | — | -| Ledger Publish Gap | stat | `Published_Ledger_Age - Validated_Ledger_Age` | — | -| State Duration Rate (Full vs Tracking) | timeseries | `rate(state_accounting_full_duration[5m]) / 1000000` | — | -| All Jobs Execution Time (Detail) | timeseries | `{__name__=~"", quantile="$quantile"}` | `quantile` | -| All Jobs Dequeue Wait (Detail) | timeseries | `{__name__=~"_q", quantile="$quantile"}` | `quantile` | -| Server State | stat | `server_info{metric="server_state"}` | `metric` | -| Uptime | stat | `server_info{metric="uptime"}` | `metric` | -| Peer Count | stat | `server_info{metric="peers"}` | `metric` | -| Validated Ledger Seq | stat | `server_info{metric="validated_ledger_seq"}` | `metric` | -| Build Version | stat | `build_info` | `version` | -| Complete Ledger Ranges | table | `complete_ledgers` | `bound`, `index` | -| Database Sizes | timeseries | `db_metrics{metric=~"db_kb_.*"}` | `metric` | -| Historical Fetch Rate | stat | `db_metrics{metric="historical_perminute"}` | `metric` | +| Panel | Type | PromQL | Labels Used | +| ------------------------------------------------------------ | ---------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------- | +| Validated Ledger Age | stat | `ledgermaster_validated_ledger_age` | — | +| Published Ledger Age | stat | `ledgermaster_published_ledger_age` | — | +| Operating Mode (Time Share) | timeseries | `rate(state_accounting_X_duration) / sum(rate(all modes))` | — | +| Operating Mode Transitions | timeseries | `state_accounting_*_transitions` | — | +| I/O Latency | timeseries | `histogram_quantile(0.95, ios_latency_bucket)` | — | +| Job Queue Depth | timeseries | `jobq_job_count` | — | +| Ledger Fetch Rate | stat | `rate(ledger_fetches[5m])` | — | +| Ledger History Mismatches | stat | `rate(ledger_history_mismatch[5m])` | — | +| Key Jobs Execution Time | timeseries | `acceptledger{quantile="$quantile"}` (+ 10 more key jobs) | `quantile` | +| Key Jobs Dequeue Wait Time | timeseries | `acceptledger_q{quantile="$quantile"}` (+ 10 more) | `quantile` | +| FullBelowCache Size | timeseries | `node_family_full_below_cache_size` | — | +| FullBelowCache Hit Rate | gauge | `node_family_full_below_cache_hit_rate` | — | +| Ledger Publish Gap | stat | `Published_Ledger_Age - Validated_Ledger_Age` | — | +| State Duration Rate (Full vs Tracking) | timeseries | `rate(state_accounting_full_duration[5m]) / 1000000` | — | +| All Jobs Execution Time (Detail) | timeseries | `{__name__=~"", quantile="$quantile"}` | `quantile` | +| All Jobs Dequeue Wait (Detail) | timeseries | `{__name__=~"_q", quantile="$quantile"}` | `quantile` | +| Server State | stat | `server_info{metric="server_state"}` | `metric` | +| Uptime | stat | `server_info{metric="uptime"}` | `metric` | +| Peer Count | stat | `server_info{metric="peers"}` | `metric` | +| Validated Ledger Seq -- Lag Behind Network Tip | stat | `max by (xrpl_network_type) (server_info{metric="validated_ledger_seq"}) - on(xrpl_network_type) group_right() server_info{metric="validated_ledger_seq"}` | `metric` | +| Validated Ledger Seq -- Convergence (Max - Min, per network) | stat | `max by (xrpl_network_type) (server_info{metric="validated_ledger_seq"}) - min by (xrpl_network_type) (server_info{metric="validated_ledger_seq"})` | `metric` | +| Build Version | stat | `build_info` | `version` | +| Complete Ledger Ranges | table | `complete_ledgers` | `bound`, `index` | +| Database Sizes | timeseries | `db_metrics{metric=~"db_kb_.*"}` | `metric` | +| Historical Fetch Rate | stat | `db_metrics{metric="historical_perminute"}` | `metric` | ### Network Traffic -- System Metrics (`network-traffic`) @@ -2288,6 +2289,33 @@ Three traps worth knowing before you edit this file: To add a third destination (PagerDuty, Opsgenie, a custom webhook), add a receiver to the relevant contact point. +#### Panel screenshots on alerts need a matching render token + +Alert notifications that carry a panel image are rendered by the `renderer` +sidecar, not by Grafana itself. Grafana 13 enables the `renderAuthJWT` feature +toggle by default, so the renderer rejects any request whose token is missing or +still the `-` default — notifications then arrive with no image. + +`docker-compose.yml` feeds both sides from one variable, so they cannot drift: +`GF_RENDERING_RENDERER_TOKEN` on the `grafana` service and `AUTH_TOKEN` on the +`renderer` service both read `${GF_RENDERING_RENDERER_TOKEN}`, defaulting to a +local development value. Override it in the environment to use your own: + +```bash +GF_RENDERING_RENDERER_TOKEN=$(openssl rand -hex 16) \ + docker compose -f docker/telemetry/docker-compose.yml up -d grafana renderer +``` + +If images stop appearing, check that the two containers agree — a token set on +only one side fails exactly this way: + +```bash +docker compose -f docker/telemetry/docker-compose.yml exec grafana \ + printenv GF_RENDERING_RENDERER_TOKEN +docker compose -f docker/telemetry/docker-compose.yml exec renderer \ + printenv AUTH_TOKEN +``` + #### Deploying alerts to Grafana Cloud Grafana Cloud has **no provisioning filesystem**, so these `apiVersion: 1` files