From 1f43baa0bb506132dfb96636c9ff5ecba2f268a1 Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Wed, 15 Jul 2026 11:32:09 +0100 Subject: [PATCH] fix(telemetry): use raw-label legend Display name; drop collector group labels The collector-built resource_group / iac_group labels did not survive the dashboards' own PromQL aggregation: panels that sum by(service_instance_id, xrpl_branch, xrpl_node_role, ...) drop any label not named in the by() clause, so the derived group labels were aggregated away and the Display name rendered empty ("Observing" with no node). Building them in the collector also coupled every dashboard to a lockstep collector redeploy on every node. Switch the legend Display name to reference the raw labels the panels already group by, which are always present by construction and need no collector support: ${__field.labels.series} ${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item} ${__field.labels.X} renders empty for a label a series lacks (verified in Grafana), so dev/mainnet nodes show just " " and perf nodes add the branch/role/work-item, with no empty brackets and no double spaces. Verified live across timeseries, stat, bargauge, piechart, gauge and cross-node aggregation panels. Revert the transform/legendgroups processor from the local collector config; this file is now identical to its pre-change state. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../grafana/dashboards/ledger-data-sync.json | 30 +++++++++---------- .../grafana/dashboards/network-traffic.json | 18 +++++------ .../dashboards/overlay-traffic-detail.json | 16 +++++----- docker/telemetry/otel-collector-config.yaml | 25 +--------------- 4 files changed, 33 insertions(+), 56 deletions(-) diff --git a/docker/telemetry/grafana/dashboards/ledger-data-sync.json b/docker/telemetry/grafana/dashboards/ledger-data-sync.json index b841c04920..cbe53cf9ef 100644 --- a/docker/telemetry/grafana/dashboards/ledger-data-sync.json +++ b/docker/telemetry/grafana/dashboards/ledger-data-sync.json @@ -42,7 +42,7 @@ ], "fieldConfig": { "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.resource_group} ${__field.labels.iac_group}", + "displayName": "${__field.labels.series} ${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}", "unit": "Bps", "custom": { "axisLabel": "Bytes In", @@ -101,7 +101,7 @@ ], "fieldConfig": { "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.resource_group} ${__field.labels.iac_group}", + "displayName": "${__field.labels.series} ${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}", "unit": "Bps", "custom": { "axisLabel": "Bytes In", @@ -148,7 +148,7 @@ ], "fieldConfig": { "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.resource_group} ${__field.labels.iac_group}", + "displayName": "${__field.labels.series} ${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}", "unit": "Bps", "custom": { "axisLabel": "Bytes In", @@ -195,7 +195,7 @@ ], "fieldConfig": { "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.resource_group} ${__field.labels.iac_group}", + "displayName": "${__field.labels.series} ${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}", "unit": "Bps", "custom": { "axisLabel": "Bytes In", @@ -254,7 +254,7 @@ ], "fieldConfig": { "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.resource_group} ${__field.labels.iac_group}", + "displayName": "${__field.labels.series} ${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}", "unit": "Bps", "custom": { "axisLabel": "Bytes In", @@ -301,7 +301,7 @@ ], "fieldConfig": { "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.resource_group} ${__field.labels.iac_group}", + "displayName": "${__field.labels.series} ${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}", "unit": "Bps", "custom": { "axisLabel": "Bytes In", @@ -348,7 +348,7 @@ ], "fieldConfig": { "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.resource_group} ${__field.labels.iac_group}", + "displayName": "${__field.labels.series} ${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}", "unit": "Bps", "custom": { "axisLabel": "Bytes In", @@ -407,7 +407,7 @@ ], "fieldConfig": { "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.resource_group} ${__field.labels.iac_group}", + "displayName": "${__field.labels.series} ${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}", "unit": "Bps", "custom": { "axisLabel": "Bytes In", @@ -454,7 +454,7 @@ ], "fieldConfig": { "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.resource_group} ${__field.labels.iac_group}", + "displayName": "${__field.labels.series} ${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}", "unit": "Bps", "custom": { "axisLabel": "Bytes In", @@ -495,7 +495,7 @@ ], "fieldConfig": { "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.resource_group} ${__field.labels.iac_group}", + "displayName": "${__field.labels.series} ${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}", "unit": "cps", "custom": { "axisLabel": "Messages In", @@ -542,7 +542,7 @@ ], "fieldConfig": { "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.resource_group} ${__field.labels.iac_group}", + "displayName": "${__field.labels.series} ${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}", "unit": "cps", "custom": { "axisLabel": "Messages In", @@ -583,7 +583,7 @@ ], "fieldConfig": { "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.resource_group} ${__field.labels.iac_group}", + "displayName": "${__field.labels.series} ${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}", "unit": "cps", "custom": { "axisLabel": "Messages In", @@ -636,7 +636,7 @@ ], "fieldConfig": { "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.resource_group} ${__field.labels.iac_group}", + "displayName": "${__field.labels.series} ${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}", "unit": "cps", "custom": { "axisLabel": "Messages In", @@ -713,7 +713,7 @@ ], "fieldConfig": { "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.resource_group} ${__field.labels.iac_group}", + "displayName": "${__field.labels.series} ${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}", "unit": "Bps", "custom": { "axisLabel": "Bytes In", @@ -761,7 +761,7 @@ ], "fieldConfig": { "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.resource_group} ${__field.labels.iac_group}", + "displayName": "${__field.labels.series} ${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}", "unit": "decbytes", "thresholds": { "mode": "absolute", diff --git a/docker/telemetry/grafana/dashboards/network-traffic.json b/docker/telemetry/grafana/dashboards/network-traffic.json index 2a764dcb52..514fe8bd2a 100644 --- a/docker/telemetry/grafana/dashboards/network-traffic.json +++ b/docker/telemetry/grafana/dashboards/network-traffic.json @@ -42,7 +42,7 @@ ], "fieldConfig": { "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.resource_group} ${__field.labels.iac_group}", + "displayName": "${__field.labels.series} ${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}", "unit": "short", "custom": { "axisLabel": "Peers", @@ -82,7 +82,7 @@ ], "fieldConfig": { "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.resource_group} ${__field.labels.iac_group}", + "displayName": "${__field.labels.series} ${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}", "unit": "short", "custom": { "axisLabel": "Disconnects per interval", @@ -128,7 +128,7 @@ ], "fieldConfig": { "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.resource_group} ${__field.labels.iac_group}", + "displayName": "${__field.labels.series} ${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}", "unit": "Bps", "custom": { "axisLabel": "Bytes", @@ -174,7 +174,7 @@ ], "fieldConfig": { "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.resource_group} ${__field.labels.iac_group}", + "displayName": "${__field.labels.series} ${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}", "unit": "cps", "custom": { "axisLabel": "Messages", @@ -226,7 +226,7 @@ ], "fieldConfig": { "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.resource_group} ${__field.labels.iac_group}", + "displayName": "${__field.labels.series} ${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}", "unit": "cps", "custom": { "axisLabel": "Messages", @@ -284,7 +284,7 @@ ], "fieldConfig": { "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.resource_group} ${__field.labels.iac_group}", + "displayName": "${__field.labels.series} ${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}", "unit": "cps", "custom": { "axisLabel": "Messages", @@ -342,7 +342,7 @@ ], "fieldConfig": { "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.resource_group} ${__field.labels.iac_group}", + "displayName": "${__field.labels.series} ${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}", "unit": "cps", "custom": { "axisLabel": "Messages", @@ -709,7 +709,7 @@ ], "fieldConfig": { "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.resource_group} ${__field.labels.iac_group}", + "displayName": "${__field.labels.series} ${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}", "unit": "Bps", "custom": { "axisLabel": "Throughput", @@ -749,7 +749,7 @@ ], "fieldConfig": { "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.resource_group} ${__field.labels.iac_group}", + "displayName": "${__field.labels.series} ${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}", "unit": "Bps", "custom": { "axisLabel": "Throughput", diff --git a/docker/telemetry/grafana/dashboards/overlay-traffic-detail.json b/docker/telemetry/grafana/dashboards/overlay-traffic-detail.json index a86b304741..dfeb4f6fe2 100644 --- a/docker/telemetry/grafana/dashboards/overlay-traffic-detail.json +++ b/docker/telemetry/grafana/dashboards/overlay-traffic-detail.json @@ -66,7 +66,7 @@ ], "fieldConfig": { "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.resource_group} ${__field.labels.iac_group}", + "displayName": "${__field.labels.series} ${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}", "unit": "cps", "custom": { "axisLabel": "Messages", @@ -136,7 +136,7 @@ ], "fieldConfig": { "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.resource_group} ${__field.labels.iac_group}", + "displayName": "${__field.labels.series} ${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}", "unit": "Bps", "custom": { "axisLabel": "Bytes", @@ -194,7 +194,7 @@ ], "fieldConfig": { "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.resource_group} ${__field.labels.iac_group}", + "displayName": "${__field.labels.series} ${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}", "unit": "cps", "custom": { "axisLabel": "Count", @@ -269,7 +269,7 @@ ], "fieldConfig": { "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.resource_group} ${__field.labels.iac_group}", + "displayName": "${__field.labels.series} ${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}", "unit": "Bps", "custom": { "axisLabel": "Bytes", @@ -327,7 +327,7 @@ ], "fieldConfig": { "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.resource_group} ${__field.labels.iac_group}", + "displayName": "${__field.labels.series} ${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}", "unit": "cps", "custom": { "axisLabel": "Messages", @@ -385,7 +385,7 @@ ], "fieldConfig": { "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.resource_group} ${__field.labels.iac_group}", + "displayName": "${__field.labels.series} ${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}", "unit": "short", "custom": { "axisLabel": "Count", @@ -460,7 +460,7 @@ ], "fieldConfig": { "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.resource_group} ${__field.labels.iac_group}", + "displayName": "${__field.labels.series} ${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}", "unit": "Bps", "custom": { "axisLabel": "Bytes", @@ -518,7 +518,7 @@ ], "fieldConfig": { "defaults": { - "displayName": "${__field.labels.series} ${__field.labels.resource_group} ${__field.labels.iac_group}", + "displayName": "${__field.labels.series} ${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}", "unit": "Bps", "custom": { "axisLabel": "Bytes", diff --git a/docker/telemetry/otel-collector-config.yaml b/docker/telemetry/otel-collector-config.yaml index a777771b84..5f23e3c932 100644 --- a/docker/telemetry/otel-collector-config.yaml +++ b/docker/telemetry/otel-collector-config.yaml @@ -60,29 +60,6 @@ processors: action: hash - key: pathfind_dest_account action: hash - # Build grouped legend labels from the tier / perf-iac resource attributes so - # dashboards reference one pre-built label per group. resource_to_telemetry_ - # conversion (below) promotes these resource attributes to Prometheus labels. - # The BRACKETS and comma separators are baked into the label VALUE here (e.g. - # "[node, mainnet]") rather than added in the dashboard Display name. This is - # deliberate: a Display name template like "... [${__field.labels.iac_group}]" - # would render an empty "[]" when the label is absent, and Grafana's - # renameByRegex transform runs on the pre-interpolation template so it cannot - # strip that "[]". Baking the brackets into the value means an absent group is - # simply an absent label -> the Display name renders nothing for it. - # - iac_group: "[branch, role, work-item]", built only when all three are - # present (perf runs). Left UNSET otherwise. - # - resource_group: "[node]" or "[node, network]" when a network is present. - # Guards check != nil and != "" because OTTL Concat renders "" for an - # absent attribute, and the perf-iac collector stamps these with an empty-string - # default outside a comparison run. - transform/legendgroups: - metric_statements: - - context: resource - statements: - - set(attributes["iac_group"], Concat(["[", Concat([attributes["xrpl.branch"], attributes["xrpl.node.role"], attributes["xrpl.work.item"]], ", "), "]"], "")) where attributes["xrpl.branch"] != nil and attributes["xrpl.branch"] != "" and attributes["xrpl.node.role"] != nil and attributes["xrpl.node.role"] != "" and attributes["xrpl.work.item"] != nil and attributes["xrpl.work.item"] != "" - - set(attributes["resource_group"], Concat(["[", attributes["service.instance.id"], "]"], "")) where attributes["service.instance.id"] != nil and attributes["service.instance.id"] != "" - - set(attributes["resource_group"], Concat(["[", attributes["service.instance.id"], ", ", attributes["xrpl.network.type"], "]"], "")) where attributes["service.instance.id"] != nil and attributes["service.instance.id"] != "" and attributes["xrpl.network.type"] != nil and attributes["xrpl.network.type"] != "" connectors: spanmetrics: @@ -152,5 +129,5 @@ service: exporters: [debug, otlp/tempo, spanmetrics] metrics: receivers: [otlp, spanmetrics] - processors: [resource/tier, resource/stripsdk, transform/legendgroups] + processors: [resource/tier, resource/stripsdk] exporters: [prometheus]