From e103bd32f65965c8bb49443d4c215cc992cc0df7 Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Mon, 13 Jul 2026 18:08:29 +0100 Subject: [PATCH] fix(telemetry): flush spanmetrics every 15s so Cloud rate panels render MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Span-derived metrics (span_calls_total, span_duration_milliseconds_*) were emitted at the spanmetrics connector default of 60s. Grafana Cloud stores one datapoint per flush, but dashboard panels use rate(...[$__rate_interval]), whose window at short ranges (<=1h) falls below 60s — leaving <2 samples per window, so those panels rendered "No data" on the Cloud datasource. (Local Prometheus scrapes at 15s, so local dashboards were unaffected.) Set metrics_flush_interval: 15s so every rate window holds >=2 samples at any dashboard range. Trade-off: ~4x span-metric datapoints pushed to Cloud. Native insight metrics (ledgermaster_*, jobq_*, peer_quality) already push faster and were never affected. Co-Authored-By: Claude Opus 4.7 --- docker/telemetry/otel-collector-config.grafanacloud.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docker/telemetry/otel-collector-config.grafanacloud.yaml b/docker/telemetry/otel-collector-config.grafanacloud.yaml index c967263816..50efeb5d57 100644 --- a/docker/telemetry/otel-collector-config.grafanacloud.yaml +++ b/docker/telemetry/otel-collector-config.grafanacloud.yaml @@ -127,6 +127,13 @@ processors: connectors: spanmetrics: namespace: "span" + # Emit span-derived metrics every 15s (default is 60s). Grafana Cloud + # ingests these as one datapoint per flush, and dashboard rate() panels + # use [$__rate_interval], which at short ranges (<=1h) can be shorter than + # the flush interval. A 60s flush left <2 samples per rate window on Cloud, + # so those panels rendered "No data". 15s keeps >=2 samples per window at + # every dashboard range, at the cost of ~4x span-metric datapoints to Cloud. + metrics_flush_interval: 15s # Resource attributes that define the resource-metrics grouping key. # All resource attributes propagate to the output metrics regardless; # label promotion is handled by resource_to_telemetry_conversion on the