diff --git a/docker/telemetry/otel-collector-config.grafanacloud.yaml b/docker/telemetry/otel-collector-config.grafanacloud.yaml index 5fe6957c72..2a717213b0 100644 --- a/docker/telemetry/otel-collector-config.grafanacloud.yaml +++ b/docker/telemetry/otel-collector-config.grafanacloud.yaml @@ -123,6 +123,15 @@ processors: - set(attributes["service_instance_id"], resource.attributes["service.instance.id"]) - set(attributes["deployment_environment"], resource.attributes["deployment.environment"]) - set(attributes["xrpl_network_type"], resource.attributes["xrpl.network.type"]) + # Copy the per-node id from the resource onto every span so TraceQL can + # filter by node (span.service_instance_id). The name matches the metric + # label written by transform/cloudlabels, so the dashboards' $node variable + # applies to both signals. + transform/spanidentity: + trace_statements: + - context: span + statements: + - set(attributes["service_instance_id"], resource.attributes["service.instance.id"]) connectors: spanmetrics: @@ -256,9 +265,20 @@ service: exporters: [spanmetrics] # Trace-STORAGE branch: 0.5% probabilistic tail sampling before Tempo # and Grafana Cloud, so stored trace volume is ~1/200 of ingested spans. + # transform/spanidentity runs after tail_sampling so only retained spans + # pay for the copy. traces/metrics does not need it: spanmetrics already + # groups by the service.instance.id resource attribute + # (resource_metrics_key_attributes). traces/store: receivers: [otlp] - processors: [tail_sampling, resource/tier, resource/stripsdk, batch] + processors: + [ + tail_sampling, + resource/tier, + resource/stripsdk, + transform/spanidentity, + batch, + ] exporters: [otlp/tempo, otlphttp/grafanacloud] # The local Prometheus scrape promotes tier/instance resource attrs to # labels via resource_to_telemetry_conversion; Grafana Cloud (OTLP) does diff --git a/docker/telemetry/otel-collector-config.yaml b/docker/telemetry/otel-collector-config.yaml index f595c43855..b5cd059fe5 100644 --- a/docker/telemetry/otel-collector-config.yaml +++ b/docker/telemetry/otel-collector-config.yaml @@ -124,6 +124,14 @@ processors: action: hash - key: pathfind_dest_account action: hash + # Copy the per-node id from the resource onto every span so TraceQL can + # filter by node (span.service_instance_id), matching the service_instance_id + # metric label the dashboards' $node variable already uses. + transform/spanidentity: + trace_statements: + - context: span + statements: + - set(attributes["service_instance_id"], resource.attributes["service.instance.id"]) connectors: spanmetrics: @@ -241,7 +249,14 @@ service: pipelines: traces: receivers: [otlp] - processors: [resource/tier, resource/stripsdk, attributes/hash, batch] + processors: + [ + resource/tier, + resource/stripsdk, + attributes/hash, + transform/spanidentity, + batch, + ] exporters: [debug, otlp/tempo, spanmetrics] metrics: receivers: [otlp, spanmetrics] diff --git a/docs/telemetry-runbook.md b/docs/telemetry-runbook.md index 8f9d473529..a8c7b38808 100644 --- a/docs/telemetry-runbook.md +++ b/docs/telemetry-runbook.md @@ -146,6 +146,16 @@ curl -s http://localhost:5015 -d '{"method":"server_info"}' | | `tls_client_cert` | (empty) | Client cert (PEM) for mutual TLS; empty = one-way TLS | | `tls_client_key` | (empty) | Private key (PEM) for `tls_client_cert` | +> **`service_instance_id` reaches traces as a span attribute too.** xrpld sends +> it as the `service.instance.id` resource attribute; the collector's +> `transform/spanidentity` processor copies it onto every span as +> `service_instance_id`, so TraceQL can filter per node +> (`{span.service_instance_id="validator-0"}`) with the same value the `$node` +> dashboard variable uses for metrics. Spans recorded before that processor was +> added do not carry it, and a TraceQL regex does **not** match a missing +> attribute — so a `$node` filter on a trace panel returns nothing for older +> data. + > **`consensus_trace_strategy` is not validated.** The parser copies the raw > string through (`TelemetryConfig.cpp:155-156`) and the only equality test in > the code is `strategy == "attribute"` (`RCLConsensus.cpp:1296`). Any other