diff --git a/docker/telemetry/docker-compose.grafanacloud.yaml b/docker/telemetry/docker-compose.grafanacloud.yaml index c2c2fde318..b78f188d53 100644 --- a/docker/telemetry/docker-compose.grafanacloud.yaml +++ b/docker/telemetry/docker-compose.grafanacloud.yaml @@ -11,16 +11,18 @@ # docker compose -f docker/telemetry/docker-compose.yml \ # -f docker/telemetry/docker-compose.grafanacloud.yaml up -d # 3. To go back to local-only, bring the stack up with just the base file. +# +# Log mount: inherited from the base compose (./data/logs -> /var/log/xrpld). +# Both nodes' xrpld cfgs write to docker/telemetry/data/logs//debug.log, +# so no per-box remap is needed here. services: otel-collector: - # Mount the Grafana Cloud variant over the default config path. This - # replaces the local-only config for this run only — the file on disk - # is untouched. volumes: + # Mount the Grafana Cloud collector config over the default path + # (replaces local-only config for this run; file on disk untouched). - ./otel-collector-config.grafanacloud.yaml:/etc/otel-collector-config.yaml:ro - # Secrets are read from the gitignored env file at compose time and - # passed into the container environment, where the collector resolves - # them via ${env:...}. Nothing is baked into the image or committed. + # Secrets from the env file, injected as container env and + # resolved by the collector via ${env:...}. env_file: - .env.grafanacloud diff --git a/docker/telemetry/otel-collector-config.grafanacloud.yaml b/docker/telemetry/otel-collector-config.grafanacloud.yaml index 1be1558708..c967263816 100644 --- a/docker/telemetry/otel-collector-config.grafanacloud.yaml +++ b/docker/telemetry/otel-collector-config.grafanacloud.yaml @@ -53,6 +53,18 @@ processors: batch: timeout: 1s send_batch_size: 100 + # Box-only (uncommitted): probabilistic 0.5% tail sampling on the + # trace-storage branch only. spanmetrics-derived metrics see 100% of spans + # (they run on a separate branch) so RED remain exact; Tempo/Cloud stores + # ~1/200 of traces to keep long-running node cost bounded. + tail_sampling: + decision_wait: 10s + num_traces: 50000 + policies: + - name: half-percent + type: probabilistic + probabilistic: + sampling_percentage: 0.5 resource/logs: attributes: - key: service.name @@ -154,8 +166,6 @@ connectors: - name: acquire_reason exporters: - debug: - verbosity: detailed otlp/tempo: endpoint: tempo:4317 tls: @@ -188,10 +198,18 @@ service: # Each pipeline keeps its local exporter(s) AND adds Grafana Cloud. # For cloud-only, drop debug/otlp/tempo, prometheus, and otlphttp/loki # from the respective exporter lists. - traces: + # 100% of spans feed the spanmetrics connector so span-derived RED + # metrics stay exact. No tail sampling on this branch. + traces/metrics: receivers: [otlp] processors: [resource/tier, resource/stripsdk, batch] - exporters: [debug, otlp/tempo, spanmetrics, otlphttp/grafanacloud] + exporters: [spanmetrics] + # Trace-STORAGE branch: 0.5% probabilistic tail sampling before Tempo + # and Grafana Cloud, so stored trace volume is ~1/200 of ingested spans. + traces/store: + receivers: [otlp] + processors: [tail_sampling, resource/tier, resource/stripsdk, batch] + exporters: [otlp/tempo, otlphttp/grafanacloud] # The local Prometheus scrape promotes tier/instance resource attrs to # labels via resource_to_telemetry_conversion; Grafana Cloud (OTLP) does # not, so it runs a separate pipeline that copies them onto datapoint