0.5 tail sampling added

Signed-off-by: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com>

docker/telemetry/docker-compose.grafanacloud.yaml

@@ -11,16 +11,18 @@
 #        docker compose -f docker/telemetry/docker-compose.yml \
 #                       -f docker/telemetry/docker-compose.grafanacloud.yaml up -d
 #   3. To go back to local-only, bring the stack up with just the base file.
+#
+# Log mount: inherited from the base compose (./data/logs -> /var/log/xrpld).
+# Both nodes' xrpld cfgs write to docker/telemetry/data/logs/<net>/debug.log,
+# so no per-box remap is needed here.
 
 services:
   otel-collector:
-    # Mount the Grafana Cloud variant over the default config path. This
-    # replaces the local-only config for this run only — the file on disk
-    # is untouched.
     volumes:
+      # Mount the Grafana Cloud collector config over the default path
+      # (replaces local-only config for this run; file on disk untouched).
       - ./otel-collector-config.grafanacloud.yaml:/etc/otel-collector-config.yaml:ro
-    # Secrets are read from the gitignored env file at compose time and
-    # passed into the container environment, where the collector resolves
-    # them via ${env:...}. Nothing is baked into the image or committed.
+    # Secrets from the env file, injected as container env and
+    # resolved by the collector via ${env:...}.
     env_file:
       - .env.grafanacloud

docker/telemetry/otel-collector-config.grafanacloud.yaml

@@ -53,6 +53,18 @@ processors:
   batch:
     timeout: 1s
     send_batch_size: 100
+  # Box-only (uncommitted): probabilistic 0.5% tail sampling on the
+  # trace-storage branch only. spanmetrics-derived metrics see 100% of spans
+  # (they run on a separate branch) so RED remain exact; Tempo/Cloud stores
+  # ~1/200 of traces to keep long-running node cost bounded.
+  tail_sampling:
+    decision_wait: 10s
+    num_traces: 50000
+    policies:
+      - name: half-percent
+        type: probabilistic
+        probabilistic:
+          sampling_percentage: 0.5
   resource/logs:
     attributes:
       - key: service.name
@@ -154,8 +166,6 @@ connectors:
       - name: acquire_reason
 
 exporters:
-  debug:
-    verbosity: detailed
   otlp/tempo:
     endpoint: tempo:4317
     tls:
@@ -188,10 +198,18 @@ service:
     # Each pipeline keeps its local exporter(s) AND adds Grafana Cloud.
     # For cloud-only, drop debug/otlp/tempo, prometheus, and otlphttp/loki
     # from the respective exporter lists.
-    traces:
+    # 100% of spans feed the spanmetrics connector so span-derived RED
+    # metrics stay exact. No tail sampling on this branch.
+    traces/metrics:
       receivers: [otlp]
       processors: [resource/tier, resource/stripsdk, batch]
-      exporters: [debug, otlp/tempo, spanmetrics, otlphttp/grafanacloud]
+      exporters: [spanmetrics]
+    # Trace-STORAGE branch: 0.5% probabilistic tail sampling before Tempo
+    # and Grafana Cloud, so stored trace volume is ~1/200 of ingested spans.
+    traces/store:
+      receivers: [otlp]
+      processors: [tail_sampling, resource/tier, resource/stripsdk, batch]
+      exporters: [otlp/tempo, otlphttp/grafanacloud]
     # The local Prometheus scrape promotes tier/instance resource attrs to
     # labels via resource_to_telemetry_conversion; Grafana Cloud (OTLP) does
     # not, so it runs a separate pipeline that copies them onto datapoint