diff --git a/docker/telemetry/docker-compose.yml b/docker/telemetry/docker-compose.yml index faeacedae3..19dde68bee 100644 --- a/docker/telemetry/docker-compose.yml +++ b/docker/telemetry/docker-compose.yml @@ -44,7 +44,14 @@ services: # batches them for efficiency, and forwards to Tempo for storage. otel-collector: image: otel/opentelemetry-collector-contrib:0.158.0 - command: ["--config=/etc/otel-collector-config.yaml"] + # Second --config layers filelog offset persistence on top of the shared + # base config; the collector deep-merges them. Only this stack keeps its + # logs across restarts, so only this stack needs it. + command: + [ + "--config=/etc/otel-collector-config.yaml", + "--config=/etc/otel-collector-filestorage.yaml", + ] ports: - "4317:4317" # OTLP gRPC - "4318:4318" # OTLP HTTP (traces + native OTel metrics) @@ -55,6 +62,8 @@ services: volumes: # Mount collector pipeline config (receivers → processors → exporters) - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml:ro + # Dev-only overlay: persist filelog read offsets across restarts + - ./otel-collector-filestorage.yaml:/etc/otel-collector-filestorage.yaml:ro # Mount the xrpld log root for the filelog receiver. The telemetry # configs write to docker/telemetry/data/logs//debug.log, so # the default source is the repo-relative ./data/logs — user-owned and diff --git a/docker/telemetry/otel-collector-config.yaml b/docker/telemetry/otel-collector-config.yaml index 2b65abf475..618eb94d5a 100644 --- a/docker/telemetry/otel-collector-config.yaml +++ b/docker/telemetry/otel-collector-config.yaml @@ -23,18 +23,6 @@ extensions: health_check: endpoint: 0.0.0.0:13133 - # Persists filelog read offsets so a collector restart resumes where it - # stopped instead of re-reading each debug.log from the top. Without this - # the receiver keeps offsets in memory only. - # - # The directory must be writable by the user the collector runs as. The - # image ships no writable directory (no /var/lib, no /tmp), so this path - # comes from a mounted volume; see the otel-collector service in - # docker-compose.yml. Point `directory` somewhere else if a deployment - # mounts its state elsewhere. - file_storage/filelog: - directory: /var/lib/otelcol/file_storage - create_directory: true receivers: otlp: @@ -53,10 +41,12 @@ receivers: # skips everything written before the receiver's first poll — so any log # line a node emitted before the collector got to it would be lost, and # nothing is read at all from a file that has stopped being written to. - # Paired with the file_storage extension above so restarting the - # collector resumes at the last offset rather than re-ingesting the file. + # + # Offsets are kept in memory here, so a restarted collector re-reads the + # file. Stacks that keep their logs across restarts layer + # otel-collector-filestorage.yaml on top to persist them; ephemeral + # stacks get a fresh log directory each run and need nothing. start_at: beginning - storage: file_storage/filelog operators: # Log format emitted by Logs::format() is: # YYYY-Mmm-DD HH:MM:SS.ffffff UTC : [trace_id=... span_id=...] @@ -201,7 +191,7 @@ exporters: enabled: true service: - extensions: [health_check, file_storage/filelog] + extensions: [health_check] pipelines: traces: receivers: [otlp] diff --git a/docker/telemetry/otel-collector-filestorage.yaml b/docker/telemetry/otel-collector-filestorage.yaml new file mode 100644 index 0000000000..5362431725 --- /dev/null +++ b/docker/telemetry/otel-collector-filestorage.yaml @@ -0,0 +1,28 @@ +# Collector overlay that persists filelog read offsets. Applied ONLY by the +# developer stack (docker/telemetry/docker-compose.yml), as a second --config +# after otel-collector-config.yaml; the collector deep-merges the two. +# +# Why this is an overlay rather than part of the base config: the base config +# is shared by every stack that runs the collector, including the ephemeral +# workload-validation stack, which creates a fresh log directory per run and so +# has nothing to resume from. The extension needs a writable directory, and the +# collector image runs as 10001:10001 with no writable path of its own, so +# requiring it in the base config would force every stack to mount a volume +# just to start. Keeping it here means the base config stays self-sufficient. +# +# The developer stack benefits because its log directory and this volume both +# survive `docker compose down`, so a restart resumes at the last offset +# instead of re-reading debug.log from the top. + +extensions: + file_storage/filelog: + directory: /var/lib/otelcol/file_storage + create_directory: true + +receivers: + filelog: + storage: file_storage/filelog + +# Lists are replaced rather than merged, so this must repeat the base entry. +service: + extensions: [health_check, file_storage/filelog] diff --git a/docs/telemetry-runbook.md b/docs/telemetry-runbook.md index bc35e8f516..3c615ff614 100644 --- a/docs/telemetry-runbook.md +++ b/docs/telemetry-runbook.md @@ -823,6 +823,8 @@ Log files are ingested by the OTel Collector's `filelog` receiver, which tails ` The receiver tails `/var/log/xrpld/*/debug.log` inside the collector container. docker-compose bind-mounts the host log root there; the source defaults to the repo-relative `docker/telemetry/data/logs`, which the telemetry configs write to (`data/logs//debug.log`) and which needs no root. To tail logs from elsewhere, set `XRPLD_LOG_DIR` before `docker compose up` (the integration test does this to point at its own workdir). The single trailing `*` matches one per-network or per-node subdirectory. +Each file is read from the beginning, because the receiver's own default (`end`) would skip anything a node wrote before the collector's first poll and would never read a log that has stopped being written to. Read offsets are held in memory by default, so a restarted collector re-reads the files it already ingested. The developer stack avoids that by layering `otel-collector-filestorage.yaml` as a second `--config`, which adds a `file_storage` extension that keeps the offsets on a named volume; a one-shot init service prepares that volume, because the collector runs as a non-root user and a fresh Docker volume is owned by root. Ephemeral stacks such as the workload validation harness create a fresh log directory per run, so they have nothing to resume from and deliberately omit the overlay. + ### LogQL Query Examples ```logql