diff --git a/OpenTelemetryPlan/09-data-collection-reference.md b/OpenTelemetryPlan/09-data-collection-reference.md index 7cb2441027..41f2a506ca 100644 --- a/OpenTelemetryPlan/09-data-collection-reference.md +++ b/OpenTelemetryPlan/09-data-collection-reference.md @@ -739,7 +739,7 @@ Phase 8 injects OTel trace context into xrpld's `Logs::format()` output, enablin Example: ``` -2024-01-15T10:30:45.123Z LedgerMaster:NFO trace_id=abc123def456789012345678abcdef01 span_id=0123456789abcdef Validated ledger 42 +2024-Jan-15 10:30:45.123456 UTC LedgerMaster:NFO trace_id=abc123def456789012345678abcdef01 span_id=0123456789abcdef Validated ledger 42 ``` - **`trace_id=`** — 32-character lowercase hex trace identifier. Links to the distributed trace in Tempo/Jaeger. @@ -776,7 +776,7 @@ Bidirectional linking between logs and traces is configured via Grafana datasour ### Loki Backend -Grafana Loki (v2.9.0) serves as the log storage backend. It receives log entries from the OTel Collector's `loki` exporter via the push API at `http://loki:3100/loki/api/v1/push`. +Grafana Loki (v3.4.2) serves as the log storage backend. It receives log entries from the OTel Collector's `otlphttp/loki` exporter via the native OTLP endpoint at `http://loki:3100/otlp`. ### LogQL Query Examples diff --git a/OpenTelemetryPlan/Phase8_taskList.md b/OpenTelemetryPlan/Phase8_taskList.md index 3f68f2c7ac..61aab8a432 100644 --- a/OpenTelemetryPlan/Phase8_taskList.md +++ b/OpenTelemetryPlan/Phase8_taskList.md @@ -76,7 +76,7 @@ - Add Loki service: ```yaml loki: - image: grafana/loki:2.9.0 + image: grafana/loki:3.4.2 ports: - "3100:3100" command: -config.file=/etc/loki/local-config.yaml @@ -127,10 +127,8 @@ - Add Loki exporter: ```yaml exporters: - otlp/loki: - endpoint: loki:3100 - tls: - insecure: true + otlphttp/loki: + endpoint: http://loki:3100/otlp ``` - Mount xrpld's log directory into the collector container via docker-compose volume diff --git a/docker/telemetry/TESTING.md b/docker/telemetry/TESTING.md index b54e952e6e..799ea5c3bf 100644 --- a/docker/telemetry/TESTING.md +++ b/docker/telemetry/TESTING.md @@ -491,7 +491,7 @@ Expected: log lines with `trace_id=<32hex> span_id=<16hex>` between the severity code and the message. Example: ``` -2024-01-15T10:30:45.123Z RPCHandler:NFO trace_id=abc123def456789012345678abcdef01 span_id=0123456789abcdef Calling server_info +2024-Jan-15 10:30:45.123456 UTC RPCHandler:NFO trace_id=abc123def456789012345678abcdef01 span_id=0123456789abcdef Calling server_info ``` Lines emitted outside of an active span (background tasks, startup) will diff --git a/docker/telemetry/docker-compose.yml b/docker/telemetry/docker-compose.yml index 2ad9fe3edb..ebc4267da6 100644 --- a/docker/telemetry/docker-compose.yml +++ b/docker/telemetry/docker-compose.yml @@ -36,10 +36,10 @@ services: volumes: # Mount collector pipeline config (receivers → processors → exporters) - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml:ro - # Phase 8: Mount rippled log directory for filelog receiver. + # Mount rippled log directory for filelog receiver. # The integration test writes logs to /tmp/xrpld-integration/; # mount it read-only so the collector can tail debug.log files. - - /tmp/xrpld-integration:/var/log/rippled:ro + - ${XRPLD_LOG_DIR:-/tmp/xrpld-integration}:/var/log/rippled:ro depends_on: - tempo - loki @@ -61,7 +61,7 @@ services: networks: - xrpld-telemetry - # Phase 8: Grafana Loki for centralized log ingestion and log-trace + # Grafana Loki for centralized log ingestion and log-trace # correlation. Loki 3.x supports native OTLP ingestion, so the OTel # Collector exports via otlphttp to Loki's /otlp endpoint. # Query logs via Grafana Explore -> Loki at http://localhost:3000. diff --git a/docker/telemetry/integration-test.sh b/docker/telemetry/integration-test.sh index 24442f9c2a..a0b1ef35de 100755 --- a/docker/telemetry/integration-test.sh +++ b/docker/telemetry/integration-test.sh @@ -73,7 +73,7 @@ check_span() { fi } -# Phase 8: Verify trace_id injection in xrpld log output. +# Verify trace_id injection in xrpld log output. # Greps all node debug.log files for the "trace_id= span_id=" # pattern that Logs::format() injects when an active OTel span exists. # Also cross-checks that a trace_id found in logs matches a trace in Tempo. @@ -562,7 +562,7 @@ check_span "peer.validation.receive" # Step 9b: Verify log-trace correlation (Phase 8) # --------------------------------------------------------------------------- log "" -log "--- Phase 8: Log-Trace Correlation ---" +log "--- Log-Trace Correlation ---" check_log_correlation # --------------------------------------------------------------------------- diff --git a/docker/telemetry/otel-collector-config.yaml b/docker/telemetry/otel-collector-config.yaml index 0d59f5dfe6..ac4f32f3c3 100644 --- a/docker/telemetry/otel-collector-config.yaml +++ b/docker/telemetry/otel-collector-config.yaml @@ -14,7 +14,7 @@ # metrics pipelines. Metrics are exported to Prometheus alongside # span-derived metrics. # -# Phase 8: The filelog receiver tails xrpld's debug.log files under +# The filelog receiver tails xrpld's debug.log files under # /var/log/rippled/ (mounted from the host). A regex_parser operator # extracts timestamp, partition, severity, and optional trace_id/span_id # fields injected by Logs::format(). Parsed logs are exported to Grafana @@ -31,7 +31,7 @@ receivers: endpoint: 0.0.0.0:4317 http: endpoint: 0.0.0.0:4318 - # Phase 8: Filelog receiver tails xrpld debug.log files for log-trace + # Filelog receiver tails xrpld debug.log files for log-trace # correlation. Extracts structured fields (timestamp, partition, severity, # trace_id, span_id, message) via regex. The trace_id and span_id are # optional — only present when the log was emitted within an active span. @@ -159,7 +159,7 @@ exporters: endpoint: tempo:4317 tls: insecure: true - # Phase 8: Export logs to Grafana Loki via OTLP/HTTP. Loki 3.x supports + # Export logs to Grafana Loki via OTLP/HTTP. Loki 3.x supports # native OTLP ingestion on its /otlp endpoint, replacing the removed # loki exporter (dropped in otel-collector-contrib v0.147.0). otlphttp/loki: @@ -184,7 +184,7 @@ service: receivers: [otlp, spanmetrics] processors: [resource/tier, resource/stripsdk, batch] exporters: [prometheus] - # Phase 8: Log pipeline ingests xrpld debug.log via filelog receiver, + # Log pipeline ingests xrpld debug.log via filelog receiver, # batches entries, and exports to Loki for log-trace correlation. logs: receivers: [filelog] diff --git a/docs/telemetry-runbook.md b/docs/telemetry-runbook.md index 2c8981127b..363f9f0aa6 100644 --- a/docs/telemetry-runbook.md +++ b/docs/telemetry-runbook.md @@ -773,7 +773,7 @@ Requires `trace_peer=1` in the `[telemetry]` config section. When xrpld is built with `telemetry=ON`, log lines emitted within an active OpenTelemetry span automatically include `trace_id` and `span_id` fields: ``` -2024-01-15T10:30:45.123Z LedgerMaster:NFO trace_id=abc123def456789012345678abcdef01 span_id=0123456789abcdef Validated ledger 42 +2024-Jan-15 10:30:45.123456 UTC LedgerMaster:NFO trace_id=abc123def456789012345678abcdef01 span_id=0123456789abcdef Validated ledger 42 ``` This enables bidirectional navigation between logs and traces in Grafana: