diff --git a/OpenTelemetryPlan/09-data-collection-reference.md b/OpenTelemetryPlan/09-data-collection-reference.md index a706a6f160..9b78c3c6ff 100644 --- a/OpenTelemetryPlan/09-data-collection-reference.md +++ b/OpenTelemetryPlan/09-data-collection-reference.md @@ -312,6 +312,8 @@ aggregation. Per the 2026-05-13 naming redesign, span-attribute keys use the > it (both `consensus.validation.send` and `peer.validation.receive`) — there > is no dotted span attribute. +The tables below list one row per attribute per subsystem, so a key shared by two subsystems (for example `ledger_seq`) appears once in each. That is 89 rows over 78 distinct keys. The §6 per-header counts use the same row-based rule, so they sum to 89. + #### RPC Attributes | Attribute | Type | Set On | Description | diff --git a/docker/telemetry/docker-compose.yml b/docker/telemetry/docker-compose.yml index 42c06ebca5..b39e60ce43 100644 --- a/docker/telemetry/docker-compose.yml +++ b/docker/telemetry/docker-compose.yml @@ -50,7 +50,14 @@ services: # batches them for efficiency, and forwards to Tempo for storage. otel-collector: image: otel/opentelemetry-collector-contrib:0.158.0 - command: ["--config=/etc/otel-collector-config.yaml"] + # Second --config layers filelog offset persistence on top of the shared + # base config; the collector deep-merges them. Only this stack keeps its + # logs across restarts, so only this stack needs it. + command: + [ + "--config=/etc/otel-collector-config.yaml", + "--config=/etc/otel-collector-filestorage.yaml", + ] ports: - "4317:4317" # OTLP gRPC - "4318:4318" # OTLP HTTP (traces + native OTel metrics) @@ -61,6 +68,8 @@ services: volumes: # Mount collector pipeline config (receivers → processors → exporters) - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml:ro + # Dev-only overlay: persist filelog read offsets across restarts + - ./otel-collector-filestorage.yaml:/etc/otel-collector-filestorage.yaml:ro # Mount the xrpld log root for the filelog receiver. The telemetry # configs write to docker/telemetry/data/logs//debug.log, so # the default source is the repo-relative ./data/logs — user-owned and diff --git a/docker/telemetry/grafana/dashboards/consensus-health.json b/docker/telemetry/grafana/dashboards/consensus-health.json index 20d1b9613c..2dc19084e3 100644 --- a/docker/telemetry/grafana/dashboards/consensus-health.json +++ b/docker/telemetry/grafana/dashboards/consensus-health.json @@ -350,6 +350,7 @@ "x": 0, "y": 41 }, + "options": { "tooltip": { "mode": "multi", @@ -855,7 +856,8 @@ "pointSize": 5, "lineWidth": 1, "fillOpacity": 0, - "gradientMode": "none" + "gradientMode": "none", + "axisLabel": "NetClock Seconds (Ripple Epoch)" } }, "overrides": [] @@ -897,8 +899,8 @@ "overrides": [ { "matcher": { - "id": "byName", - "options": "Vote Bins" + "id": "byRegexp", + "options": ".*vote_bins.*" }, "properties": [ { @@ -913,8 +915,8 @@ }, { "matcher": { - "id": "byName", - "options": "Resolution" + "id": "byRegexp", + "options": ".*close_resolution_ms.*" }, "properties": [ { diff --git a/docker/telemetry/integration-test.sh b/docker/telemetry/integration-test.sh index af92135da6..2254ac4399 100755 --- a/docker/telemetry/integration-test.sh +++ b/docker/telemetry/integration-test.sh @@ -62,7 +62,10 @@ die() { check_span() { local op="$1" local count - count=$(curl -sf "$TEMPO/api/search" \ + # -G is required: it moves the urlencoded params into the query string. + # Without it curl POSTs them as a request body, and Tempo answers 200 + # while ignoring the query — so every span name would look present. + count=$(curl -sfG "$TEMPO/api/search" \ --data-urlencode "q={resource.service.name=\"xrpld\" && name=\"$op\"}" \ --data-urlencode "limit=5" | jq '.traces | length' 2>/dev/null || echo 0) @@ -184,6 +187,23 @@ mkdir -p "$WORKDIR" # --------------------------------------------------------------------------- # Step 2: Start observability stack # --------------------------------------------------------------------------- + +# From here on the script owns the docker stack and the xrpld nodes, so an +# abort must tear them down instead of leaving them behind. A run that +# reaches the summary deliberately leaves everything up for inspection +# (see the header comment), so the trap only fires before that point. +RUN_COMPLETED=0 +on_exit() { + local status=$? + if [ "$RUN_COMPLETED" -eq 0 ]; then + log "Aborted with exit status $status — tearing down." + cleanup + fi +} +trap on_exit EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + log "Starting observability stack..." # Point the collector's log mount at this test's workdir so it tails the # per-node debug.log files this script generates. The compose default @@ -414,12 +434,14 @@ log "Waiting for nodes to reach 'proposing' state (timeout: ${CONSENSUS_TIMEOUT} start_time=$(date +%s) nodes_ready=0 +consensus_timed_out=0 while [ "$nodes_ready" -lt "$NUM_NODES" ]; do elapsed=$(($(date +%s) - start_time)) if [ "$elapsed" -ge "$CONSENSUS_TIMEOUT" ]; then fail "Consensus timeout after ${CONSENSUS_TIMEOUT}s ($nodes_ready/$NUM_NODES nodes ready)" log "Continuing with partial consensus..." + consensus_timed_out=1 break fi @@ -442,7 +464,10 @@ echo "" if [ "$nodes_ready" -eq "$NUM_NODES" ]; then ok "All $NUM_NODES nodes reached 'proposing' state" -else +elif [ "$consensus_timed_out" -eq 0 ]; then + # The timeout branch above already called fail(), so reporting again here + # would count one timeout twice. Only reachable if the loop ever gains + # another early exit. fail "Only $nodes_ready/$NUM_NODES nodes reached 'proposing' state" fi @@ -486,9 +511,11 @@ log "Submitting Payment transaction..." # Generate a destination wallet log " Generating destination wallet..." +# Guarded: under set -e an unguarded curl failure would abort the whole +# script, so the fallback below could never run. wallet_result=$(curl -sf "http://localhost:$RPC_PORT_BASE" \ - -d '{"method":"wallet_propose"}') -DEST_ACCOUNT=$(echo "$wallet_result" | jq -r '.result.account_id' 2>/dev/null) + -d '{"method":"wallet_propose"}') || wallet_result="" +DEST_ACCOUNT=$(echo "$wallet_result" | jq -r '.result.account_id' 2>/dev/null || echo "") if [ -z "$DEST_ACCOUNT" ] || [ "$DEST_ACCOUNT" = "null" ]; then fail "Could not generate destination wallet" DEST_ACCOUNT="rrrrrrrrrrrrrrrrrrrrrhoLvTp" # ACCOUNT_ZERO fallback @@ -497,13 +524,13 @@ log " Destination: $DEST_ACCOUNT" # Get genesis account info acct_result=$(curl -sf "http://localhost:$RPC_PORT_BASE" \ - -d "{\"method\":\"account_info\",\"params\":[{\"account\":\"$GENESIS_ACCOUNT\"}]}") + -d "{\"method\":\"account_info\",\"params\":[{\"account\":\"$GENESIS_ACCOUNT\"}]}") || acct_result="" seq_num=$(echo "$acct_result" | jq -r '.result.account_data.Sequence' 2>/dev/null || echo "unknown") log " Genesis account sequence: $seq_num" # Submit payment submit_result=$(curl -sf "http://localhost:$RPC_PORT_BASE" \ - -d "{\"method\":\"submit\",\"params\":[{\"secret\":\"$GENESIS_SEED\",\"tx_json\":{\"TransactionType\":\"Payment\",\"Account\":\"$GENESIS_ACCOUNT\",\"Destination\":\"$DEST_ACCOUNT\",\"Amount\":\"10000000\"}}]}") + -d "{\"method\":\"submit\",\"params\":[{\"secret\":\"$GENESIS_SEED\",\"tx_json\":{\"TransactionType\":\"Payment\",\"Account\":\"$GENESIS_ACCOUNT\",\"Destination\":\"$DEST_ACCOUNT\",\"Amount\":\"10000000\"}}]}") || submit_result="" engine_result=$(echo "$submit_result" | jq -r '.result.engine_result' 2>/dev/null || echo "unknown") tx_hash=$(echo "$submit_result" | jq -r '.result.tx_json.hash' 2>/dev/null || echo "unknown") @@ -534,7 +561,7 @@ fi log "" log "--- RPC Spans ---" -check_span "rpc.request" +check_span "rpc.http_request" check_span "rpc.process" check_span "rpc.command.server_info" check_span "rpc.command.server_state" @@ -625,7 +652,7 @@ check_otel_metric() { # Node health gauges (ObservableGauge — no _total suffix) check_otel_metric "ledgermaster_validated_ledger_age" check_otel_metric "ledgermaster_published_ledger_age" -check_otel_metric "job_count" +check_otel_metric "jobq_job_count" # State accounting check_otel_metric "state_accounting_full_duration" @@ -719,6 +746,11 @@ check_otel_metric 'validation_agreement{metric="missed_7d"}' # --------------------------------------------------------------------------- # Step 11: Summary # --------------------------------------------------------------------------- + +# All checks are done, so the run counts as complete: keep the stack and the +# nodes up for inspection even when some checks failed. +RUN_COMPLETED=1 + echo "" echo "===========================================================" echo " INTEGRATION TEST RESULTS" diff --git a/docker/telemetry/otel-collector-config.yaml b/docker/telemetry/otel-collector-config.yaml index 3d4e5c5134..f595c43855 100644 --- a/docker/telemetry/otel-collector-config.yaml +++ b/docker/telemetry/otel-collector-config.yaml @@ -23,18 +23,6 @@ extensions: health_check: endpoint: 0.0.0.0:13133 - # Persists filelog read offsets so a collector restart resumes where it - # stopped instead of re-reading each debug.log from the top. Without this - # the receiver keeps offsets in memory only. - # - # The directory must be writable by the user the collector runs as. The - # image ships no writable directory (no /var/lib, no /tmp), so this path - # comes from a mounted volume; see the otel-collector service in - # docker-compose.yml. Point `directory` somewhere else if a deployment - # mounts its state elsewhere. - file_storage/filelog: - directory: /var/lib/otelcol/file_storage - create_directory: true receivers: otlp: @@ -53,10 +41,12 @@ receivers: # skips everything written before the receiver's first poll — so any log # line a node emitted before the collector got to it would be lost, and # nothing is read at all from a file that has stopped being written to. - # Paired with the file_storage extension above so restarting the - # collector resumes at the last offset rather than re-ingesting the file. + # + # Offsets are kept in memory here, so a restarted collector re-reads the + # file. Stacks that keep their logs across restarts layer + # otel-collector-filestorage.yaml on top to persist them; ephemeral + # stacks get a fresh log directory each run and need nothing. start_at: beginning - storage: file_storage/filelog operators: # Log format emitted by Logs::format() is: # YYYY-Mmm-DD HH:MM:SS.ffffff UTC : [trace_id=... span_id=...] @@ -247,7 +237,7 @@ exporters: enabled: true service: - extensions: [health_check, file_storage/filelog] + extensions: [health_check] pipelines: traces: receivers: [otlp] diff --git a/docker/telemetry/otel-collector-filestorage.yaml b/docker/telemetry/otel-collector-filestorage.yaml new file mode 100644 index 0000000000..5362431725 --- /dev/null +++ b/docker/telemetry/otel-collector-filestorage.yaml @@ -0,0 +1,28 @@ +# Collector overlay that persists filelog read offsets. Applied ONLY by the +# developer stack (docker/telemetry/docker-compose.yml), as a second --config +# after otel-collector-config.yaml; the collector deep-merges the two. +# +# Why this is an overlay rather than part of the base config: the base config +# is shared by every stack that runs the collector, including the ephemeral +# workload-validation stack, which creates a fresh log directory per run and so +# has nothing to resume from. The extension needs a writable directory, and the +# collector image runs as 10001:10001 with no writable path of its own, so +# requiring it in the base config would force every stack to mount a volume +# just to start. Keeping it here means the base config stays self-sufficient. +# +# The developer stack benefits because its log directory and this volume both +# survive `docker compose down`, so a restart resumes at the last offset +# instead of re-reading debug.log from the top. + +extensions: + file_storage/filelog: + directory: /var/lib/otelcol/file_storage + create_directory: true + +receivers: + filelog: + storage: file_storage/filelog + +# Lists are replaced rather than merged, so this must repeat the base entry. +service: + extensions: [health_check, file_storage/filelog] diff --git a/docs/telemetry-runbook.md b/docs/telemetry-runbook.md index 2fd1588fc7..ad9ef2909d 100644 --- a/docs/telemetry-runbook.md +++ b/docs/telemetry-runbook.md @@ -2650,6 +2650,8 @@ Log files are ingested by the OTel Collector's `filelog` receiver, which tails ` The receiver tails `/var/log/xrpld/*/debug.log` inside the collector container. docker-compose bind-mounts the host log root there; the source defaults to the repo-relative `docker/telemetry/data/logs`, which the telemetry configs write to (`data/logs//debug.log`) and which needs no root. To tail logs from elsewhere, set `XRPLD_LOG_DIR` before `docker compose up` (the integration test does this to point at its own workdir). The single trailing `*` matches one per-network or per-node subdirectory. +Each file is read from the beginning, because the receiver's own default (`end`) would skip anything a node wrote before the collector's first poll and would never read a log that has stopped being written to. Read offsets are held in memory by default, so a restarted collector re-reads the files it already ingested. The developer stack avoids that by layering `otel-collector-filestorage.yaml` as a second `--config`, which adds a `file_storage` extension that keeps the offsets on a named volume; a one-shot init service prepares that volume, because the collector runs as a non-root user and a fresh Docker volume is owned by root. Ephemeral stacks such as the workload validation harness create a fresh log directory per run, so they have nothing to resume from and deliberately omit the overlay. + ### LogQL Query Examples The OTel Collector emits logs to Loki with `service_name="xrpld"` (not `job="xrpld"`).