Merge branch 'pratik/otel-phase9-metric-gap-fill' into pratik/otel-phase10-workload-validation

Carries the phase-6 telemetry-doc and integration-test fixes to the tip.
Merged cleanly with no conflicts.

OpenTelemetryPlan/09-data-collection-reference.md

@@ -312,6 +312,8 @@ aggregation. Per the 2026-05-13 naming redesign, span-attribute keys use the
 >   it (both `consensus.validation.send` and `peer.validation.receive`) — there
 >   is no dotted span attribute.
 
+The tables below list one row per attribute per subsystem, so a key shared by two subsystems (for example `ledger_seq`) appears once in each. That is 89 rows over 78 distinct keys. The §6 per-header counts use the same row-based rule, so they sum to 89.
+
 #### RPC Attributes
 
 | Attribute              | Type    | Set On                            | Description                                      |

docker/telemetry/docker-compose.yml

@@ -50,7 +50,14 @@ services:
   # batches them for efficiency, and forwards to Tempo for storage.
   otel-collector:
     image: otel/opentelemetry-collector-contrib:0.158.0
-    command: ["--config=/etc/otel-collector-config.yaml"]
+    # Second --config layers filelog offset persistence on top of the shared
+    # base config; the collector deep-merges them. Only this stack keeps its
+    # logs across restarts, so only this stack needs it.
+    command:
+      [
+        "--config=/etc/otel-collector-config.yaml",
+        "--config=/etc/otel-collector-filestorage.yaml",
+      ]
     ports:
       - "4317:4317" # OTLP gRPC
       - "4318:4318" # OTLP HTTP (traces + native OTel metrics)
@@ -61,6 +68,8 @@ services:
     volumes:
       # Mount collector pipeline config (receivers → processors → exporters)
       - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml:ro
+      # Dev-only overlay: persist filelog read offsets across restarts
+      - ./otel-collector-filestorage.yaml:/etc/otel-collector-filestorage.yaml:ro
       # Mount the xrpld log root for the filelog receiver. The telemetry
       # configs write to docker/telemetry/data/logs/<network>/debug.log, so
       # the default source is the repo-relative ./data/logs — user-owned and

docker/telemetry/grafana/dashboards/consensus-health.json

@@ -350,6 +350,7 @@
         "x": 0,
         "y": 41
       },
+
       "options": {
         "tooltip": {
           "mode": "multi",
@@ -855,7 +856,8 @@
             "pointSize": 5,
             "lineWidth": 1,
             "fillOpacity": 0,
-            "gradientMode": "none"
+            "gradientMode": "none",
+            "axisLabel": "NetClock Seconds (Ripple Epoch)"
           }
         },
         "overrides": []
@@ -897,8 +899,8 @@
         "overrides": [
           {
             "matcher": {
-              "id": "byName",
-              "options": "Vote Bins"
+              "id": "byRegexp",
+              "options": ".*vote_bins.*"
             },
             "properties": [
               {
@@ -913,8 +915,8 @@
           },
           {
             "matcher": {
-              "id": "byName",
-              "options": "Resolution"
+              "id": "byRegexp",
+              "options": ".*close_resolution_ms.*"
             },
             "properties": [
               {

docker/telemetry/integration-test.sh

@@ -62,7 +62,10 @@ die() {
 check_span() {
     local op="$1"
     local count
-    count=$(curl -sf "$TEMPO/api/search" \
+    # -G is required: it moves the urlencoded params into the query string.
+    # Without it curl POSTs them as a request body, and Tempo answers 200
+    # while ignoring the query — so every span name would look present.
+    count=$(curl -sfG "$TEMPO/api/search" \
         --data-urlencode "q={resource.service.name=\"xrpld\" && name=\"$op\"}" \
         --data-urlencode "limit=5" |
         jq '.traces | length' 2>/dev/null || echo 0)
@@ -184,6 +187,23 @@ mkdir -p "$WORKDIR"
 # ---------------------------------------------------------------------------
 # Step 2: Start observability stack
 # ---------------------------------------------------------------------------
+
+# From here on the script owns the docker stack and the xrpld nodes, so an
+# abort must tear them down instead of leaving them behind. A run that
+# reaches the summary deliberately leaves everything up for inspection
+# (see the header comment), so the trap only fires before that point.
+RUN_COMPLETED=0
+on_exit() {
+    local status=$?
+    if [ "$RUN_COMPLETED" -eq 0 ]; then
+        log "Aborted with exit status $status — tearing down."
+        cleanup
+    fi
+}
+trap on_exit EXIT
+trap 'exit 130' INT
+trap 'exit 143' TERM
+
 log "Starting observability stack..."
 # Point the collector's log mount at this test's workdir so it tails the
 # per-node debug.log files this script generates. The compose default
@@ -414,12 +434,14 @@ log "Waiting for nodes to reach 'proposing' state (timeout: ${CONSENSUS_TIMEOUT}
 
 start_time=$(date +%s)
 nodes_ready=0
+consensus_timed_out=0
 
 while [ "$nodes_ready" -lt "$NUM_NODES" ]; do
     elapsed=$(($(date +%s) - start_time))
     if [ "$elapsed" -ge "$CONSENSUS_TIMEOUT" ]; then
         fail "Consensus timeout after ${CONSENSUS_TIMEOUT}s ($nodes_ready/$NUM_NODES nodes ready)"
         log "Continuing with partial consensus..."
+        consensus_timed_out=1
         break
     fi
 
@@ -442,7 +464,10 @@ echo ""
 
 if [ "$nodes_ready" -eq "$NUM_NODES" ]; then
     ok "All $NUM_NODES nodes reached 'proposing' state"
-else
+elif [ "$consensus_timed_out" -eq 0 ]; then
+    # The timeout branch above already called fail(), so reporting again here
+    # would count one timeout twice. Only reachable if the loop ever gains
+    # another early exit.
     fail "Only $nodes_ready/$NUM_NODES nodes reached 'proposing' state"
 fi
 
@@ -486,9 +511,11 @@ log "Submitting Payment transaction..."
 
 # Generate a destination wallet
 log "  Generating destination wallet..."
+# Guarded: under set -e an unguarded curl failure would abort the whole
+# script, so the fallback below could never run.
 wallet_result=$(curl -sf "http://localhost:$RPC_PORT_BASE" \
-    -d '{"method":"wallet_propose"}')
-DEST_ACCOUNT=$(echo "$wallet_result" | jq -r '.result.account_id' 2>/dev/null)
+    -d '{"method":"wallet_propose"}') || wallet_result=""
+DEST_ACCOUNT=$(echo "$wallet_result" | jq -r '.result.account_id' 2>/dev/null || echo "")
 if [ -z "$DEST_ACCOUNT" ] || [ "$DEST_ACCOUNT" = "null" ]; then
     fail "Could not generate destination wallet"
     DEST_ACCOUNT="rrrrrrrrrrrrrrrrrrrrrhoLvTp" # ACCOUNT_ZERO fallback
@@ -497,13 +524,13 @@ log "  Destination: $DEST_ACCOUNT"
 
 # Get genesis account info
 acct_result=$(curl -sf "http://localhost:$RPC_PORT_BASE" \
-    -d "{\"method\":\"account_info\",\"params\":[{\"account\":\"$GENESIS_ACCOUNT\"}]}")
+    -d "{\"method\":\"account_info\",\"params\":[{\"account\":\"$GENESIS_ACCOUNT\"}]}") || acct_result=""
 seq_num=$(echo "$acct_result" | jq -r '.result.account_data.Sequence' 2>/dev/null || echo "unknown")
 log "  Genesis account sequence: $seq_num"
 
 # Submit payment
 submit_result=$(curl -sf "http://localhost:$RPC_PORT_BASE" \
-    -d "{\"method\":\"submit\",\"params\":[{\"secret\":\"$GENESIS_SEED\",\"tx_json\":{\"TransactionType\":\"Payment\",\"Account\":\"$GENESIS_ACCOUNT\",\"Destination\":\"$DEST_ACCOUNT\",\"Amount\":\"10000000\"}}]}")
+    -d "{\"method\":\"submit\",\"params\":[{\"secret\":\"$GENESIS_SEED\",\"tx_json\":{\"TransactionType\":\"Payment\",\"Account\":\"$GENESIS_ACCOUNT\",\"Destination\":\"$DEST_ACCOUNT\",\"Amount\":\"10000000\"}}]}") || submit_result=""
 
 engine_result=$(echo "$submit_result" | jq -r '.result.engine_result' 2>/dev/null || echo "unknown")
 tx_hash=$(echo "$submit_result" | jq -r '.result.tx_json.hash' 2>/dev/null || echo "unknown")
@@ -534,7 +561,7 @@ fi
 
 log ""
 log "--- RPC Spans ---"
-check_span "rpc.request"
+check_span "rpc.http_request"
 check_span "rpc.process"
 check_span "rpc.command.server_info"
 check_span "rpc.command.server_state"
@@ -625,7 +652,7 @@ check_otel_metric() {
 # Node health gauges (ObservableGauge — no _total suffix)
 check_otel_metric "ledgermaster_validated_ledger_age"
 check_otel_metric "ledgermaster_published_ledger_age"
-check_otel_metric "job_count"
+check_otel_metric "jobq_job_count"
 
 # State accounting
 check_otel_metric "state_accounting_full_duration"
@@ -719,6 +746,11 @@ check_otel_metric 'validation_agreement{metric="missed_7d"}'
 # ---------------------------------------------------------------------------
 # Step 11: Summary
 # ---------------------------------------------------------------------------
+
+# All checks are done, so the run counts as complete: keep the stack and the
+# nodes up for inspection even when some checks failed.
+RUN_COMPLETED=1
+
 echo ""
 echo "==========================================================="
 echo "  INTEGRATION TEST RESULTS"

docker/telemetry/otel-collector-config.yaml

@@ -23,18 +23,6 @@
 extensions:
   health_check:
     endpoint: 0.0.0.0:13133
-  # Persists filelog read offsets so a collector restart resumes where it
-  # stopped instead of re-reading each debug.log from the top. Without this
-  # the receiver keeps offsets in memory only.
-  #
-  # The directory must be writable by the user the collector runs as. The
-  # image ships no writable directory (no /var/lib, no /tmp), so this path
-  # comes from a mounted volume; see the otel-collector service in
-  # docker-compose.yml. Point `directory` somewhere else if a deployment
-  # mounts its state elsewhere.
-  file_storage/filelog:
-    directory: /var/lib/otelcol/file_storage
-    create_directory: true
 
 receivers:
   otlp:
@@ -53,10 +41,12 @@ receivers:
     # skips everything written before the receiver's first poll — so any log
     # line a node emitted before the collector got to it would be lost, and
     # nothing is read at all from a file that has stopped being written to.
-    # Paired with the file_storage extension above so restarting the
-    # collector resumes at the last offset rather than re-ingesting the file.
+    #
+    # Offsets are kept in memory here, so a restarted collector re-reads the
+    # file. Stacks that keep their logs across restarts layer
+    # otel-collector-filestorage.yaml on top to persist them; ephemeral
+    # stacks get a fresh log directory each run and need nothing.
     start_at: beginning
-    storage: file_storage/filelog
     operators:
       # Log format emitted by Logs::format() is:
       #   YYYY-Mmm-DD HH:MM:SS.ffffff UTC <partition>:<severity> [trace_id=... span_id=...] <message>
@@ -247,7 +237,7 @@ exporters:
       enabled: true
 
 service:
-  extensions: [health_check, file_storage/filelog]
+  extensions: [health_check]
   pipelines:
     traces:
       receivers: [otlp]

docker/telemetry/otel-collector-filestorage.yaml

@@ -0,0 +1,28 @@
+# Collector overlay that persists filelog read offsets. Applied ONLY by the
+# developer stack (docker/telemetry/docker-compose.yml), as a second --config
+# after otel-collector-config.yaml; the collector deep-merges the two.
+#
+# Why this is an overlay rather than part of the base config: the base config
+# is shared by every stack that runs the collector, including the ephemeral
+# workload-validation stack, which creates a fresh log directory per run and so
+# has nothing to resume from. The extension needs a writable directory, and the
+# collector image runs as 10001:10001 with no writable path of its own, so
+# requiring it in the base config would force every stack to mount a volume
+# just to start. Keeping it here means the base config stays self-sufficient.
+#
+# The developer stack benefits because its log directory and this volume both
+# survive `docker compose down`, so a restart resumes at the last offset
+# instead of re-reading debug.log from the top.
+
+extensions:
+  file_storage/filelog:
+    directory: /var/lib/otelcol/file_storage
+    create_directory: true
+
+receivers:
+  filelog:
+    storage: file_storage/filelog
+
+# Lists are replaced rather than merged, so this must repeat the base entry.
+service:
+  extensions: [health_check, file_storage/filelog]

docs/telemetry-runbook.md

@@ -2650,6 +2650,8 @@ Log files are ingested by the OTel Collector's `filelog` receiver, which tails `
 
 The receiver tails `/var/log/xrpld/*/debug.log` inside the collector container. docker-compose bind-mounts the host log root there; the source defaults to the repo-relative `docker/telemetry/data/logs`, which the telemetry configs write to (`data/logs/<network>/debug.log`) and which needs no root. To tail logs from elsewhere, set `XRPLD_LOG_DIR` before `docker compose up` (the integration test does this to point at its own workdir). The single trailing `*` matches one per-network or per-node subdirectory.
 
+Each file is read from the beginning, because the receiver's own default (`end`) would skip anything a node wrote before the collector's first poll and would never read a log that has stopped being written to. Read offsets are held in memory by default, so a restarted collector re-reads the files it already ingested. The developer stack avoids that by layering `otel-collector-filestorage.yaml` as a second `--config`, which adds a `file_storage` extension that keeps the offsets on a named volume; a one-shot init service prepares that volume, because the collector runs as a non-root user and a fresh Docker volume is owned by root. Ephemeral stacks such as the workload validation harness create a fresh log directory per run, so they have nothing to resume from and deliberately omit the overlay.
+
 ### LogQL Query Examples
 
 The OTel Collector emits logs to Loki with `service_name="xrpld"` (not `job="xrpld"`).