rippled/docker/telemetry/integration-test.sh

#!/usr/bin/env bash
# Integration test for rippled OpenTelemetry instrumentation.
#
# Launches a 6-node xrpld consensus network with telemetry enabled,
# exercises RPC / transaction / consensus code paths, then verifies
# that the expected spans and metrics appear in Tempo and Prometheus.
#
# Usage:
#   bash docker/telemetry/integration-test.sh
#
# Prerequisites:
#   - .build/xrpld built with telemetry=ON
#   - docker compose (v2)
#   - curl, jq
#
# The script leaves the observability stack and xrpld nodes running
# so you can manually inspect Tempo (localhost:3200) and Grafana
# (localhost:3000). Run with --cleanup to tear down instead.

set -euo pipefail

# ---------------------------------------------------------------------------
# Configuration
# ---------------------------------------------------------------------------
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
XRPLD="$REPO_ROOT/.build/xrpld"
COMPOSE_FILE="$SCRIPT_DIR/docker-compose.yml"
STANDALONE_CFG="$SCRIPT_DIR/xrpld-telemetry.cfg"
WORKDIR="${WORKDIR:-/tmp/xrpld-integration}"
NUM_NODES=6
PEER_PORT_BASE=51235
RPC_PORT_BASE=5005
CONSENSUS_TIMEOUT=120
GENESIS_ACCOUNT="rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh"
GENESIS_SEED="snoPBrXtMeMyMHUVTgbuqAfg1SUTb"
DEST_ACCOUNT="" # Generated dynamically via wallet_propose
TEMPO="http://localhost:3200"
PROM="http://localhost:9090"
LOKI="http://localhost:3100"
# How long to wait for a log line to travel file -> file_log receiver -> batch
# processor -> Loki. The batch timeout is 1s, so this is mostly ingestion slack.
LOKI_INGEST_TIMEOUT=30

# Hard ceiling on every curl probe below. curl has no overall timeout of its
# own, so a server that accepts the connection and then never answers parks a
# poll loop forever and its attempt count stops bounding anything. 5 s is well
# above a healthy reply, so only a wedged server hits the ceiling.
CURL_MAX_TIME=5

# Counters for pass/fail
PASS=0
FAIL=0

# Unix seconds just before this run's nodes start. Every Tempo search is
# bounded to this run, so a previous run's traces cannot satisfy an assertion.
# Set in Step 5; check_span refuses to run while it is empty.
RUN_START=""

# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
log() { printf "\033[1;34m[INFO]\033[0m  %s\n" "$*"; }
ok() {
    printf "\033[1;32m[PASS]\033[0m  %s\n" "$*"
    PASS=$((PASS + 1))
}
fail() {
    printf "\033[1;31m[FAIL]\033[0m  %s\n" "$*"
    FAIL=$((FAIL + 1))
}
die() {
    printf "\033[1;31m[ERROR]\033[0m %s\n" "$*" >&2
    exit 1
}

check_span() {
    local op="$1"
    local count
    # -G is required: it moves the urlencoded params into the query string.
    # Without it curl POSTs them as a request body, and Tempo answers 200
    # while ignoring the query — so every span name would look present.
    #
    # start/end bound the search to this run. Tempo keeps blocks for
    # block_retention (tempo.yaml, 1h) on a named volume, so without a bound
    # an older run's spans answer for this one. The end margin covers spans
    # exported while this query is in flight.
    [ -n "$RUN_START" ] || die "check_span called before RUN_START was set"
    count=$(curl -sfG --max-time "$CURL_MAX_TIME" "$TEMPO/api/search" \
        --data-urlencode "q={resource.service.name=\"xrpld\" && name=\"$op\"}" \
        --data-urlencode "start=$RUN_START" \
        --data-urlencode "end=$(($(date +%s) + 60))" \
        --data-urlencode "limit=5" |
        jq '.traces | length' 2>/dev/null || echo 0)
    if [ "$count" -gt 0 ]; then
        ok "$op  ($count traces)"
    else
        fail "$op  (0 traces)"
    fi
}

# Verify trace_id injection in xrpld log output.
# Greps all node debug.log files for the "trace_id=<hex> span_id=<hex>"
# pattern that Logs::format() injects when an active OTel span exists.
# Also cross-checks that a trace_id found in logs matches a trace in Tempo.
check_log_correlation() {
    log "Checking log-trace correlation..."

    local total_matches=0
    local files_scanned=0
    local sample_trace_id=""

    for i in $(seq 1 "$NUM_NODES"); do
        local logfile="$WORKDIR/Node-$i/debug.log"
        if [ ! -f "$logfile" ]; then
            continue
        fi
        files_scanned=$((files_scanned + 1))
        local matches
        matches=$(grep -c 'trace_id=[a-f0-9]\{32\} span_id=[a-f0-9]\{16\}' "$logfile") || matches=0
        total_matches=$((total_matches + matches))
        # Capture the first trace_id we find for cross-referencing with Tempo
        if [ -z "$sample_trace_id" ] && [ "$matches" -gt 0 ]; then
            # -m1 makes grep stop after the first match and exit normally.
            # Piping into `head -1` instead closes the pipe under grep, and
            # under `set -o pipefail` the resulting SIGPIPE (141) aborts the
            # whole run. It only bites once the log is bigger than the pipe
            # buffer, so it reads as a flaky test.
            sample_trace_id=$(grep -m1 -o 'trace_id=[a-f0-9]\{32\}' "$logfile" | cut -d= -f2)
        fi
    done

    if [ "$files_scanned" -eq 0 ]; then
        fail "Log correlation: no debug.log files found in $WORKDIR/Node-*/"
        return
    fi

    if [ "$total_matches" -gt 0 ]; then
        ok "Log correlation: found $total_matches log lines with trace_id ($files_scanned nodes scanned)"
    else
        fail "Log correlation: no trace_id found in any node debug.log ($files_scanned nodes scanned)"
    fi

    # Cross-check: verify the sample trace_id exists in Tempo
    if [ -n "$sample_trace_id" ]; then
        local trace_found
        # Tempo /api/traces/{id} returns OTLP shape: {"batches":[...]}
        trace_found=$(curl -sf "$TEMPO/api/traces/$sample_trace_id" |
            jq '.batches | length' 2>/dev/null) || trace_found=0
        if [ "$trace_found" -gt 0 ]; then
            ok "Log-Tempo cross-check: trace_id=$sample_trace_id found in Tempo"
        else
            fail "Log-Tempo cross-check: trace_id=$sample_trace_id NOT found in Tempo"
        fi

        check_loki_ingestion "$sample_trace_id"
    fi
}

# Verify the log line actually reached Loki, not just the local file.
#
# Without this the log-correlation check passes on a stack whose log mount is
# wrong or whose Loki exporter is broken, because reading the file and reading
# Tempo both still work. This is the only assertion that exercises the
# file_log -> Loki hop, so it is what makes the log pipeline tested rather than
# merely configured.
#
# Uses /query_range, not /query: Loki rejects a bare log selector on the instant
# endpoint with HTTP 400 and a text/plain body, so jq could never parse it.
# Bounds are unix nanoseconds, matching workload/validate_telemetry.py.
check_loki_ingestion() {
    local trace_id="$1"
    local lines=0
    local start_ns end_ns

    for attempt in $(seq 1 "$LOKI_INGEST_TIMEOUT"); do
        end_ns=$(($(date +%s) * 1000000000))
        # Look back over the whole run, not a fixed window: the entry carries
        # the timestamp parsed out of the log line, not its ingestion time.
        start_ns=$((end_ns - 86400000000000))
        lines=$(curl -sfG "$LOKI/loki/api/v1/query_range" \
            --data-urlencode "query={service_name=\"xrpld\"} |= \"$trace_id\"" \
            --data-urlencode "start=$start_ns" \
            --data-urlencode "end=$end_ns" \
            --data-urlencode "limit=5" \
            --data-urlencode "direction=backward" |
            jq '[.data.result[].values | length] | add // 0' 2>/dev/null) || lines=0
        if [ "${lines:-0}" -gt 0 ]; then
            ok "Loki ingestion: trace_id=$trace_id found in Loki ($lines lines, attempt $attempt)"
            return
        fi
        sleep 1
    done

    fail "Loki ingestion: trace_id=$trace_id never reached Loki after ${LOKI_INGEST_TIMEOUT}s"
}

cleanup() {
    log "Cleaning up..."
    # Kill xrpld nodes
    for i in $(seq 1 "$NUM_NODES"); do
        local pidfile="$WORKDIR/Node-$i/xrpld.pid"
        if [ -f "$pidfile" ]; then
            kill "$(cat "$pidfile")" 2>/dev/null || true
            rm -f "$pidfile"
        fi
    done
    # Also kill any straggling xrpld processes from our workdir
    pkill -f "$WORKDIR" 2>/dev/null || true
    # Stop docker stack. -v also drops the tempo-data volume: plain `down`
    # keeps it, and retained traces would then answer a later run's searches.
    docker compose -f "$COMPOSE_FILE" down -v 2>/dev/null || true
    # Remove workdir
    rm -rf "$WORKDIR"
    log "Cleanup complete."
}

# Handle --cleanup flag
if [ "${1:-}" = "--cleanup" ]; then
    cleanup
    exit 0
fi

# ---------------------------------------------------------------------------
# Step 0: Prerequisites
# ---------------------------------------------------------------------------
log "Checking prerequisites..."

command -v docker >/dev/null 2>&1 || die "docker not found"
docker compose version >/dev/null 2>&1 || die "docker compose (v2) not found"
command -v curl >/dev/null 2>&1 || die "curl not found"
command -v jq >/dev/null 2>&1 || die "jq not found"
[ -x "$XRPLD" ] || die "xrpld binary not found at $XRPLD (build with telemetry=ON)"
[ -f "$COMPOSE_FILE" ] || die "docker-compose.yml not found at $COMPOSE_FILE"
[ -f "$STANDALONE_CFG" ] || die "xrpld-telemetry.cfg not found at $STANDALONE_CFG"

log "All prerequisites met."

# ---------------------------------------------------------------------------
# Step 1: Clean previous run
# ---------------------------------------------------------------------------
log "Cleaning previous run data..."
for i in $(seq 1 "$NUM_NODES"); do
    pidfile="$WORKDIR/Node-$i/xrpld.pid"
    if [ -f "$pidfile" ]; then
        kill "$(cat "$pidfile")" 2>/dev/null || true
    fi
done
pkill -f "$WORKDIR" 2>/dev/null || true
# Kill any xrpld using the standalone config (from key generation)
pkill -f "xrpld-telemetry.cfg" 2>/dev/null || true
sleep 2
rm -rf "$WORKDIR"
# A run that reached the summary left the stack up, so nothing has torn it
# down. Do it here, with -v: Tempo's traces and Prometheus' samples must not
# survive into this run, or an assertion can pass on the previous run's data.
docker compose -f "$COMPOSE_FILE" down -v 2>/dev/null || true
mkdir -p "$WORKDIR"

# ---------------------------------------------------------------------------
# Step 2: Start observability stack
# ---------------------------------------------------------------------------

# From here on the script owns the docker stack and the xrpld nodes, so an
# abort must tear them down instead of leaving them behind. A run that
# reaches the summary deliberately leaves everything up for inspection
# (see the header comment), so the trap only fires before that point.
RUN_COMPLETED=0
on_exit() {
    local status=$?
    if [ "$RUN_COMPLETED" -eq 0 ]; then
        log "Aborted with exit status $status — tearing down."
        cleanup
    fi
}
trap on_exit EXIT
trap 'exit 130' INT
trap 'exit 143' TERM

log "Starting observability stack..."
# Point the collector's log mount at this test's workdir so it tails the
# per-node debug.log files this script generates. The compose default
# (./data/logs) is for user-run xrpld; the test owns its own log root.
XRPLD_LOG_DIR="$WORKDIR" docker compose -f "$COMPOSE_FILE" up -d

log "Waiting for otel-collector to be ready..."
for attempt in $(seq 1 30); do
    # The OTLP HTTP endpoint returns 405 for GET (expects POST), which
    # means it is listening.  curl -sf would fail on 405, so we check
    # the HTTP status code explicitly.
    status=$(curl -so /dev/null -w '%{http_code}' --max-time "$CURL_MAX_TIME" http://localhost:4318/ 2>/dev/null || echo 000)
    if [ "$status" != "000" ]; then
        log "otel-collector ready (attempt $attempt, HTTP $status)."
        break
    fi
    if [ "$attempt" -eq 30 ]; then
        die "otel-collector not ready after 30s"
    fi
    sleep 1
done

log "Waiting for Tempo to be ready..."
for attempt in $(seq 1 30); do
    if curl -sf --max-time "$CURL_MAX_TIME" "$TEMPO/ready" >/dev/null 2>&1; then
        log "Tempo ready (attempt $attempt)."
        break
    fi
    if [ "$attempt" -eq 30 ]; then
        die "Tempo not ready after 30s"
    fi
    sleep 1
done

log "Waiting for Loki to be ready..."
for attempt in $(seq 1 60); do
    if curl -sf "$LOKI/ready" >/dev/null 2>&1; then
        log "Loki ready (attempt $attempt)."
        break
    fi
    if [ "$attempt" -eq 60 ]; then
        die "Loki not ready after 60s"
    fi
    sleep 1
done

# ---------------------------------------------------------------------------
# Step 3: Generate validator keys
# ---------------------------------------------------------------------------
log "Generating $NUM_NODES validator key pairs..."

# Start a temporary standalone xrpld for key generation
TEMP_DATA="$WORKDIR/temp-keygen"
mkdir -p "$TEMP_DATA"

# Create a minimal temp config for key generation
TEMP_CFG="$TEMP_DATA/xrpld.cfg"
cat >"$TEMP_CFG" <<EOCFG
[server]
port_rpc_temp

[port_rpc_temp]
port = 5099
ip = 127.0.0.1
admin = 127.0.0.1
protocol = http

[node_db]
type=NuDB
path=$TEMP_DATA/nudb
online_delete=256

[database_path]
$TEMP_DATA/db

[debug_logfile]
$TEMP_DATA/debug.log

[ssl_verify]
0
EOCFG

"$XRPLD" --conf "$TEMP_CFG" -a --start >"$TEMP_DATA/stdout.log" 2>&1 &
TEMP_PID=$!
log "Temporary xrpld started (PID $TEMP_PID), waiting for RPC..."

for attempt in $(seq 1 30); do
    if curl -sf --max-time "$CURL_MAX_TIME" http://localhost:5099 -d '{"method":"server_info"}' >/dev/null 2>&1; then
        log "Temporary xrpld RPC ready (attempt $attempt)."
        break
    fi
    if [ "$attempt" -eq 30 ]; then
        kill "$TEMP_PID" 2>/dev/null || true
        die "Temporary xrpld RPC not ready after 30s"
    fi
    sleep 1
done

declare -a SEEDS
declare -a PUBKEYS

for i in $(seq 1 "$NUM_NODES"); do
    result=$(curl -sf --max-time "$CURL_MAX_TIME" http://localhost:5099 -d '{"method":"validation_create"}')
    seed=$(echo "$result" | jq -r '.result.validation_seed')
    pubkey=$(echo "$result" | jq -r '.result.validation_public_key')
    if [ -z "$seed" ] || [ "$seed" = "null" ]; then
        kill "$TEMP_PID" 2>/dev/null || true
        die "Failed to generate key pair $i"
    fi
    SEEDS+=("$seed")
    PUBKEYS+=("$pubkey")
    log "  Node $i: $pubkey"
done

kill "$TEMP_PID" 2>/dev/null || true
wait "$TEMP_PID" 2>/dev/null || true
rm -rf "$TEMP_DATA"
log "Key generation complete."

# ---------------------------------------------------------------------------
# Step 4: Generate node configs and validators.txt
# ---------------------------------------------------------------------------
log "Generating node configs..."

# Create shared validators.txt
VALIDATORS_FILE="$WORKDIR/validators.txt"
{
    echo "[validators]"
    for i in $(seq 0 $((NUM_NODES - 1))); do
        echo "${PUBKEYS[$i]}"
    done
} >"$VALIDATORS_FILE"

# Create per-node configs
for i in $(seq 1 "$NUM_NODES"); do
    NODE_DIR="$WORKDIR/Node-$i"
    mkdir -p "$NODE_DIR/nudb" "$NODE_DIR/db"

    RPC_PORT=$((RPC_PORT_BASE + i - 1))
    PEER_PORT=$((PEER_PORT_BASE + i - 1))
    SEED="${SEEDS[$((i - 1))]}"

    # Build ips_fixed list (all peers except self)
    IPS_FIXED=""
    for j in $(seq 1 "$NUM_NODES"); do
        if [ "$j" -ne "$i" ]; then
            IPS_FIXED="${IPS_FIXED}127.0.0.1 $((PEER_PORT_BASE + j - 1))
"
        fi
    done

    cat >"$NODE_DIR/xrpld.cfg" <<EOCFG
[server]
port_rpc
port_peer

[port_rpc]
port = $RPC_PORT
ip = 127.0.0.1
admin = 127.0.0.1
protocol = http

[port_peer]
port = $PEER_PORT
ip = 0.0.0.0
protocol = peer

# A private id, so telemetry stamps xrpl.network.type=unknown. The config
# default is id 0, which maps to "mainnet" -- this cluster's spans and metrics
# would then share dashboard series with real mainnet data.
[network_id]
1025

[node_db]
type=NuDB
path=$NODE_DIR/nudb
online_delete=256

[database_path]
$NODE_DIR/db

[debug_logfile]
$NODE_DIR/debug.log

[validation_seed]
$SEED

[validators_file]
$VALIDATORS_FILE

[ips_fixed]
${IPS_FIXED}
[peer_private]
1

[telemetry]
enabled=1
service_instance_id=Node-${i}
traces_endpoint=http://localhost:4318/v1/traces
metrics_endpoint=http://localhost:4318/v1/metrics
batch_size=512
batch_delay_ms=2000
max_queue_size=2048
trace_rpc=1
trace_transactions=1
trace_consensus=1
trace_peer=1
trace_ledger=1

[insight]
# server=otel is the only load-bearing key here -- it selects OTelCollector so
# beast::insight metrics leave over OTLP. No prefix is set on purpose: on this
# path it is inert, because OTelCollector's formatName() only lowercases the raw
# instrument name and the one place the class reads prefix_ is its startup log
# line. The service is identified by the OTel resource service.name.
server=otel
endpoint=http://localhost:4318/v1/metrics
service_instance_id=Node-${i}

[rpc_startup]
{ "command": "log_level", "severity": "warning" }

[ssl_verify]
0
EOCFG

    log "  Node $i config: RPC=$RPC_PORT, Peer=$PEER_PORT"
done

# ---------------------------------------------------------------------------
# Step 5: Start all 6 nodes
# ---------------------------------------------------------------------------
log "Starting $NUM_NODES xrpld nodes..."

# Lower bound for every Tempo search below. Only these nodes have a
# [telemetry] section, so nothing before this instant belongs to this run.
RUN_START=$(date +%s)

for i in $(seq 1 "$NUM_NODES"); do
    NODE_DIR="$WORKDIR/Node-$i"
    "$XRPLD" --conf "$NODE_DIR/xrpld.cfg" --start >"$NODE_DIR/stdout.log" 2>&1 &
    echo $! >"$NODE_DIR/xrpld.pid"
    log "  Node $i started (PID $(cat "$NODE_DIR/xrpld.pid"))"
done

# Give nodes a moment to initialize
sleep 5

# ---------------------------------------------------------------------------
# Step 6: Wait for consensus
# ---------------------------------------------------------------------------
log "Waiting for nodes to reach 'proposing' state (timeout: ${CONSENSUS_TIMEOUT}s)..."

start_time=$(date +%s)
nodes_ready=0
consensus_timed_out=0

while [ "$nodes_ready" -lt "$NUM_NODES" ]; do
    elapsed=$(($(date +%s) - start_time))
    if [ "$elapsed" -ge "$CONSENSUS_TIMEOUT" ]; then
        fail "Consensus timeout after ${CONSENSUS_TIMEOUT}s ($nodes_ready/$NUM_NODES nodes ready)"
        log "Continuing with partial consensus..."
        consensus_timed_out=1
        break
    fi

    nodes_ready=0
    for i in $(seq 1 "$NUM_NODES"); do
        RPC_PORT=$((RPC_PORT_BASE + i - 1))
        state=$(curl -sf --max-time "$CURL_MAX_TIME" "http://localhost:$RPC_PORT" \
            -d '{"method":"server_info"}' 2>/dev/null |
            jq -r '.result.info.server_state' 2>/dev/null || echo "unreachable")
        if [ "$state" = "proposing" ]; then
            nodes_ready=$((nodes_ready + 1))
        fi
    done
    printf "\r  %d/%d nodes proposing (%ds elapsed)..." "$nodes_ready" "$NUM_NODES" "$elapsed"
    if [ "$nodes_ready" -lt "$NUM_NODES" ]; then
        sleep 3
    fi
done
echo ""

if [ "$nodes_ready" -eq "$NUM_NODES" ]; then
    ok "All $NUM_NODES nodes reached 'proposing' state"
elif [ "$consensus_timed_out" -eq 0 ]; then
    # The timeout branch above already called fail(), so reporting again here
    # would count one timeout twice. Only reachable if the loop ever gains
    # another early exit.
    fail "Only $nodes_ready/$NUM_NODES nodes reached 'proposing' state"
fi

# ---------------------------------------------------------------------------
# Step 6b: Wait for validated ledger
# ---------------------------------------------------------------------------
log "Waiting for first validated ledger..."
for attempt in $(seq 1 60); do
    val_seq=$(curl -sf --max-time "$CURL_MAX_TIME" "http://localhost:$RPC_PORT_BASE" \
        -d '{"method":"server_info"}' 2>/dev/null |
        jq -r '.result.info.validated_ledger.seq // 0' 2>/dev/null || echo 0)
    if [ "$val_seq" -gt 2 ] 2>/dev/null; then
        ok "First validated ledger: seq $val_seq"
        break
    fi
    if [ "$attempt" -eq 60 ]; then
        fail "No validated ledger after 60s"
    fi
    sleep 1
done

# ---------------------------------------------------------------------------
# Step 7: Exercise RPC spans
# ---------------------------------------------------------------------------
log "Exercising RPC spans..."

curl -sf --max-time "$CURL_MAX_TIME" "http://localhost:$RPC_PORT_BASE" \
    -d '{"method":"server_info"}' >/dev/null
curl -sf --max-time "$CURL_MAX_TIME" "http://localhost:$RPC_PORT_BASE" \
    -d '{"method":"server_state"}' >/dev/null
curl -sf --max-time "$CURL_MAX_TIME" "http://localhost:$RPC_PORT_BASE" \
    -d '{"method":"ledger","params":[{"ledger_index":"current"}]}' >/dev/null

log "RPC commands sent. Waiting 5s for batch export..."
sleep 5

# ---------------------------------------------------------------------------
# Step 8: Submit transaction
# ---------------------------------------------------------------------------
log "Submitting Payment transaction..."

# Generate a destination wallet
log "  Generating destination wallet..."
# Guarded: under set -e an unguarded curl failure would abort the whole
# script, so the fallback below could never run.
wallet_result=$(curl -sf --max-time "$CURL_MAX_TIME" "http://localhost:$RPC_PORT_BASE" \
    -d '{"method":"wallet_propose"}') || wallet_result=""
DEST_ACCOUNT=$(echo "$wallet_result" | jq -r '.result.account_id' 2>/dev/null || echo "")
if [ -z "$DEST_ACCOUNT" ] || [ "$DEST_ACCOUNT" = "null" ]; then
    fail "Could not generate destination wallet"
    DEST_ACCOUNT="rrrrrrrrrrrrrrrrrrrrrhoLvTp" # ACCOUNT_ZERO fallback
fi
log "  Destination: $DEST_ACCOUNT"

# Get genesis account info
acct_result=$(curl -sf --max-time "$CURL_MAX_TIME" "http://localhost:$RPC_PORT_BASE" \
    -d "{\"method\":\"account_info\",\"params\":[{\"account\":\"$GENESIS_ACCOUNT\"}]}") || acct_result=""
seq_num=$(echo "$acct_result" | jq -r '.result.account_data.Sequence' 2>/dev/null || echo "unknown")
log "  Genesis account sequence: $seq_num"

# Submit payment
submit_result=$(curl -sf --max-time "$CURL_MAX_TIME" "http://localhost:$RPC_PORT_BASE" \
    -d "{\"method\":\"submit\",\"params\":[{\"secret\":\"$GENESIS_SEED\",\"tx_json\":{\"TransactionType\":\"Payment\",\"Account\":\"$GENESIS_ACCOUNT\",\"Destination\":\"$DEST_ACCOUNT\",\"Amount\":\"10000000\"}}]}") || submit_result=""

engine_result=$(echo "$submit_result" | jq -r '.result.engine_result' 2>/dev/null || echo "unknown")
tx_hash=$(echo "$submit_result" | jq -r '.result.tx_json.hash' 2>/dev/null || echo "unknown")

if [ "$engine_result" = "tesSUCCESS" ] || [ "$engine_result" = "terQUEUED" ]; then
    ok "Transaction submitted: $engine_result (hash: ${tx_hash:0:16}...)"
else
    fail "Transaction submission: $engine_result"
    log "  Full response: $(echo "$submit_result" | jq -c .result 2>/dev/null)"
fi

log "Waiting 15s for consensus round + batch export..."
sleep 15

# ---------------------------------------------------------------------------
# Step 9: Verify Tempo traces
# ---------------------------------------------------------------------------
log "Verifying spans in Tempo..."

# Check service registration
services=$(curl -sf --max-time "$CURL_MAX_TIME" "$TEMPO/api/v2/search/tag/resource.service.name/values" |
    jq -r '.tagValues[].value' 2>/dev/null || echo "")
if echo "$services" | grep -q "xrpld"; then
    ok "Service 'xrpld' registered in Tempo"
else
    fail "Service 'xrpld' NOT found in Tempo (found: $services)"
fi

log ""
log "--- RPC Spans ---"
check_span "rpc.http_request"
check_span "rpc.process"
check_span "rpc.command.server_info"
check_span "rpc.command.server_state"
check_span "rpc.command.ledger"

log ""
log "--- Transaction Spans ---"
check_span "tx.process"
check_span "tx.receive"
check_span "tx.apply"

log ""
log "--- Consensus Spans ---"
check_span "consensus.proposal.send"
check_span "consensus.ledger_close"
check_span "consensus.accept"
check_span "consensus.validation.send"

log ""
log "--- Ledger Spans ---"
check_span "ledger.build"
check_span "ledger.validate"
check_span "ledger.store"

log ""
log "--- Peer Spans (trace_peer=1) ---"
check_span "peer.proposal.receive"
check_span "peer.validation.receive"

# ---------------------------------------------------------------------------
# Step 9b: Verify log-trace correlation
# ---------------------------------------------------------------------------
log ""
log "--- Log-Trace Correlation ---"
check_log_correlation

# ---------------------------------------------------------------------------
# Step 10: Verify Prometheus span_metrics
# ---------------------------------------------------------------------------
log ""
log "--- Spanmetrics ---"
log "Waiting 20s for Prometheus scrape cycle..."
sleep 20

# Names come from the spanmetrics connector's `namespace: "span"` in
# otel-collector-config.yaml. Without that namespace the connector emits
# traces_span_metrics_*, so these queries must move whenever it changes.
calls_count=$(curl -sf --max-time "$CURL_MAX_TIME" "$PROM/api/v1/query?query=span_calls_total" |
    jq '.data.result | length' 2>/dev/null || echo 0)
if [ "$calls_count" -gt 0 ]; then
    ok "Prometheus: span_calls_total ($calls_count series)"
else
    fail "Prometheus: span_calls_total (0 series)"
fi

duration_count=$(curl -sf --max-time "$CURL_MAX_TIME" "$PROM/api/v1/query?query=span_duration_milliseconds_count" |
    jq '.data.result | length' 2>/dev/null || echo 0)
if [ "$duration_count" -gt 0 ]; then
    ok "Prometheus: duration histogram ($duration_count series)"
else
    fail "Prometheus: duration histogram (0 series)"
fi

# Check Grafana
if curl -sf --max-time "$CURL_MAX_TIME" http://localhost:3000/api/health >/dev/null 2>&1; then
    ok "Grafana: healthy at localhost:3000"
else
    fail "Grafana: not reachable at localhost:3000"
fi

# ---------------------------------------------------------------------------
# Step 10b: Verify native OTel metrics in Prometheus (beast::insight)
# ---------------------------------------------------------------------------
log ""
log "--- Native OTel Metrics (beast::insight via OTLP) ---"
log "Waiting 20s for OTLP metric export + Prometheus scrape..."
sleep 20

check_otel_metric() {
    local metric_name="$1"
    local result
    result=$(curl -sf --max-time "$CURL_MAX_TIME" "$PROM/api/v1/query?query=$metric_name" |
        jq '.data.result | length' 2>/dev/null || echo 0)
    if [ "$result" -gt 0 ]; then
        ok "OTel: $metric_name ($result series)"
    else
        fail "OTel: $metric_name (0 series)"
    fi
}

# Names are what OTelCollector::formatName() produces: the beast::insight
# name lowercased with '.' and ' ' mapped to '_', any group() segment kept, and
# no prefix. The [insight] prefix knob is logged at startup and never applied on
# this path, and the collector's prometheus exporter sets no namespace, so a
# name carrying a product prefix or capitals cannot match any exported series.

# Node health gauges (ObservableGauge — no _total suffix)
check_otel_metric "ledgermaster_validated_ledger_age"
check_otel_metric "ledgermaster_published_ledger_age"
check_otel_metric "jobq_job_count"

# State accounting
check_otel_metric "state_accounting_full_duration"

# Peer finder
check_otel_metric "peer_finder_active_inbound_peers"
check_otel_metric "peer_finder_active_outbound_peers"

# RPC counters (Counter — Prometheus adds _total suffix automatically)
check_otel_metric "rpc_requests_total"

# Overlay traffic — one series per TrafficCount category; "total" is the
# aggregate category.
check_otel_metric "total_bytes_in"

# Verify StatsD receiver is NOT required (no statsd receiver in pipeline)
log ""
log "--- Verify StatsD receiver is not required ---"
# StatsD listens on UDP 8125, so probe with a UDP-aware tool, not curl (TCP).
if command -v ss >/dev/null 2>&1; then
    if ss -ulnp 2>/dev/null | grep -q ":8125"; then
        fail "StatsD port 8125 appears to be listening (should not be needed)"
    else
        ok "StatsD port 8125 is not listening (not required)"
    fi
else
    log "ss not found -- skipping StatsD UDP port check"
fi

# ---------------------------------------------------------------------------
# Step 10c: Verify OTel SDK Metrics
# ---------------------------------------------------------------------------
log ""
log "--- OTel SDK Metrics (MetricsRegistry) ---"
log "Waiting 15s for OTel metric export + Prometheus scrape..."
sleep 15

check_otel_metric() {
    local metric_name="$1"
    local result
    result=$(curl -sf "$PROM/api/v1/query?query=$metric_name" |
        jq '.data.result | length' 2>/dev/null || echo 0)
    if [ "$result" -gt 0 ]; then
        ok "OTel: $metric_name ($result series)"
    else
        fail "OTel: $metric_name (0 series)"
    fi
}

# NodeStore I/O
check_otel_metric 'nodestore_state{metric="node_reads_total"}'
check_otel_metric 'nodestore_state{metric="write_load"}'

# Cache hit rates
check_otel_metric 'cache_metrics{metric="SLE_hit_rate"}'
check_otel_metric 'cache_metrics{metric="treenode_cache_size"}'

# TxQ metrics
check_otel_metric 'txq_metrics{metric="txq_count"}'
check_otel_metric 'txq_metrics{metric="txq_reference_fee_level"}'

# Per-RPC metrics
check_otel_metric "rpc_method_started_total"
check_otel_metric "rpc_method_finished_total"

# Per-job metrics
check_otel_metric "job_queued_total"
check_otel_metric "job_finished_total"

# Counted object instances
check_otel_metric "object_count"

# Load factor breakdown
check_otel_metric 'load_factor_metrics{metric="load_factor"}'
check_otel_metric 'load_factor_metrics{metric="load_factor_server"}'

# ValidationTracker rolling-window agreement gauge.
# MetricsRegistry::registerValidationAgreementGauge() publishes
# validation_agreement with a `metric` label for each window
# (1h / 24h / 7d) plus the matching agreement/miss counts. The 7-day
# window matches the external xrpl-validator-dashboard parity target.
check_otel_metric 'validation_agreement{metric="agreement_pct_1h"}'
check_otel_metric 'validation_agreement{metric="agreement_pct_24h"}'
check_otel_metric 'validation_agreement{metric="agreement_pct_7d"}'
check_otel_metric 'validation_agreement{metric="agreements_1h"}'
check_otel_metric 'validation_agreement{metric="missed_1h"}'
check_otel_metric 'validation_agreement{metric="agreements_24h"}'
check_otel_metric 'validation_agreement{metric="missed_24h"}'
check_otel_metric 'validation_agreement{metric="agreements_7d"}'
check_otel_metric 'validation_agreement{metric="missed_7d"}'

# ---------------------------------------------------------------------------
# Step 11: Summary
# ---------------------------------------------------------------------------

# All checks are done, so the run counts as complete: keep the stack and the
# nodes up for inspection even when some checks failed.
RUN_COMPLETED=1

echo ""
echo "==========================================================="
echo "  INTEGRATION TEST RESULTS"
echo "==========================================================="
printf "  \033[1;32mPASSED: %d\033[0m\n" "$PASS"
printf "  \033[1;31mFAILED: %d\033[0m\n" "$FAIL"
echo "==========================================================="
echo ""
echo "  Observability stack is running:"
echo ""
echo "    Tempo:         http://localhost:3200"
echo "    Grafana:       http://localhost:3000"
echo "    Prometheus:    http://localhost:9090"
echo "    Loki:          http://localhost:3100"
echo ""
echo "  xrpld nodes (6) are running:"
for i in $(seq 1 "$NUM_NODES"); do
    RPC_PORT=$((RPC_PORT_BASE + i - 1))
    PEER_PORT=$((PEER_PORT_BASE + i - 1))
    echo "    Node $i: RPC=localhost:$RPC_PORT  Peer=:$PEER_PORT  PID=$(cat "$WORKDIR/Node-$i/xrpld.pid" 2>/dev/null || echo 'unknown')"
done
echo ""
echo "  To tear down:"
echo "    bash docker/telemetry/integration-test.sh --cleanup"
echo ""
echo "==========================================================="

if [ "$FAIL" -gt 0 ]; then
    exit 1
fi