diff --git a/docker/telemetry/TESTING.md b/docker/telemetry/TESTING.md index f43eb2d1cb..f3192088ae 100644 --- a/docker/telemetry/TESTING.md +++ b/docker/telemetry/TESTING.md @@ -244,12 +244,9 @@ online_delete=256 /tmp/xrpld-integration/validators.txt [ips_fixed] -127.0.0.1 51235 -127.0.0.1 51236 -127.0.0.1 51237 -127.0.0.1 51238 -127.0.0.1 51239 -127.0.0.1 51240 +{one "127.0.0.1 " line for each port in 51235-51240 except this node's +own 51234 + node_number — a node must not list itself as a fixed peer, so +each config carries five lines, not six} [peer_private] 1 @@ -478,7 +475,7 @@ Pre-configured datasources: ss -tlnp | grep ":$p " && echo "port $p in use" done ``` -2. Verify `[ips_fixed]` lists all 6 peer ports +2. Verify `[ips_fixed]` lists the 5 other peer ports, and not the node's own 3. Verify `validators.txt` has all 6 public keys 4. Check node debug logs: `tail -50 /tmp/xrpld-integration/node1/debug.log` 5. Ensure `[peer_private]` is set to `1` (prevents reaching out to public network) diff --git a/docker/telemetry/integration-test.sh b/docker/telemetry/integration-test.sh index d9a95dd063..b0df20f01c 100755 --- a/docker/telemetry/integration-test.sh +++ b/docker/telemetry/integration-test.sh @@ -38,10 +38,21 @@ DEST_ACCOUNT="" # Generated dynamically via wallet_propose TEMPO="http://localhost:3200" PROM="http://localhost:9090" +# Hard ceiling on every curl probe below. curl has no overall timeout of its +# own, so a server that accepts the connection and then never answers parks a +# poll loop forever and its attempt count stops bounding anything. 5 s is well +# above a healthy reply, so only a wedged server hits the ceiling. +CURL_MAX_TIME=5 + # Counters for pass/fail PASS=0 FAIL=0 +# Unix seconds just before this run's nodes start. Every Tempo search is +# bounded to this run, so a previous run's traces cannot satisfy an assertion. +# Set in Step 5; check_span refuses to run while it is empty. +RUN_START="" + # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- @@ -65,8 +76,16 @@ check_span() { # -G is required: it moves the urlencoded params into the query string. # Without it curl POSTs them as a request body, and Tempo answers 200 # while ignoring the query — so every span name would look present. - count=$(curl -sfG "$TEMPO/api/search" \ + # + # start/end bound the search to this run. Tempo keeps blocks for + # block_retention (tempo.yaml, 1h) on a named volume, so without a bound + # an older run's spans answer for this one. The end margin covers spans + # exported while this query is in flight. + [ -n "$RUN_START" ] || die "check_span called before RUN_START was set" + count=$(curl -sfG --max-time "$CURL_MAX_TIME" "$TEMPO/api/search" \ --data-urlencode "q={resource.service.name=\"xrpld\" && name=\"$op\"}" \ + --data-urlencode "start=$RUN_START" \ + --data-urlencode "end=$(($(date +%s) + 60))" \ --data-urlencode "limit=5" | jq '.traces | length' 2>/dev/null || echo 0) if [ "$count" -gt 0 ]; then @@ -161,7 +180,7 @@ for attempt in $(seq 1 30); do # The OTLP HTTP endpoint returns 405 for GET (expects POST), which # means it is listening. curl -sf would fail on 405, so we check # the HTTP status code explicitly. - status=$(curl -so /dev/null -w '%{http_code}' http://localhost:4318/ 2>/dev/null || echo 000) + status=$(curl -so /dev/null -w '%{http_code}' --max-time "$CURL_MAX_TIME" http://localhost:4318/ 2>/dev/null || echo 000) if [ "$status" != "000" ]; then log "otel-collector ready (attempt $attempt, HTTP $status)." break @@ -174,7 +193,7 @@ done log "Waiting for Tempo to be ready..." for attempt in $(seq 1 30); do - if curl -sf "$TEMPO/ready" >/dev/null 2>&1; then + if curl -sf --max-time "$CURL_MAX_TIME" "$TEMPO/ready" >/dev/null 2>&1; then log "Tempo ready (attempt $attempt)." break fi @@ -225,7 +244,7 @@ TEMP_PID=$! log "Temporary xrpld started (PID $TEMP_PID), waiting for RPC..." for attempt in $(seq 1 30); do - if curl -sf http://localhost:5099 -d '{"method":"server_info"}' >/dev/null 2>&1; then + if curl -sf --max-time "$CURL_MAX_TIME" http://localhost:5099 -d '{"method":"server_info"}' >/dev/null 2>&1; then log "Temporary xrpld RPC ready (attempt $attempt)." break fi @@ -240,7 +259,7 @@ declare -a SEEDS declare -a PUBKEYS for i in $(seq 1 "$NUM_NODES"); do - result=$(curl -sf http://localhost:5099 -d '{"method":"validation_create"}') + result=$(curl -sf --max-time "$CURL_MAX_TIME" http://localhost:5099 -d '{"method":"validation_create"}') seed=$(echo "$result" | jq -r '.result.validation_seed') pubkey=$(echo "$result" | jq -r '.result.validation_public_key') if [ -z "$seed" ] || [ "$seed" = "null" ]; then @@ -365,6 +384,10 @@ done # --------------------------------------------------------------------------- log "Starting $NUM_NODES xrpld nodes..." +# Lower bound for every Tempo search below. Only these nodes have a +# [telemetry] section, so nothing before this instant belongs to this run. +RUN_START=$(date +%s) + for i in $(seq 1 "$NUM_NODES"); do NODE_DIR="$WORKDIR/node$i" "$XRPLD" --conf "$NODE_DIR/xrpld.cfg" --start >"$NODE_DIR/stdout.log" 2>&1 & @@ -396,7 +419,7 @@ while [ "$nodes_ready" -lt "$NUM_NODES" ]; do nodes_ready=0 for i in $(seq 1 "$NUM_NODES"); do RPC_PORT=$((RPC_PORT_BASE + i - 1)) - state=$(curl -sf "http://localhost:$RPC_PORT" \ + state=$(curl -sf --max-time "$CURL_MAX_TIME" "http://localhost:$RPC_PORT" \ -d '{"method":"server_info"}' 2>/dev/null | jq -r '.result.info.server_state' 2>/dev/null || echo "unreachable") if [ "$state" = "proposing" ]; then @@ -424,7 +447,7 @@ fi # --------------------------------------------------------------------------- log "Waiting for first validated ledger..." for attempt in $(seq 1 60); do - val_seq=$(curl -sf "http://localhost:$RPC_PORT_BASE" \ + val_seq=$(curl -sf --max-time "$CURL_MAX_TIME" "http://localhost:$RPC_PORT_BASE" \ -d '{"method":"server_info"}' 2>/dev/null | jq -r '.result.info.validated_ledger.seq // 0' 2>/dev/null || echo 0) if [ "$val_seq" -gt 2 ] 2>/dev/null; then @@ -442,11 +465,11 @@ done # --------------------------------------------------------------------------- log "Exercising RPC spans..." -curl -sf "http://localhost:$RPC_PORT_BASE" \ +curl -sf --max-time "$CURL_MAX_TIME" "http://localhost:$RPC_PORT_BASE" \ -d '{"method":"server_info"}' >/dev/null -curl -sf "http://localhost:$RPC_PORT_BASE" \ +curl -sf --max-time "$CURL_MAX_TIME" "http://localhost:$RPC_PORT_BASE" \ -d '{"method":"server_state"}' >/dev/null -curl -sf "http://localhost:$RPC_PORT_BASE" \ +curl -sf --max-time "$CURL_MAX_TIME" "http://localhost:$RPC_PORT_BASE" \ -d '{"method":"ledger","params":[{"ledger_index":"current"}]}' >/dev/null log "RPC commands sent. Waiting 5s for batch export..." @@ -461,7 +484,7 @@ log "Submitting Payment transaction..." log " Generating destination wallet..." # Guarded: under set -e an unguarded curl failure would abort the whole # script, so the fallback below could never run. -wallet_result=$(curl -sf "http://localhost:$RPC_PORT_BASE" \ +wallet_result=$(curl -sf --max-time "$CURL_MAX_TIME" "http://localhost:$RPC_PORT_BASE" \ -d '{"method":"wallet_propose"}') || wallet_result="" DEST_ACCOUNT=$(echo "$wallet_result" | jq -r '.result.account_id' 2>/dev/null || echo "") if [ -z "$DEST_ACCOUNT" ] || [ "$DEST_ACCOUNT" = "null" ]; then @@ -471,13 +494,13 @@ fi log " Destination: $DEST_ACCOUNT" # Get genesis account info -acct_result=$(curl -sf "http://localhost:$RPC_PORT_BASE" \ +acct_result=$(curl -sf --max-time "$CURL_MAX_TIME" "http://localhost:$RPC_PORT_BASE" \ -d "{\"method\":\"account_info\",\"params\":[{\"account\":\"$GENESIS_ACCOUNT\"}]}") || acct_result="" seq_num=$(echo "$acct_result" | jq -r '.result.account_data.Sequence' 2>/dev/null || echo "unknown") log " Genesis account sequence: $seq_num" # Submit payment -submit_result=$(curl -sf "http://localhost:$RPC_PORT_BASE" \ +submit_result=$(curl -sf --max-time "$CURL_MAX_TIME" "http://localhost:$RPC_PORT_BASE" \ -d "{\"method\":\"submit\",\"params\":[{\"secret\":\"$GENESIS_SEED\",\"tx_json\":{\"TransactionType\":\"Payment\",\"Account\":\"$GENESIS_ACCOUNT\",\"Destination\":\"$DEST_ACCOUNT\",\"Amount\":\"10000000\"}}]}") || submit_result="" engine_result=$(echo "$submit_result" | jq -r '.result.engine_result' 2>/dev/null || echo "unknown") @@ -499,7 +522,7 @@ sleep 15 log "Verifying spans in Tempo..." # Check service registration -services=$(curl -sf "$TEMPO/api/v2/search/tag/resource.service.name/values" | +services=$(curl -sf --max-time "$CURL_MAX_TIME" "$TEMPO/api/v2/search/tag/resource.service.name/values" | jq -r '.tagValues[].value' 2>/dev/null || echo "") if echo "$services" | grep -q "xrpld"; then ok "Service 'xrpld' registered in Tempo" @@ -550,7 +573,7 @@ sleep 20 # Names come from the spanmetrics connector's `namespace: "span"` in # otel-collector-config.yaml. Without that namespace the connector emits # traces_span_metrics_*, so these queries must move whenever it changes. -calls_count=$(curl -sf "$PROM/api/v1/query?query=span_calls_total" | +calls_count=$(curl -sf --max-time "$CURL_MAX_TIME" "$PROM/api/v1/query?query=span_calls_total" | jq '.data.result | length' 2>/dev/null || echo 0) if [ "$calls_count" -gt 0 ]; then ok "Prometheus: span_calls_total ($calls_count series)" @@ -558,7 +581,7 @@ else fail "Prometheus: span_calls_total (0 series)" fi -duration_count=$(curl -sf "$PROM/api/v1/query?query=span_duration_milliseconds_count" | +duration_count=$(curl -sf --max-time "$CURL_MAX_TIME" "$PROM/api/v1/query?query=span_duration_milliseconds_count" | jq '.data.result | length' 2>/dev/null || echo 0) if [ "$duration_count" -gt 0 ]; then ok "Prometheus: duration histogram ($duration_count series)" @@ -567,7 +590,7 @@ else fi # Check Grafana -if curl -sf http://localhost:3000/api/health >/dev/null 2>&1; then +if curl -sf --max-time "$CURL_MAX_TIME" http://localhost:3000/api/health >/dev/null 2>&1; then ok "Grafana: healthy at localhost:3000" else fail "Grafana: not reachable at localhost:3000" @@ -584,7 +607,7 @@ sleep 20 check_otel_metric() { local metric_name="$1" local result - result=$(curl -sf "$PROM/api/v1/query?query=$metric_name" | + result=$(curl -sf --max-time "$CURL_MAX_TIME" "$PROM/api/v1/query?query=$metric_name" | jq '.data.result | length' 2>/dev/null || echo 0) if [ "$result" -gt 0 ]; then ok "OTel: $metric_name ($result series)"