mirror of
https://github.com/XRPLF/rippled.git
synced 2026-10-04 02:38:02 +00:00
50eff17dd4
The comment measured date +%s%N cost 'on a dev box'; say 'on one Linux host' instead. The number is the point, not where it was taken.
409 lines
18 KiB
Bash
Executable File
409 lines
18 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# collect_system_metrics.sh — Collect CPU, memory, and RPC latency metrics
|
|
# from running xrpld nodes for benchmark comparison.
|
|
#
|
|
# Samples system metrics at regular intervals and writes a JSON summary.
|
|
# Used by benchmark.sh for baseline vs telemetry comparison.
|
|
#
|
|
# Usage:
|
|
# ./collect_system_metrics.sh <rpc_ports_csv> <duration_seconds> <output_file> [pids_csv]
|
|
#
|
|
# pids_csv narrows process sampling to exactly those pids. Without it the scope
|
|
# is every xrpld on the host, which averages in any other cluster's nodes and
|
|
# reports the largest of them as the RSS peak. benchmark.sh passes its own pids
|
|
# because run-full-validation.sh leaves five validation nodes running while the
|
|
# benchmark's three start, and diluting the arms alike hides the delta.
|
|
#
|
|
# Example:
|
|
# ./collect_system_metrics.sh "5005,5006,5007" 300 /tmp/metrics-baseline.json
|
|
# ./collect_system_metrics.sh "5020,5021,5022" 120 /tmp/m.json "8801,8802,8803"
|
|
#
|
|
# Output JSON format:
|
|
# {
|
|
# "cpu_pct_avg": 12.5,
|
|
# "memory_rss_mb_peak": 450.2,
|
|
# "rpc_p99_ms": 15.3,
|
|
# "tps": 4.8,
|
|
# "consensus_round_mean_ms": 3200,
|
|
# "metrics_complete": true,
|
|
# "samples": 60
|
|
# }
|
|
#
|
|
# Exit codes:
|
|
# 0 Every metric was measured; "metrics_complete" is true.
|
|
# 1 Cannot run at all: bad arguments, no GNU date with %N, or a failed
|
|
# process sample. No output file is written.
|
|
# 3 Output file was written, but at least one measurement source was empty,
|
|
# so the affected metrics are 0 placeholders and "metrics_complete" is
|
|
# false. Callers must treat this as inconclusive, never as a pass.
|
|
|
|
set -euo pipefail
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Colored output helpers
|
|
# ---------------------------------------------------------------------------
|
|
log() { printf "\033[1;34m[METRICS]\033[0m %s\n" "$*"; }
|
|
ok() { printf "\033[1;32m[METRICS]\033[0m %s\n" "$*"; }
|
|
# Warnings go to stderr so they never mix into the JSON echoed on stdout.
|
|
warn() { printf "\033[1;33m[METRICS]\033[0m %s\n" "$*" >&2; }
|
|
die() {
|
|
printf "\033[1;31m[METRICS]\033[0m %s\n" "$*" >&2
|
|
exit 1
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Argument parsing
|
|
# ---------------------------------------------------------------------------
|
|
usage() {
|
|
echo "Usage: $0 <rpc_ports_csv> <duration_seconds> <output_file>"
|
|
echo ""
|
|
echo "Arguments:"
|
|
echo " rpc_ports_csv Comma-separated RPC ports (e.g., 5005,5006,5007)"
|
|
echo " duration_seconds How long to collect metrics"
|
|
echo " output_file Path to write JSON results"
|
|
echo " pids_csv Optional: sample only these pids, not every host xrpld"
|
|
exit 1
|
|
}
|
|
|
|
if [ $# -lt 3 ]; then
|
|
usage
|
|
fi
|
|
|
|
RPC_PORTS_CSV="$1"
|
|
DURATION="$2"
|
|
OUTPUT_FILE="$3"
|
|
PIDS_CSV="${4:-}"
|
|
|
|
# Reject a malformed pid list rather than silently sampling the whole host,
|
|
# which is the outcome this argument exists to prevent.
|
|
case "$PIDS_CSV" in
|
|
'') ;;
|
|
*[!0-9,]*) die "pids_csv must be a comma-separated list of pids, got '$PIDS_CSV'" ;;
|
|
esac
|
|
|
|
IFS=',' read -ra RPC_PORTS <<<"$RPC_PORTS_CSV"
|
|
|
|
# consensus_round_mean_ms below counts DISTINCT ledger sequences seen by this
|
|
# loop, so it cannot resolve a close interval shorter than SAMPLE_INTERVAL. At
|
|
# 5 s it read back 5000 ms for every close time from 2 s to 5 s, so a 10%
|
|
# regression measured 0% and the benchmark's 1% consensus gate could never fire.
|
|
# 2 s is the close-time floor itself (ledgerMinClose, ConsensusParms.h:93), which
|
|
# is enough: measured against this file's own arithmetic, a 10% regression shows
|
|
# up as at least 9.3% anywhere in the 2 s to 5 s band. 1 s only doubles the
|
|
# probe load for slightly worse numbers.
|
|
SAMPLE_INTERVAL=2
|
|
|
|
# Hard ceiling on every RPC probe below. curl applies no overall timeout of its
|
|
# own, so a node that accepts the connection and then stops answering — what a
|
|
# stalled job queue looks like from outside — parks the sampling loop for the
|
|
# rest of the run. 5 s is some thousands of times a healthy server_info, so it
|
|
# bounds a wedged node's cost to a couple of lost samples while never
|
|
# truncating a real reply. A probe that hits the ceiling exits
|
|
# non-zero and is therefore skipped rather than recorded, which is the same
|
|
# rule the latency loop already applies to a refused connection; if every
|
|
# probe hits it, the empty file trips the placeholder warning below.
|
|
CURL_MAX_TIME=5
|
|
|
|
# Reject anything the sample arithmetic cannot use, instead of silently
|
|
# treating it as 0.
|
|
case "$DURATION" in
|
|
'' | *[!0-9]*) die "duration_seconds must be a positive integer, got '$DURATION'" ;;
|
|
esac
|
|
|
|
# Normalise to base 10 once, right after the digits check. Bash arithmetic
|
|
# reads a leading zero as octal, so "08" aborted with "value too great for
|
|
# base" and "0100" was silently taken as 64. Doing it here also keeps the
|
|
# value a valid JSON number in the output below, where "08" is not.
|
|
DURATION=$((10#$DURATION))
|
|
|
|
# Round up, so a duration shorter than one interval still takes one sample
|
|
# rather than truncating to zero and emitting an all-zero JSON.
|
|
SAMPLES=$(((DURATION + SAMPLE_INTERVAL - 1) / SAMPLE_INTERVAL))
|
|
if [ "$SAMPLES" -lt 1 ]; then
|
|
die "duration_seconds=$DURATION yields $SAMPLES samples; need at least 1"
|
|
fi
|
|
|
|
log "Collecting metrics for ${DURATION}s (${SAMPLES} samples, ${#RPC_PORTS[@]} nodes)..."
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Nanosecond clock
|
|
#
|
|
# GNU date supports "+%s%N". BSD/macOS date has no %N and echoes it back
|
|
# literally, which used to abort the sampling loop under `set -e` and still
|
|
# exit 0 with an all-zero JSON — a silent false pass.
|
|
#
|
|
# The clock has to be cheap as well as precise, because the latency it
|
|
# measures is compared against a 2 ms threshold. Measured on one Linux host:
|
|
# `date +%s%N` costs ~1.2 ms per call, forking python3 for the same value ~13 ms.
|
|
# Two calls bracket every request, so a python3 fallback would add ~26 ms of
|
|
# its own overhead to a 2 ms budget and make the number meaningless. There is
|
|
# no cheap alternative worth having, so probe once and refuse to run without
|
|
# GNU date rather than report a figure that is quietly an order of magnitude
|
|
# wrong.
|
|
# ---------------------------------------------------------------------------
|
|
if [[ ! "$(date +%s%N 2>/dev/null)" =~ ^[0-9]+$ ]]; then
|
|
die "GNU coreutils date with %N is required for RPC latency timing; this date does not support it"
|
|
fi
|
|
|
|
# Echo the current time in nanoseconds since the epoch.
|
|
now_ns() {
|
|
date +%s%N
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Temporary files for aggregation
|
|
# ---------------------------------------------------------------------------
|
|
TMPDIR_METRICS="$(mktemp -d)"
|
|
CPU_FILE="$TMPDIR_METRICS/cpu.txt"
|
|
MEM_FILE="$TMPDIR_METRICS/mem.txt"
|
|
RPC_FILE="$TMPDIR_METRICS/rpc.txt"
|
|
LEDGER_FILE="$TMPDIR_METRICS/ledger.txt"
|
|
|
|
touch "$CPU_FILE" "$MEM_FILE" "$RPC_FILE" "$LEDGER_FILE"
|
|
|
|
cleanup() {
|
|
rm -rf "$TMPDIR_METRICS"
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Get initial ledger sequence for TPS calculation
|
|
# ---------------------------------------------------------------------------
|
|
INITIAL_SEQ=0
|
|
INITIAL_TIME=$(date +%s)
|
|
for port in "${RPC_PORTS[@]}"; do
|
|
seq=$(curl -sf --max-time "$CURL_MAX_TIME" "http://localhost:$port" \
|
|
-d '{"method":"server_info"}' 2>/dev/null |
|
|
jq -r '.result.info.validated_ledger.seq // 0' 2>/dev/null || echo 0)
|
|
if [ "$seq" -gt "$INITIAL_SEQ" ]; then
|
|
INITIAL_SEQ=$seq
|
|
fi
|
|
done
|
|
log "Initial validated ledger seq: $INITIAL_SEQ"
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Sampling loop
|
|
# ---------------------------------------------------------------------------
|
|
for sample in $(seq 1 "$SAMPLES"); do
|
|
# Sample CPU% and RSS for the xrpld processes. One ps pass feeds both
|
|
# files, so the two numbers always come from the same instant.
|
|
#
|
|
# Selection is on argv[0]'s basename, NOT on "the command line mentions
|
|
# xrpld". The loose form matched every bystander whose command line
|
|
# happened to contain the string: this harness's own launcher (invoked as
|
|
# `--xrpld .build/xrpld`, whose argv[0] is bash), an editor's clangd, and
|
|
# any shell sitting in a directory with xrpld in its path. Measured
|
|
# against a live 5-node cluster, that pulled in 8-15 processes instead of
|
|
# 5, halved the CPU average with idle bystanders, and reported clangd's
|
|
# 1.1 GB RSS as xrpld's peak against a 5 MB threshold. `ps -C xrpld` is
|
|
# not an alternative: xrpld renames itself, so its comm is "xrpld-main"
|
|
# and -C matches nothing. "rippled" is accepted alongside "xrpld" so a
|
|
# rename of the binary cannot silently zero the collector.
|
|
#
|
|
# Scope is the pid list when one was given, and the whole host otherwise.
|
|
# Host scope averages in any other cluster's xrpld and reports the largest
|
|
# of them as the RSS peak, which reads as noise against a 5 MB threshold.
|
|
#
|
|
# A %cpu of exactly 0.0 is a real reading and is counted — dropping idle
|
|
# samples would inflate the average — while non-numeric output is
|
|
# rejected by the pattern. An RSS of 0 is not a live process, so it
|
|
# contributes no memory sample; counting it would leave the file non-empty
|
|
# and mark a dead cluster's 0 MB peak as a complete measurement.
|
|
ps -eo pid=,%cpu=,rss=,args= |
|
|
awk -v cpu_file="$CPU_FILE" -v mem_file="$MEM_FILE" -v pids="$PIDS_CSV" '
|
|
BEGIN { if (pids != "") { n = split(pids, a, ","); for (i = 1; i <= n; i++) want[a[i]] = 1 } }
|
|
pids != "" && !($1 in want) { next }
|
|
pids == "" && $4 !~ /(^|\/)(xrpld|rippled)$/ { next }
|
|
$2 ~ /^[0-9]+(\.[0-9]+)?$/ { cpu_sum += $2; cpu_n++ }
|
|
$3 ~ /^[0-9]+$/ && $3 + 0 > 0 { printf("%.2f\n", $3 / 1024) >> mem_file }
|
|
END {
|
|
# With an explicit pid list the expected count is known, so a
|
|
# dead node is detectable. Without it, exit 1 would fire on any
|
|
# host with no xrpld at all, which the empty-file check below
|
|
# already reports.
|
|
if (pids != "" && cpu_n != n) exit 1
|
|
if (cpu_n > 0) printf("%.2f\n", cpu_sum / cpu_n) >> cpu_file
|
|
}
|
|
' || die "process sampling failed on sample $sample/$SAMPLES: expected $(echo "$PIDS_CSV" | tr -cd , | wc -c)+1 live pids"
|
|
|
|
# Collect RPC latency from each node. Only a successful call is a latency
|
|
# measurement: a refused connection returns in well under a millisecond,
|
|
# and recording that as ~0 ms would pull the reported p99 down.
|
|
for port in "${RPC_PORTS[@]}"; do
|
|
start_ns=$(now_ns)
|
|
if curl -sf --max-time "$CURL_MAX_TIME" "http://localhost:$port" \
|
|
-d '{"method":"server_info"}' >/dev/null 2>&1; then
|
|
end_ns=$(now_ns)
|
|
latency_ms=$(((end_ns - start_ns) / 1000000))
|
|
echo "$latency_ms" >>"$RPC_FILE"
|
|
fi
|
|
done
|
|
|
|
# Record current validated ledger seq.
|
|
for port in "${RPC_PORTS[@]}"; do
|
|
seq=$(curl -sf --max-time "$CURL_MAX_TIME" "http://localhost:$port" \
|
|
-d '{"method":"server_info"}' 2>/dev/null |
|
|
jq -r '.result.info.validated_ledger.seq // 0' 2>/dev/null || echo 0)
|
|
echo "$seq" >>"$LEDGER_FILE"
|
|
break # Only need one node's seq per sample.
|
|
done
|
|
|
|
# Progress indicator.
|
|
if [ $((sample % 10)) -eq 0 ]; then
|
|
log " Sample $sample/$SAMPLES..."
|
|
fi
|
|
|
|
sleep "$SAMPLE_INTERVAL"
|
|
done
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Compute aggregated metrics
|
|
# ---------------------------------------------------------------------------
|
|
log "Computing aggregated metrics..."
|
|
|
|
# Cleared by any empty measurement source. A 0 metric is otherwise
|
|
# indistinguishable from a real reading, so the flag is exported in the JSON
|
|
# and drives the exit-3 contract documented in the header.
|
|
METRICS_COMPLETE=true
|
|
|
|
# CPU average.
|
|
if [ -s "$CPU_FILE" ]; then
|
|
CPU_AVG=$(awk '{ sum += $1; n++ } END { if (n>0) printf "%.2f", sum/n; else print "0" }' "$CPU_FILE")
|
|
else
|
|
# Now that the selector cannot match this harness's own processes, an
|
|
# empty file means no xrpld process was running for any sample.
|
|
warn "No CPU samples collected (no xrpld process matched); cpu_pct_avg is a 0 placeholder"
|
|
CPU_AVG="0"
|
|
METRICS_COMPLETE=false
|
|
fi
|
|
|
|
# Memory peak RSS (MB).
|
|
if [ -s "$MEM_FILE" ]; then
|
|
MEM_PEAK=$(sort -n "$MEM_FILE" | tail -1)
|
|
else
|
|
warn "No memory samples collected (no xrpld process matched); memory_rss_mb_peak is a 0 placeholder"
|
|
MEM_PEAK="0"
|
|
METRICS_COMPLETE=false
|
|
fi
|
|
|
|
# RPC latency p99 (ms).
|
|
if [ -s "$RPC_FILE" ]; then
|
|
RPC_COUNT=$(wc -l <"$RPC_FILE")
|
|
# Nearest-rank p99: ceil(count * 99 / 100), clamped into [1, count].
|
|
# Integer arithmetic avoids both the floor bias of the old bc expression
|
|
# and the bc dependency, and the lower clamp keeps sed off line address 0
|
|
# (a file with no trailing newline makes wc -l report 0).
|
|
P99_INDEX=$(((RPC_COUNT * 99 + 99) / 100))
|
|
if [ "$P99_INDEX" -lt 1 ]; then
|
|
P99_INDEX=1
|
|
fi
|
|
if [ "$P99_INDEX" -gt "$RPC_COUNT" ]; then
|
|
P99_INDEX="$RPC_COUNT"
|
|
fi
|
|
RPC_P99=$(sort -n "$RPC_FILE" | sed -n "${P99_INDEX}p")
|
|
if [ -z "$RPC_P99" ]; then
|
|
warn "RPC latency file has no line $P99_INDEX; rpc_p99_ms is a 0 placeholder"
|
|
RPC_P99="0"
|
|
METRICS_COMPLETE=false
|
|
fi
|
|
else
|
|
warn "No successful RPC probes; rpc_p99_ms is a 0 placeholder"
|
|
RPC_P99="0"
|
|
METRICS_COMPLETE=false
|
|
fi
|
|
|
|
# TPS calculation from ledger sequence advancement.
|
|
FINAL_SEQ=0
|
|
for port in "${RPC_PORTS[@]}"; do
|
|
seq=$(curl -sf --max-time "$CURL_MAX_TIME" "http://localhost:$port" \
|
|
-d '{"method":"server_info"}' 2>/dev/null |
|
|
jq -r '.result.info.validated_ledger.seq // 0' 2>/dev/null || echo 0)
|
|
if [ "$seq" -gt "$FINAL_SEQ" ]; then
|
|
FINAL_SEQ=$seq
|
|
fi
|
|
done
|
|
FINAL_TIME=$(date +%s)
|
|
ELAPSED=$((FINAL_TIME - INITIAL_TIME))
|
|
LEDGER_ADVANCE=$((FINAL_SEQ - INITIAL_SEQ))
|
|
if [ "$ELAPSED" -gt 0 ] && [ "$LEDGER_ADVANCE" -gt 0 ]; then
|
|
# Rough TPS: assume ~avg_txs_per_ledger * ledgers / elapsed.
|
|
# Without tx count, use ledger close rate as proxy.
|
|
#
|
|
# awk rather than bc, because bc omits the leading zero: `scale=2` prints
|
|
# ".25", not "0.25", and a bare ".25" is not valid JSON. A value below 1 is
|
|
# the normal case here, not an edge case — ledgers close every few seconds,
|
|
# so advance/elapsed is well under 1 for any realistic window. jq happens
|
|
# to accept the malformed form, which is why it survived earlier checks,
|
|
# but a strict parser rejects the whole file. awk's %.2f always pads.
|
|
TPS=$(awk -v a="$LEDGER_ADVANCE" -v b="$ELAPSED" 'BEGIN { printf "%.2f", a / b }')
|
|
else
|
|
# No ledger advance means the cluster produced nothing to divide, so 0 here
|
|
# is an absence of data, not a measured rate. Flagged like every other empty
|
|
# source so the run reports itself inconclusive instead of a real 0 TPS.
|
|
warn "Ledger seq did not advance (advance=$LEDGER_ADVANCE over ${ELAPSED}s); tps is a 0 placeholder"
|
|
TPS="0"
|
|
METRICS_COMPLETE=false
|
|
fi
|
|
|
|
# Mean inter-ledger interval in ms: ELAPSED / (distinct ledgers - 1) * 1000.
|
|
#
|
|
# ELAPSED, not DURATION: DURATION is what the caller asked for, while the loop
|
|
# also spends time on its own probes, so it always runs longer. TPS above uses
|
|
# ELAPSED for the same reason.
|
|
#
|
|
# This is a MEAN, not a percentile — the JSON key says so. It still cannot
|
|
# resolve a close interval at or below SAMPLE_INTERVAL, because LEDGER_FILE
|
|
# gets at most one sequence per sample. Read it as a coarse trend.
|
|
if [ -s "$LEDGER_FILE" ]; then
|
|
UNIQUE_LEDGERS=$(sort -u "$LEDGER_FILE" | wc -l)
|
|
# The > 1 test also keeps the divisor below at 1 or more.
|
|
if [ "$UNIQUE_LEDGERS" -gt 1 ]; then
|
|
# awk rather than bc, for the same reason as the TPS calculation above.
|
|
# bc is an optional package, and the `|| echo "0"` this replaces turned
|
|
# a missing or failing bc into a 0 that read exactly like a measured
|
|
# value: no warn(), and METRICS_COMPLETE left true, so the run reported
|
|
# a complete measurement and exited 0. awk is already required by the
|
|
# sampling loop and the CPU average, so computing this with awk removes
|
|
# the failure path rather than reporting it. The divisor is >= 1 by the
|
|
# test above.
|
|
CONSENSUS_MEAN=$(awk -v d="$ELAPSED" -v u="$UNIQUE_LEDGERS" \
|
|
'BEGIN { printf "%.0f", d * 1000 / (u - 1) }')
|
|
else
|
|
warn "Ledger seq never advanced ($UNIQUE_LEDGERS distinct); consensus_round_mean_ms is a 0 placeholder"
|
|
CONSENSUS_MEAN="0"
|
|
METRICS_COMPLETE=false
|
|
fi
|
|
else
|
|
warn "No ledger samples collected; consensus_round_mean_ms is a 0 placeholder"
|
|
CONSENSUS_MEAN="0"
|
|
METRICS_COMPLETE=false
|
|
fi
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Write output JSON
|
|
# ---------------------------------------------------------------------------
|
|
cat >"$OUTPUT_FILE" <<EOF_JSON
|
|
{
|
|
"cpu_pct_avg": $CPU_AVG,
|
|
"memory_rss_mb_peak": $MEM_PEAK,
|
|
"rpc_p99_ms": $RPC_P99,
|
|
"tps": $TPS,
|
|
"consensus_round_mean_ms": $CONSENSUS_MEAN,
|
|
"metrics_complete": $METRICS_COMPLETE,
|
|
"samples": $SAMPLES,
|
|
"duration_seconds": $DURATION,
|
|
"node_count": ${#RPC_PORTS[@]},
|
|
"initial_ledger_seq": $INITIAL_SEQ,
|
|
"final_ledger_seq": $FINAL_SEQ
|
|
}
|
|
EOF_JSON
|
|
|
|
ok "Metrics written to $OUTPUT_FILE"
|
|
cat "$OUTPUT_FILE"
|
|
|
|
# The file is always written first so CI still has an artifact to publish.
|
|
if [ "$METRICS_COMPLETE" != "true" ]; then
|
|
warn "metrics_complete=false — treat this run as inconclusive (exit 3)"
|
|
exit 3
|
|
fi
|