From bcf2d098d36833fe75aa7287e682fad141f487f3 Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Wed, 9 Sep 2026 12:25:17 +0100 Subject: [PATCH 01/16] fix(telemetry): require https for metrics_endpoint under mTLS With tls_client_cert set, only traces_endpoint was checked for an https scheme. Telemetry::makeMetricExporter() attaches the client certificate and key to the metric exporter whenever use_tls=1, and metrics_endpoint defaults to a plain http URL, so an operator who set up mTLS and overrode only traces_endpoint exported every metric in the clear with the configured client identity unused. Check both endpoints, and state the requirement under metrics_endpoint and tls_client_cert in the example config. Four config tests cover an explicit http metrics endpoint, the omitted-key default, both endpoints on https, and a one-way-TLS control that must stay accepted. --- cfg/xrpld-example.cfg | 19 +++-- src/libxrpl/telemetry/TelemetryConfig.cpp | 15 ++-- .../libxrpl/telemetry/TelemetryConfig.cpp | 78 +++++++++++++++++++ 3 files changed, 101 insertions(+), 11 deletions(-) diff --git a/cfg/xrpld-example.cfg b/cfg/xrpld-example.cfg index 93c430b0c2..3116a01420 100644 --- a/cfg/xrpld-example.cfg +++ b/cfg/xrpld-example.cfg @@ -1762,12 +1762,13 @@ validators.txt # To enable mTLS, both tls_client_cert and tls_client_key must be # specified. If only one is provided, xrpld will fail to start. Providing # them while use_tls=0 also fails to start, rather than being ignored. -# traces_endpoint must be an https:// URL, because that scheme is what -# makes the exporter present the certificate at all. With use_tls=1 each -# path is opened at startup, so one that does not exist or cannot be read -# fails to start too, rather than failing later as an opaque TLS -# handshake error. All four checks apply only when enabled=1; with -# telemetry disabled these settings are read but never validated. +# traces_endpoint and metrics_endpoint must both be https:// URLs, because +# that scheme is what makes each exporter present the certificate at all. +# With use_tls=1 each path is opened at startup, so one that does not +# exist or cannot be read fails to start too, rather than failing later +# as an opaque TLS handshake error. All four checks apply only when +# enabled=1; with telemetry disabled these settings are read but never +# validated. # # tls_client_key= # @@ -1853,6 +1854,12 @@ validators.txt # defaults. # Default: http://localhost:4318/v1/metrics. # +# The scheme decides encryption here exactly as it does for +# traces_endpoint, so setting tls_client_cert requires this URL to start +# with https:// as well — including leaving it at the default above. +# Overriding only traces_endpoint therefore makes xrpld fail to start, +# rather than export metrics without the client identity. +# # metric_export_interval_ms=1000 # # Gap in milliseconds between two metric exports. Must be greater than 0. diff --git a/src/libxrpl/telemetry/TelemetryConfig.cpp b/src/libxrpl/telemetry/TelemetryConfig.cpp index 86fb632185..469ffc3fc9 100644 --- a/src/libxrpl/telemetry/TelemetryConfig.cpp +++ b/src/libxrpl/telemetry/TelemetryConfig.cpp @@ -387,13 +387,18 @@ makeTelemetrySetup( } // Still inside the enabled branch, and checked before the files are - // opened so a scheme problem is not hidden behind a path problem. The - // exporter reads TLS off the endpoint scheme, so a client certificate is - // only presented on an https endpoint. tls_ca_cert is left out of this - // check: it only names a trust store, while a client certificate is this - // node's own identity and has to reach the collector to mean anything. + // opened so a scheme problem is not hidden behind a path problem. Each + // exporter reads TLS off its own endpoint scheme, and both are handed + // the client certificate, so both endpoints have to be https. Checking + // only one leaves the other signal exporting in the clear without this + // node's identity. tls_ca_cert is left out of this check: it only names + // a trust store, while a client certificate is this node's own identity + // and has to reach the collector to mean anything. if (!setup.tlsClientCertPath.empty()) + { requireHttpsEndpoint(setup.tracesEndpoint, key::tracesEndpoint); + requireHttpsEndpoint(setup.metricsEndpoint, key::metricsEndpoint); + } // Still inside the enabled branch. The exporter opens these files only // when TLS is on, so check them only then: a bad path behind use_tls=0 diff --git a/src/tests/libxrpl/telemetry/TelemetryConfig.cpp b/src/tests/libxrpl/telemetry/TelemetryConfig.cpp index da39b6eab4..f8ecaa7de7 100644 --- a/src/tests/libxrpl/telemetry/TelemetryConfig.cpp +++ b/src/tests/libxrpl/telemetry/TelemetryConfig.cpp @@ -79,6 +79,19 @@ constexpr char const* httpsEndpoint = "https://collector:4318/v1/traces"; constexpr char const* defaultEndpoint = "http://localhost:4318/v1/traces"; constexpr char const* schemeError = "must start with 'https://'"; +/** + * The same four values for the metric signal. + * + * The guard covers both endpoints, so every case that expects parsing to + * succeed has to set this key too. Spelled separately from the trace values so + * a case can put one signal on https and the other on http, which is the + * configuration that used to pass. + */ +constexpr char const* keyMetricsEndpoint = "metrics_endpoint"; +constexpr char const* metricsHttpEndpoint = "http://collector:4318/v1/metrics"; +constexpr char const* metricsHttpsEndpoint = "https://collector:4318/v1/metrics"; +constexpr char const* defaultMetricsEndpoint = "http://localhost:4318/v1/metrics"; + /** * Build a [telemetry] section carrying only the `enabled` key. * @@ -366,6 +379,7 @@ TEST(TelemetryConfig, mtls_cert_and_key_both_set) Section section = mtls::makeSection(true); section.set("use_tls", "1"); section.set(mtls::keyEndpoint, mtls::httpsEndpoint); + section.set(mtls::keyMetricsEndpoint, mtls::metricsHttpsEndpoint); section.set(mtls::keyClientCert, cert); section.set(mtls::keyClientKey, key); @@ -568,6 +582,7 @@ TEST(TelemetryConfig, tls_readable_files_are_accepted) Section section = mtls::makeSection(true); section.set("use_tls", "1"); section.set(mtls::keyEndpoint, mtls::httpsEndpoint); + section.set(mtls::keyMetricsEndpoint, mtls::metricsHttpsEndpoint); section.set("tls_ca_cert", ca); section.set(mtls::keyClientCert, cert); section.set(mtls::keyClientKey, key); @@ -666,22 +681,85 @@ TEST(TelemetryConfig, mtls_client_cert_on_an_https_endpoint_is_accepted) { // The same configuration as the two cases above with only the scheme // changed, so nothing but the scheme can explain the different outcome. + // Both endpoints are https, which is the only shape the guard accepts. TempDir const dir; auto const cert = mtls::writeCertFile(dir.file("c.pem")); auto const key = mtls::writeCertFile(dir.file("k.pem")); Section section = mtls::makeSection(true); section.set("use_tls", "1"); section.set(mtls::keyEndpoint, mtls::httpsEndpoint); + section.set(mtls::keyMetricsEndpoint, mtls::metricsHttpsEndpoint); section.set(mtls::keyClientCert, cert); section.set(mtls::keyClientKey, key); telemetry::Telemetry::Setup setup; ASSERT_NO_THROW(setup = mtls::parseSection(section)); EXPECT_EQ(setup.tracesEndpoint, mtls::httpsEndpoint); + EXPECT_EQ(setup.metricsEndpoint, mtls::metricsHttpsEndpoint); EXPECT_EQ(setup.tlsClientCertPath, cert); EXPECT_EQ(setup.tlsClientKeyPath, key); } +TEST(TelemetryConfig, mtls_client_cert_on_a_plain_http_metrics_endpoint_throws) +{ + // traces_endpoint is https and only metrics_endpoint is not, so the trace + // guard cannot be what fires. Before the metric endpoint was checked this + // configuration started the node and exported every metric in the clear, + // with the client certificate attached to the exporter and never used. + TempDir const dir; + Section section = mtls::makeSection(true); + section.set("use_tls", "1"); + section.set(mtls::keyEndpoint, mtls::httpsEndpoint); + section.set(mtls::keyMetricsEndpoint, mtls::metricsHttpEndpoint); + section.set(mtls::keyClientCert, mtls::writeCertFile(dir.file("c.pem"))); + section.set(mtls::keyClientKey, mtls::writeCertFile(dir.file("k.pem"))); + + EXPECT_THAT( + [§ion] { mtls::parseSection(section); }, + ThrowsMessage(AllOf( + HasSubstr(mtls::schemeError), + HasSubstr(mtls::keyMetricsEndpoint), + HasSubstr(mtls::metricsHttpEndpoint)))); +} + +TEST(TelemetryConfig, mtls_client_cert_with_the_default_metrics_endpoint_throws) +{ + // The key is absent, so the built-in default applies, and that default is a + // plain http URL. This is the shape an operator reaches by setting up mTLS + // and overriding only traces_endpoint, which makes it the case worth having. + TempDir const dir; + Section section = mtls::makeSection(true); + section.set("use_tls", "1"); + section.set(mtls::keyEndpoint, mtls::httpsEndpoint); + section.set(mtls::keyClientCert, mtls::writeCertFile(dir.file("c.pem"))); + section.set(mtls::keyClientKey, mtls::writeCertFile(dir.file("k.pem"))); + + EXPECT_THAT( + [§ion] { mtls::parseSection(section); }, + ThrowsMessage(AllOf( + HasSubstr(mtls::schemeError), + HasSubstr(mtls::keyMetricsEndpoint), + HasSubstr(mtls::defaultMetricsEndpoint)))); +} + +TEST(TelemetryConfig, one_way_tls_on_a_plain_http_metrics_endpoint_is_accepted) +{ + // The control for the metric guard's scope, matching the trace one below: + // same plain http metrics endpoint and use_tls=1, but no client identity to + // lose. Widen the guard to every use_tls=1 node and this case starts failing. + TempDir const dir; + auto const ca = mtls::writeCertFile(dir.file("ca.pem")); + Section section = mtls::makeSection(true); + section.set("use_tls", "1"); + section.set(mtls::keyMetricsEndpoint, mtls::metricsHttpEndpoint); + section.set("tls_ca_cert", ca); + + telemetry::Telemetry::Setup setup; + ASSERT_NO_THROW(setup = mtls::parseSection(section)); + EXPECT_EQ(setup.metricsEndpoint, mtls::metricsHttpEndpoint); + EXPECT_TRUE(setup.tlsClientCertPath.empty()); +} + TEST(TelemetryConfig, mtls_scheme_check_is_case_sensitive_like_the_exporter) { // The exporter compares the scheme byte for byte, so "HTTPS://" leaves it From 697653927422c40fdee9827d24783050836a60fa Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Wed, 9 Sep 2026 13:13:41 +0100 Subject: [PATCH 02/16] fix(telemetry): bound the integration-test probes and align its config template curl applies no overall timeout of its own, so a server that accepts the connection and then stops answering parks a poll loop for the rest of the run and the loop's attempt count stops bounding anything. Add a CURL_MAX_TIME ceiling and apply it to all 18 executable probes in integration-test.sh. TESTING.md's manual node-config template also disagreed with what the script writes, so a reader following it could not reproduce the automated path: - no [insight] stanza, so no beast::insight metric leaves the node at all and Step 10b's ten rippled_* assertions cannot pass - [ips_fixed] listed all six peer ports including the node's own The log level is deliberately untouched: the template and the script agree on warning here. --- docker/telemetry/TESTING.md | 18 +++++++----- docker/telemetry/integration-test.sh | 42 ++++++++++++++++------------ 2 files changed, 35 insertions(+), 25 deletions(-) diff --git a/docker/telemetry/TESTING.md b/docker/telemetry/TESTING.md index 01843a00d7..9e610b3cf2 100644 --- a/docker/telemetry/TESTING.md +++ b/docker/telemetry/TESTING.md @@ -244,12 +244,9 @@ online_delete=256 /tmp/xrpld-integration/validators.txt [ips_fixed] -127.0.0.1 51235 -127.0.0.1 51236 -127.0.0.1 51237 -127.0.0.1 51238 -127.0.0.1 51239 -127.0.0.1 51240 +{one "127.0.0.1 " line for each port in 51235-51240 except this node's +own 51234 + node_number — a node must not list itself as a fixed peer, so +each config carries five lines, not six} [peer_private] 1 @@ -266,6 +263,13 @@ trace_consensus=1 trace_peer=1 trace_ledger=1 +[insight] +server=statsd +address=127.0.0.1:8125 +prefix={the same prefix the [insight] block in integration-test.sh sets — it +becomes the Prometheus metric-name prefix, so any other value renames every +beast::insight metric} + [rpc_startup] { "command": "log_level", "severity": "warning" } @@ -470,7 +474,7 @@ Pre-configured datasources: ss -tlnp | grep ":$p " && echo "port $p in use" done ``` -2. Verify `[ips_fixed]` lists all 6 peer ports +2. Verify `[ips_fixed]` lists the 5 other peer ports, and not the node's own 3. Verify `validators.txt` has all 6 public keys 4. Check node debug logs: `tail -50 /tmp/xrpld-integration/node1/debug.log` 5. Ensure `[peer_private]` is set to `1` (prevents reaching out to public network) diff --git a/docker/telemetry/integration-test.sh b/docker/telemetry/integration-test.sh index ddc9e79522..af5a29339c 100755 --- a/docker/telemetry/integration-test.sh +++ b/docker/telemetry/integration-test.sh @@ -38,6 +38,12 @@ DEST_ACCOUNT="" # Generated dynamically via wallet_propose TEMPO="http://localhost:3200" PROM="http://localhost:9090" +# Hard ceiling on every curl probe below. curl has no overall timeout of its +# own, so a server that accepts the connection and then never answers parks a +# poll loop forever and its attempt count stops bounding anything. 5 s is well +# above a healthy reply, so only a wedged server hits the ceiling. +CURL_MAX_TIME=5 + # Counters for pass/fail PASS=0 FAIL=0 @@ -76,7 +82,7 @@ check_span() { # block_retention (tempo.yaml, 1h) on a named volume, so without a bound # an older run's spans answer for this one. The end margin covers spans # exported while this query is in flight. - count=$(curl -sfG "$TEMPO/api/search" \ + count=$(curl -sfG --max-time "$CURL_MAX_TIME" "$TEMPO/api/search" \ --data-urlencode "q={resource.service.name=\"xrpld\" && name=\"$op\"}" \ --data-urlencode "start=$RUN_START" \ --data-urlencode "end=$(($(date +%s) + 60))" \ @@ -179,7 +185,7 @@ for attempt in $(seq 1 30); do # The OTLP HTTP endpoint returns 405 for GET (expects POST), which # means it is listening. curl -sf would fail on 405, so we check # the HTTP status code explicitly. - status=$(curl -so /dev/null -w '%{http_code}' http://localhost:4318/ 2>/dev/null || echo 000) + status=$(curl -so /dev/null -w '%{http_code}' --max-time "$CURL_MAX_TIME" http://localhost:4318/ 2>/dev/null || echo 000) if [ "$status" != "000" ]; then log "otel-collector ready (attempt $attempt, HTTP $status)." break @@ -192,7 +198,7 @@ done log "Waiting for Tempo to be ready..." for attempt in $(seq 1 30); do - if curl -sf "$TEMPO/ready" >/dev/null 2>&1; then + if curl -sf --max-time "$CURL_MAX_TIME" "$TEMPO/ready" >/dev/null 2>&1; then log "Tempo ready (attempt $attempt)." break fi @@ -243,7 +249,7 @@ TEMP_PID=$! log "Temporary xrpld started (PID $TEMP_PID), waiting for RPC..." for attempt in $(seq 1 30); do - if curl -sf http://localhost:5099 -d '{"method":"server_info"}' >/dev/null 2>&1; then + if curl -sf --max-time "$CURL_MAX_TIME" http://localhost:5099 -d '{"method":"server_info"}' >/dev/null 2>&1; then log "Temporary xrpld RPC ready (attempt $attempt)." break fi @@ -258,7 +264,7 @@ declare -a SEEDS declare -a PUBKEYS for i in $(seq 1 "$NUM_NODES"); do - result=$(curl -sf http://localhost:5099 -d '{"method":"validation_create"}') + result=$(curl -sf --max-time "$CURL_MAX_TIME" http://localhost:5099 -d '{"method":"validation_create"}') seed=$(echo "$result" | jq -r '.result.validation_seed') pubkey=$(echo "$result" | jq -r '.result.validation_public_key') if [ -z "$seed" ] || [ "$seed" = "null" ]; then @@ -414,7 +420,7 @@ while [ "$nodes_ready" -lt "$NUM_NODES" ]; do nodes_ready=0 for i in $(seq 1 "$NUM_NODES"); do RPC_PORT=$((RPC_PORT_BASE + i - 1)) - state=$(curl -sf "http://localhost:$RPC_PORT" \ + state=$(curl -sf --max-time "$CURL_MAX_TIME" "http://localhost:$RPC_PORT" \ -d '{"method":"server_info"}' 2>/dev/null | jq -r '.result.info.server_state' 2>/dev/null || echo "unreachable") if [ "$state" = "proposing" ]; then @@ -442,7 +448,7 @@ fi # --------------------------------------------------------------------------- log "Waiting for first validated ledger..." for attempt in $(seq 1 60); do - val_seq=$(curl -sf "http://localhost:$RPC_PORT_BASE" \ + val_seq=$(curl -sf --max-time "$CURL_MAX_TIME" "http://localhost:$RPC_PORT_BASE" \ -d '{"method":"server_info"}' 2>/dev/null | jq -r '.result.info.validated_ledger.seq // 0' 2>/dev/null || echo 0) if [ "$val_seq" -gt 2 ] 2>/dev/null; then @@ -460,11 +466,11 @@ done # --------------------------------------------------------------------------- log "Exercising RPC spans..." -curl -sf "http://localhost:$RPC_PORT_BASE" \ +curl -sf --max-time "$CURL_MAX_TIME" "http://localhost:$RPC_PORT_BASE" \ -d '{"method":"server_info"}' >/dev/null -curl -sf "http://localhost:$RPC_PORT_BASE" \ +curl -sf --max-time "$CURL_MAX_TIME" "http://localhost:$RPC_PORT_BASE" \ -d '{"method":"server_state"}' >/dev/null -curl -sf "http://localhost:$RPC_PORT_BASE" \ +curl -sf --max-time "$CURL_MAX_TIME" "http://localhost:$RPC_PORT_BASE" \ -d '{"method":"ledger","params":[{"ledger_index":"current"}]}' >/dev/null log "RPC commands sent. Waiting 5s for batch export..." @@ -479,7 +485,7 @@ log "Submitting Payment transaction..." log " Generating destination wallet..." # Guarded: under set -e an unguarded curl failure would abort the whole # script, so the fallback below could never run. -wallet_result=$(curl -sf "http://localhost:$RPC_PORT_BASE" \ +wallet_result=$(curl -sf --max-time "$CURL_MAX_TIME" "http://localhost:$RPC_PORT_BASE" \ -d '{"method":"wallet_propose"}') || wallet_result="" DEST_ACCOUNT=$(echo "$wallet_result" | jq -r '.result.account_id' 2>/dev/null || echo "") if [ -z "$DEST_ACCOUNT" ] || [ "$DEST_ACCOUNT" = "null" ]; then @@ -489,13 +495,13 @@ fi log " Destination: $DEST_ACCOUNT" # Get genesis account info -acct_result=$(curl -sf "http://localhost:$RPC_PORT_BASE" \ +acct_result=$(curl -sf --max-time "$CURL_MAX_TIME" "http://localhost:$RPC_PORT_BASE" \ -d "{\"method\":\"account_info\",\"params\":[{\"account\":\"$GENESIS_ACCOUNT\"}]}") || acct_result="" seq_num=$(echo "$acct_result" | jq -r '.result.account_data.Sequence' 2>/dev/null || echo "unknown") log " Genesis account sequence: $seq_num" # Submit payment -submit_result=$(curl -sf "http://localhost:$RPC_PORT_BASE" \ +submit_result=$(curl -sf --max-time "$CURL_MAX_TIME" "http://localhost:$RPC_PORT_BASE" \ -d "{\"method\":\"submit\",\"params\":[{\"secret\":\"$GENESIS_SEED\",\"tx_json\":{\"TransactionType\":\"Payment\",\"Account\":\"$GENESIS_ACCOUNT\",\"Destination\":\"$DEST_ACCOUNT\",\"Amount\":\"10000000\"}}]}") || submit_result="" engine_result=$(echo "$submit_result" | jq -r '.result.engine_result' 2>/dev/null || echo "unknown") @@ -517,7 +523,7 @@ sleep 15 log "Verifying spans in Tempo..." # Check service registration -services=$(curl -sf "$TEMPO/api/v2/search/tag/resource.service.name/values" | +services=$(curl -sf --max-time "$CURL_MAX_TIME" "$TEMPO/api/v2/search/tag/resource.service.name/values" | jq -r '.tagValues[].value' 2>/dev/null || echo "") # Whole-line match: a substring match would also accept a value that merely # contains "xrpld". This endpoint ignores start/end (measured), so its only @@ -568,7 +574,7 @@ log "--- Spanmetrics ---" log "Waiting 20s for Prometheus scrape cycle..." sleep 20 -calls_count=$(curl -sf "$PROM/api/v1/query?query=traces_span_metrics_calls_total" | +calls_count=$(curl -sf --max-time "$CURL_MAX_TIME" "$PROM/api/v1/query?query=traces_span_metrics_calls_total" | jq '.data.result | length' 2>/dev/null || echo 0) if [ "$calls_count" -gt 0 ]; then ok "Prometheus: traces_span_metrics_calls_total ($calls_count series)" @@ -576,7 +582,7 @@ else fail "Prometheus: traces_span_metrics_calls_total (0 series)" fi -duration_count=$(curl -sf "$PROM/api/v1/query?query=traces_span_metrics_duration_milliseconds_count" | +duration_count=$(curl -sf --max-time "$CURL_MAX_TIME" "$PROM/api/v1/query?query=traces_span_metrics_duration_milliseconds_count" | jq '.data.result | length' 2>/dev/null || echo 0) if [ "$duration_count" -gt 0 ]; then ok "Prometheus: duration histogram ($duration_count series)" @@ -585,7 +591,7 @@ else fi # Check Grafana -if curl -sf http://localhost:3000/api/health >/dev/null 2>&1; then +if curl -sf --max-time "$CURL_MAX_TIME" http://localhost:3000/api/health >/dev/null 2>&1; then ok "Grafana: healthy at localhost:3000" else fail "Grafana: not reachable at localhost:3000" @@ -602,7 +608,7 @@ sleep 20 check_statsd_metric() { local metric_name="$1" local result - result=$(curl -sf "$PROM/api/v1/query?query=$metric_name" | + result=$(curl -sf --max-time "$CURL_MAX_TIME" "$PROM/api/v1/query?query=$metric_name" | jq '.data.result | length' 2>/dev/null || echo 0) if [ "$result" -gt 0 ]; then ok "StatsD: $metric_name ($result series)" From d972772747151393e63435962035c3738d77b911 Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Wed, 9 Sep 2026 13:14:15 +0100 Subject: [PATCH 03/16] docs(telemetry): match the manual test template to the OTel insight path This branch switches integration-test.sh to [insight] server=otel and adds an assertion that no StatsD listener is needed, but TESTING.md still described the metrics it verifies as StatsD-derived and its manual node-config template had no [insight] stanza at all. CollectorManagerImp falls through to NullCollector when server is neither statsd nor otel, so a reader building configs from that template got zero beast::insight metrics. The template also omitted service_instance_id and metrics_endpoint, which the script writes; without the former every node is indistinguishable in the $node dashboard filter. --- docker/telemetry/TESTING.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docker/telemetry/TESTING.md b/docker/telemetry/TESTING.md index cd2040c1c1..f43eb2d1cb 100644 --- a/docker/telemetry/TESTING.md +++ b/docker/telemetry/TESTING.md @@ -163,7 +163,7 @@ Run the integration test script: bash docker/telemetry/integration-test.sh ``` -It checks prerequisites, clears the previous run, brings up the observability stack, generates six validator key pairs and their node configs, starts the nodes, waits for consensus and then for a validated ledger, exercises RPC and submits a transaction, verifies traces in Tempo and both the spanmetrics and the StatsD-derived metrics in Prometheus, then prints a summary and leaves the stack running. +It checks prerequisites, clears the previous run, brings up the observability stack, generates six validator key pairs and their node configs, starts the nodes, waits for consensus and then for a validated ledger, exercises RPC and submits a transaction, verifies traces in Tempo and both the spanmetrics and the native `beast::insight` metrics that arrive over OTLP in Prometheus, checks that no StatsD listener is needed, then prints a summary and leaves the stack running. The script announces each step as it runs, so read its `Step N:` headers for the authoritative sequence — they are not restated here, because a numbered copy of them drifts as soon as a step is added. @@ -256,7 +256,9 @@ online_delete=256 [telemetry] enabled=1 +service_instance_id=Node-{N} traces_endpoint=http://localhost:4318/v1/traces +metrics_endpoint=http://localhost:4318/v1/metrics batch_size=512 batch_delay_ms=2000 max_queue_size=2048 @@ -266,6 +268,10 @@ trace_consensus=1 trace_peer=1 trace_ledger=1 +[insight] +server=otel +endpoint=http://localhost:4318/v1/metrics + [rpc_startup] { "command": "log_level", "severity": "warning" } From 8d2e2d15af374f7a5702c9d83af5137f84244cff Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Wed, 9 Sep 2026 13:14:21 +0100 Subject: [PATCH 04/16] docs(telemetry): correct stale code citations and the log-correlation claim Every citation below was checked against the file it names: - LedgerMaster.cpp:463 is fixIndex, not the ledger.store span; that guard is at :470 and the insert it wraps at :476 - LedgerMaster.cpp:987 is the tvc assignment, which sits BEFORE the tvc < minVal return at :988; the ledger.validate span opens at :1003 - ServerHandler.cpp:705 is inside makeJsonError; processRequest is at :718 - docker-compose.yml:71 and :75 are comments in the collector's volume block; the loki service is at :112 and its config command at :116 Two claims were also wrong rather than merely stale. Log-trace correlation is gated in CI, because the workflow passes no --skip-loki, and the separate check in integration-test.sh is run by no workflow at all. The Loki label note described the Grafana Cloud collector config rather than the local one: only the cloud variant sets job=xrpld, and the local config's own comment says to select on service_name. The dashboards carry 35 Loki queries, not 38. --- .../05-configuration-reference.md | 2 +- OpenTelemetryPlan/06-implementation-phases.md | 11 ++++++++--- OpenTelemetryPlan/07-observability-backends.md | 2 +- OpenTelemetryPlan/Phase10_taskList.md | 5 +++-- docker/telemetry/TESTING.md | 18 ++++++++++-------- docs/telemetry-runbook.md | 6 +++--- 6 files changed, 26 insertions(+), 18 deletions(-) diff --git a/OpenTelemetryPlan/05-configuration-reference.md b/OpenTelemetryPlan/05-configuration-reference.md index 29614bac71..ab473e667c 100644 --- a/OpenTelemetryPlan/05-configuration-reference.md +++ b/OpenTelemetryPlan/05-configuration-reference.md @@ -584,7 +584,7 @@ Fluentd or PerfLog change. Two pieces: > only an **allow-listed** set of resource attributes to indexed stream labels > (`service.name`, `service.namespace`, `service.instance.id`, > `deployment.environment`, the `k8s.*`/`cloud.*` keys); `job` is not on that -> list, and this repo ships no Loki config override — `docker-compose.yml:75` +> list, and this repo ships no Loki config override — `docker-compose.yml:116` > starts Loki with the image's built-in `/etc/loki/local-config.yaml`. `job` > therefore lands in **structured metadata**, which cannot appear in a stream > selector, so `{job="xrpld"}` returns an empty result rather than an error. diff --git a/OpenTelemetryPlan/06-implementation-phases.md b/OpenTelemetryPlan/06-implementation-phases.md index 7ebcc77d1b..850ccf982d 100644 --- a/OpenTelemetryPlan/06-implementation-phases.md +++ b/OpenTelemetryPlan/06-implementation-phases.md @@ -734,13 +734,18 @@ flowchart LR absent or the context is invalid (`Log.cpp:310-318`) - [x] Loki ingests xrpld logs via OTel Collector filelog receiver — `otel-collector-config.yaml:38` (`filelog`); `loki` service in - `docker-compose.yml:71` + `docker-compose.yml:112` - [x] Grafana Tempo → Loki one-click correlation works — `provisioning/datasources/tempo.yaml:32` (`tracesToLogs`) - [x] Grafana Loki → Tempo reverse lookup works via derived field — `provisioning/datasources/loki.yaml:16` (`derivedFields`) -- [ ] Integration test verifies trace_id presence in logs — implemented in the - Phase 10 harness, but CI runs it with `--skip-loki`, so it is not gated +- [ ] Integration test verifies trace_id presence in logs — CI gates this + through the Phase 10 harness's `validate_telemetry.py`, whose + `log.trace_id_present` and `log.trace_id_cross_reference` checks run + because the workflow passes no `--skip-loki`. That harness and + `.github/workflows/telemetry-validation.yml` live on the Phase 10 branch, + not here. `docker/telemetry/integration-test.sh:79-126` carries a separate + trace_id-in-logs check that no workflow under `.github/workflows/` runs - [ ] No performance regression from trace_id injection (< 0.1% overhead) — needs the Phase 10 benchmark suite diff --git a/OpenTelemetryPlan/07-observability-backends.md b/OpenTelemetryPlan/07-observability-backends.md index daa7fa9693..8dcfe61982 100644 --- a/OpenTelemetryPlan/07-observability-backends.md +++ b/OpenTelemetryPlan/07-observability-backends.md @@ -507,7 +507,7 @@ These are journal (`debug.log`) lines, not PerfLog lines — see §7.7.2. > **allow-listed** set of resource attributes to indexed stream labels > (`service.name`, `service.namespace`, `service.instance.id`, > `deployment.environment`, `k8s.*`, `cloud.*`), and `job` is not on it. This -> repo mounts no Loki config override (`docker-compose.yml:75` uses the image's +> repo mounts no Loki config override (`docker-compose.yml:116` uses the image's > built-in `local-config.yaml`), so `job` lands in **structured metadata** — > queryable only with a `|` filter after a selector, never as the selector > itself. A `{job="xrpld"}` query returns empty with no error, which is why this diff --git a/OpenTelemetryPlan/Phase10_taskList.md b/OpenTelemetryPlan/Phase10_taskList.md index 28a1cd28fb..b985b054f7 100644 --- a/OpenTelemetryPlan/Phase10_taskList.md +++ b/OpenTelemetryPlan/Phase10_taskList.md @@ -150,7 +150,7 @@ Before Phases 1-9 can be considered production-ready, we need proof that: - HTTP: `rpc.http_request` → `rpc.process` → `rpc.command.*` - WebSocket: `rpc.ws_message` → `rpc.command.*` — **there is no `rpc.process` on the WS path**. `rpc.process` is created only in - `ServerHandler::processRequest()` (`ServerHandler.cpp:705`), reached from + `ServerHandler::processRequest()` (`ServerHandler.cpp:718`), reached from `processSession(Session, coro)`, i.e. HTTP only. Under WS-only load `rpc.process` never appears, and `rpc.command.*` parents directly to `rpc.ws_message`. @@ -276,7 +276,8 @@ Before Phases 1-9 can be considered production-ready, we need proof that: (totals computed dynamically from `expected_spans.json` / `expected_metrics.json`, not the stale 16 / 22 figures) - [ ] Log-trace correlation validated end-to-end (Loki ↔ Tempo) — implemented, - but CI runs with `--skip-loki`, so it is not gated + and gated in CI: the workflow passes no `--skip-loki`, so + `validate_telemetry.py` builds and runs both log-correlation checks - [ ] All 14 harness-asserted Grafana dashboards render data (no empty panels); 15 on disk - [ ] Benchmark shows < 3% CPU overhead, < 5MB memory overhead diff --git a/docker/telemetry/TESTING.md b/docker/telemetry/TESTING.md index 73daa140c4..412db2d331 100644 --- a/docker/telemetry/TESTING.md +++ b/docker/telemetry/TESTING.md @@ -631,20 +631,22 @@ curl -sG "http://localhost:3100/loki/api/v1/query" \ Expected: > 0 results. -> **Use `service_name`, not `job`.** The collector's `resource/logs` processor -> applies an `upsert` to **both** `service.name=xrpld` and `job=xrpld` -> (`otel-collector-config.yaml:57-70`), and its comment says the `job` attribute -> is there so operators can paste `{job="xrpld"}`. That does not work: on OTLP -> ingest Loki promotes only an allow-listed set of resource attributes to indexed -> stream labels (`service.name` → `service_name`, plus `service.namespace`, +> **Use `service_name`, not `job`.** The local stack's `resource/logs` processor +> sets one key, `service.name=xrpld` (`otel-collector-config.yaml:84-86`); its +> comment there explains that a custom `job` attribute is not promoted to a +> stream label and tells you to select on `service_name`. Only the Grafana Cloud +> variant also sets `job=xrpld` (`otel-collector-config.grafanacloud.yaml:73-75`). +> Either way `{job="xrpld"}` does not work as a selector: on OTLP ingest Loki +> promotes only an allow-listed set of resource attributes to indexed stream +> labels (`service.name` → `service_name`, plus `service.namespace`, > `service.instance.id`, `deployment.environment`, `k8s.*`, `cloud.*`), and `job` > is not on the list. This repo mounts no Loki config override — the `loki` > service runs the image's built-in `/etc/loki/local-config.yaml` -> (`docker-compose.yml:75`) — so `job` lands in **structured metadata**, which +> (`docker-compose.yml:116`) — so `job` lands in **structured metadata**, which > cannot be a stream selector. `{job="xrpld"}` therefore returns **zero results > with no error**, which reads exactly like "logs are not being ingested". If > this query is empty, check `{service_name="xrpld"}` before debugging the -> pipeline. All 38 Loki queries in the shipped dashboards select on +> pipeline. All 35 Loki queries in the shipped dashboards select on > `service_name`; none uses `job`. ### Step 4: Verify Grafana Tempo-to-Loki correlation diff --git a/docs/telemetry-runbook.md b/docs/telemetry-runbook.md index adc65f20fe..e1dc0ef7ad 100644 --- a/docs/telemetry-runbook.md +++ b/docs/telemetry-runbook.md @@ -985,7 +985,7 @@ flowchart TB (`tvc < minVal`) it returns early with no promotion — a built ledger that loses is abandoned ([LedgerMaster.cpp:980](../src/xrpld/app/ledger/detail/LedgerMaster.cpp#L980); [docs/consensus.md:50](consensus.md)). The `ledger.validate` span is emitted only - inside `checkAccept` ([LedgerMaster.cpp:987](../src/xrpld/app/ledger/detail/LedgerMaster.cpp#L987)). + inside `checkAccept` ([LedgerMaster.cpp:1003](../src/xrpld/app/ledger/detail/LedgerMaster.cpp#L1003)). - **validation-send guard**: broadcast only if `validating_ && isCompatible && !consensusFail && canValidateSeq(seq)` — silently suppressed for incompatible ledgers or an already-validated seq @@ -1152,8 +1152,8 @@ are pending a code fix: - **`ledger.acquire` / `ledger.store` / `ledger.validate` are not reliably roots either.** All three use `SpanGuard::span` ([InboundLedger.cpp:113](../src/xrpld/app/ledger/detail/InboundLedger.cpp#L113), - [LedgerMaster.cpp:463](../src/xrpld/app/ledger/detail/LedgerMaster.cpp#L463), - [987](../src/xrpld/app/ledger/detail/LedgerMaster.cpp#L987)), which inherits the + [LedgerMaster.cpp:470](../src/xrpld/app/ledger/detail/LedgerMaster.cpp#L470), + [1003](../src/xrpld/app/ledger/detail/LedgerMaster.cpp#L1003)), which inherits the ambient span ([SpanGuard.cpp:233](../src/libxrpl/telemetry/SpanGuard.cpp#L233)) rather than `freshRoot` ([245](../src/libxrpl/telemetry/SpanGuard.cpp#L245)) — the same defect as From d2cefa05d9a313c588adb3900253e2f4e3feebe1 Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Wed, 9 Sep 2026 13:14:44 +0100 Subject: [PATCH 05/16] fix(telemetry): make the load generators fail loudly instead of exiting 0 Three ways a run could produce no traffic and still report success: - tx_submitter logged a funding shortfall and returned an empty stats object; main() then printed the summary and exited 0, so the failure only surfaced later as "spans missing", which points nowhere. It now records setup_failed in the summary and exits 1 after the report is written. - --weights was checked for valid JSON but not for a positive sum. An all-zero mapping reached random.choices, which raises ValueError from inside the dispatch loop where only CancelledError is caught. Rejected at parse time now, in both generators. - a profile phase declaring neither rpc nor tx logged a warning and returned no error. Both error rates short-circuit to 0.0 when nothing was sent, so a mistyped key produced zero traffic and still passed the exit gate. That phase is now an error. --- .../telemetry/workload/rpc_load_generator.py | 6 ++++++ docker/telemetry/workload/tx_submitter.py | 19 +++++++++++++++++++ .../workload/workload_orchestrator.py | 11 +++++++++-- 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/docker/telemetry/workload/rpc_load_generator.py b/docker/telemetry/workload/rpc_load_generator.py index 36834de9ef..cdd9156e0b 100644 --- a/docker/telemetry/workload/rpc_load_generator.py +++ b/docker/telemetry/workload/rpc_load_generator.py @@ -770,6 +770,12 @@ def main() -> None: try: custom = json.loads(args.weights) weights = {k: int(v) for k, v in custom.items()} + if not weights or sum(weights.values()) <= 0: + logger.error( + "Invalid --weights: the values must sum to more than 0, got %s", + weights, + ) + sys.exit(1) logger.info("Using custom weights: %s", weights) except (json.JSONDecodeError, ValueError) as exc: logger.error("Invalid --weights JSON: %s", exc) diff --git a/docker/telemetry/workload/tx_submitter.py b/docker/telemetry/workload/tx_submitter.py index 807c44a702..b05b567bbf 100644 --- a/docker/telemetry/workload/tx_submitter.py +++ b/docker/telemetry/workload/tx_submitter.py @@ -192,6 +192,8 @@ class TxStats: total_errors: Transactions that returned an error engine_result. by_type: Per-transaction-type count of submissions. errors_by_type: Per-transaction-type count of errors. + setup_failed: True if account setup never produced enough funded + accounts, so the timed loop never ran. """ total_submitted: int = 0 @@ -199,6 +201,7 @@ class TxStats: total_errors: int = 0 by_type: dict[str, int] = field(default_factory=dict) errors_by_type: dict[str, int] = field(default_factory=dict) + setup_failed: bool = False def record(self, tx_type: str, success: bool) -> None: """Record the result of a transaction submission.""" @@ -223,6 +226,7 @@ class TxStats: ), "by_type": self.by_type, "errors_by_type": self.errors_by_type, + "setup_failed": self.setup_failed, } @@ -978,6 +982,10 @@ async def run_submitter( len(accounts), len(created), ) + # The caller turns this into a non-zero exit. Without it a funding + # failure looks like a clean run of zero transactions, and the run + # only fails later as "spans missing", which points nowhere. + stats.setup_failed = True return stats logger.info( @@ -1078,6 +1086,12 @@ def main() -> None: try: custom = json.loads(args.weights) weights = {k: int(v) for k, v in custom.items()} + if not weights or sum(weights.values()) <= 0: + logger.error( + "Invalid --weights: the values must sum to more than 0, got %s", + weights, + ) + sys.exit(1) logger.info("Using custom weights: %s", weights) except (json.JSONDecodeError, ValueError) as exc: logger.error("Invalid --weights JSON: %s", exc) @@ -1101,6 +1115,11 @@ def main() -> None: json.dump(summary, f, indent=2) logger.info("Summary written to %s", args.output) + # After the report is written, so the failure is still diagnosable. + if stats.setup_failed: + logger.error("Account setup failed; no transactions were submitted.") + sys.exit(1) + if __name__ == "__main__": main() diff --git a/docker/telemetry/workload/workload_orchestrator.py b/docker/telemetry/workload/workload_orchestrator.py index 35ca4ded55..3df9d80ea4 100755 --- a/docker/telemetry/workload/workload_orchestrator.py +++ b/docker/telemetry/workload/workload_orchestrator.py @@ -436,9 +436,16 @@ async def run_phase( tasks = _launch_phase_tasks(phase, endpoints, report_dir, prefix) if not tasks: - logger.warning( - "Phase %d: %s — no workload configured, skipping", phase_idx + 1, name + # An error, not a warning. The exit gate is built from phase errors and + # from error RATES, and both rates short-circuit to 0.0 when nothing was + # sent -- so a profile with a mistyped key ("rpcs", "RPC") would produce + # no traffic at all and still exit 0. + message = ( + f"phase {phase_idx + 1} '{name}' configures no workload: " + "it declares neither 'rpc' nor 'tx'" ) + logger.error("%s", message) + result.errors.append(message) return result for label, report_path, task in tasks: From c8d9d881136e9eb96ba2af17465785f21a88a157 Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Wed, 9 Sep 2026 13:15:23 +0100 Subject: [PATCH 06/16] fix(telemetry): measure telemetry overhead under load, on this cluster only The overhead benchmark generated no workload. Each arm was start_cluster -> collect_metrics -> stop_cluster, and collect_metrics only ran the sampler, so the only client traffic was the sampler's own server_info probes at under 1 request/sec. The hottest instrumented paths -- tx.*, txq.*, the transactor stage spans, every rpc.command.* other than server_info -- were never entered, which is where per-operation span cost appears. Both arms now drive rpc_load_generator and tx_submitter at one fixed rate for the whole window, over a [port_ws] listener present in both arms so the listener is not part of the delta. A flat rate rather than a workload profile, because both arms must issue the same work and a profile's phase shaping only adds variance. The sampler also selected xrpld host-wide. run-full-validation.sh leaves its five validation nodes running while the benchmark's three start, so both arms averaged eight processes -- diluting the CPU delta and making memory_rss_mb_peak report a validation node either way. It now takes an optional pid list, and the benchmark passes its own nodes' pids and refuses to measure if it cannot collect them all. consensus_round_mean_ms counted distinct ledger sequences seen by a loop that sampled every 5 s, so it read back 5000 ms for every close time from 2 s to 5 s and a 10% regression measured 0%. Sampling at 2 s -- the close-time floor from ConsensusParms.h:93 -- resolves a 10% regression as at least 9.3%. It also divided by the requested DURATION rather than the measured ELAPSED, which the TPS calculation in the same file already used. Key generation, the workdir setup and the seed read exited 1 under errexit, the code this script reserves for a measured threshold breach, so an infrastructure failure was reported as "telemetry is too expensive". They map to cannot_measure now. No guard is added after the config heredoc: a guard there is read as the heredoc's first line, lands in the generated config and never runs. curl probes across the harness had no --max-time, so a server that accepts the connection and then stops answering blocks forever and the loops' attempt counts stop bounding anything. --- docker/telemetry/workload/benchmark.sh | 184 ++++++++++++++++-- .../workload/collect_system_metrics.sh | 78 ++++++-- .../workload/generate-validator-keys.sh | 24 ++- .../telemetry/workload/run-full-validation.sh | 16 +- 4 files changed, 261 insertions(+), 41 deletions(-) diff --git a/docker/telemetry/workload/benchmark.sh b/docker/telemetry/workload/benchmark.sh index 4c0ea99167..da22b350b1 100755 --- a/docker/telemetry/workload/benchmark.sh +++ b/docker/telemetry/workload/benchmark.sh @@ -1,11 +1,18 @@ #!/usr/bin/env bash # benchmark.sh — Performance benchmark for rippled telemetry overhead. # -# Runs two identical workloads against a rippled cluster: +# Runs the same client workload twice against a rippled cluster: # 1. Baseline: telemetry disabled ([telemetry] enabled=0) -# 2. Telemetry: full telemetry enabled (traces + StatsD + all categories) +# 2. Telemetry: full telemetry enabled (traces + native OTel metrics) # -# Compares CPU, memory, RPC latency, TPS, and consensus round time. +# Both arms drive rpc_load_generator.py and tx_submitter.py at one fixed rate for +# the whole sample window, so the delta is attributable to telemetry rather than +# to a difference in offered load. The workload is not optional: with only the +# sampler's own ~1 request/sec of server_info, the tx.*, txq.*, transactor-stage +# and non-server_info rpc.command.* spans are never entered, and those are where +# a per-operation span cost shows up. A pass on an idle cluster says nothing. +# +# Compares CPU, memory, RPC latency, TPS, and mean consensus round time. # Outputs a Markdown table with pass/fail against configured thresholds. # # Usage: @@ -76,6 +83,30 @@ WORKDIR="/tmp/xrpld-benchmark" RESULTS_DIR="$SCRIPT_DIR/benchmark-results" RPC_PORT_BASE=5020 PEER_PORT_BASE=51250 +# Above run-full-validation.sh's 6006.. so both harnesses can share a box. +WS_PORT_BASE=6020 + +# Head start the generators get before the sampler opens its window. +# tx_submitter.py creates and funds eight accounts from genesis and then waits +# for those payments to validate, so without a lead the first seconds of every +# window carry no transaction load. Identical in both arms, so it cancels out. +WORKLOAD_LEAD_SEC=20 + +# One flat offered rate, not a workload-profiles.json profile: both arms must +# issue the same work for the delta to mean anything, and a profile's phase +# shaping only adds variance. Payment-only for the same reason -- a rejected +# transaction costs a different amount of work than an applied one. +WORKLOAD_RPC_RATE="${BENCH_RPC_RATE:-30}" +WORKLOAD_TX_TPS="${BENCH_TX_TPS:-3}" + +# This arm's generator pids, reaped by wait_workload and killed by the trap. +WORKLOAD_PIDS=() + +# Hard ceiling on every RPC probe below. curl applies no overall timeout of its +# own, so a node that accepts the connection and then stops answering parks the +# poll loop for the rest of the run. The loops here count attempts, not seconds, +# so without this their stated timeouts are not bounds at all. +CURL_MAX_TIME="${CURL_MAX_TIME:-5}" # --------------------------------------------------------------------------- # Argument parsing @@ -121,8 +152,12 @@ done command -v jq >/dev/null 2>&1 || cannot_measure "jq not found" command -v bc >/dev/null 2>&1 || cannot_measure "bc not found" command -v curl >/dev/null 2>&1 || cannot_measure "curl not found" +command -v python3 >/dev/null 2>&1 || + cannot_measure "python3 not found (the load generators need it)" +python3 -c 'import websockets' 2>/dev/null || + cannot_measure "python3 'websockets' package not found -- pip install -r $SCRIPT_DIR/requirements.txt" -mkdir -p "$RESULTS_DIR" +mkdir -p "$RESULTS_DIR" || cannot_measure "Could not create the results directory $RESULTS_DIR" TIMESTAMP=$(date +%Y%m%d_%H%M%S) # --------------------------------------------------------------------------- @@ -139,11 +174,14 @@ start_cluster() { log "Starting $NUM_NODES-node cluster ($label, telemetry=$telemetry_enabled)..." - rm -rf "$WORKDIR" - mkdir -p "$WORKDIR" + rm -rf "$WORKDIR" || cannot_measure "Could not clear the workdir $WORKDIR" + mkdir -p "$WORKDIR" || cannot_measure "Could not create the workdir $WORKDIR" - # Generate keys using first node. - bash "$SCRIPT_DIR/generate-validator-keys.sh" "$XRPLD" "$NUM_NODES" "$WORKDIR" + # Generate keys using a temporary standalone node. The helper fails through + # its own die(), which exits 1 -- the code this script reserves for a + # measured breach. Remap it, or a keygen failure reads as "too expensive". + bash "$SCRIPT_DIR/generate-validator-keys.sh" "$XRPLD" "$NUM_NODES" "$WORKDIR" || + cannot_measure "generate-validator-keys.sh failed; no keys for the $NUM_NODES-node cluster" # Set before the spawn loop so a failure part-way through it still gets # cleaned up by the EXIT trap. @@ -152,14 +190,27 @@ start_cluster() { # Build per-node configs. for i in $(seq 1 "$NUM_NODES"); do local node_dir="$WORKDIR/node$i" - mkdir -p "$node_dir/nudb" "$node_dir/db" + mkdir -p "$node_dir/nudb" "$node_dir/db" || + cannot_measure "Could not create node$i directories under $node_dir" local rpc_port rpc_port=$((RPC_PORT_BASE + i - 1)) local peer_port peer_port=$((PEER_PORT_BASE + i - 1)) + local ws_port + ws_port=$((WS_PORT_BASE + i - 1)) + # Split from the declaration on purpose: `local seed=$(...)` reports + # local's status, not jq's, so the guard below would never fire. local seed - seed=$(jq -r ".[$((i - 1))].seed" "$WORKDIR/validator-keys.json") + seed=$(jq -r ".[$((i - 1))].seed" "$WORKDIR/validator-keys.json") || + cannot_measure "Could not read node$i's seed from $WORKDIR/validator-keys.json" + # jq prints "null" and exits 0 when the array is short, so the exit + # status alone does not catch a truncated key file. + case "$seed" in + "" | null) + cannot_measure "node$i has no seed in $WORKDIR/validator-keys.json" + ;; + esac # Build ips_fixed list. local ips_fixed="" @@ -200,9 +251,13 @@ endpoint=http://localhost:4318/v1/metrics" enabled=0" fi + # No `|| cannot_measure` here: a guard after `<"$node_dir/xrpld.cfg" </dev/null | jq -r '.result.info.server_state' 2>/dev/null || echo "") if [ "$state" = "proposing" ]; then @@ -324,7 +388,7 @@ stop_cluster() { # after argument parsing so the handler name always resolves. Without it, any # failure between start_cluster and stop_cluster leaks the xrpld children # along with their RPC ports (5020+) and peer ports (51250+). -trap stop_cluster EXIT +trap 'stop_workload; stop_cluster' EXIT # Build RPC ports CSV string. rpc_ports_csv() { @@ -342,13 +406,101 @@ rpc_ports_csv() { # source came back empty (3). An all-zero or partial sample set clears every # threshold, so an incomplete leg aborts with "cannot measure" instead of being # compared and passed. +# Echoes one ws:// endpoint per node, space separated. +ws_endpoints() { + local i out="" + for i in $(seq 1 "$NUM_NODES"); do + out="$out ws://localhost:$((WS_PORT_BASE + i - 1))" + done + printf '%s' "${out# }" +} + +# This cluster's xrpld pids, comma separated, for the sampler's process filter. +# Without it the sampler matches every xrpld on the host: run-full-validation.sh +# leaves five validation nodes running while these three start, so both arms +# average eight processes and the delta is diluted away. +node_pids_csv() { + local i out="" pid + for i in $(seq 1 "$NUM_NODES"); do + pid=$(cat "$WORKDIR/node$i/xrpld.pid" 2>/dev/null) || continue + [ -n "$pid" ] && out="$out,$pid" + done + printf '%s' "${out#,}" +} + +# Starts this arm's generators, then waits out the funding lead so the sampler's +# whole window is under load. Logs and JSON summaries go to RESULTS_DIR, not +# WORKDIR: the next arm's start_cluster rm -rf's WORKDIR. +start_workload() { + local label="$1" + local gen_duration=$((DURATION + WORKLOAD_LEAD_SEC)) + local logdir="$RESULTS_DIR/workload-${TIMESTAMP}" + mkdir -p "$logdir" || cannot_measure "Could not create the workload log dir $logdir" + + log "Starting workload ($label): ${WORKLOAD_RPC_RATE} rpc/s + ${WORKLOAD_TX_TPS} tps for ${gen_duration}s..." + + # shellcheck disable=SC2046 # --endpoints takes a list; splitting is intended + python3 "$SCRIPT_DIR/rpc_load_generator.py" \ + --endpoints $(ws_endpoints) \ + --rate "$WORKLOAD_RPC_RATE" \ + --duration "$gen_duration" \ + --output "$logdir/$label-rpc.json" \ + >"$logdir/$label-rpc.log" 2>&1 & + WORKLOAD_PIDS+=("$!") + + python3 "$SCRIPT_DIR/tx_submitter.py" \ + --endpoint "ws://localhost:$WS_PORT_BASE" \ + --tps "$WORKLOAD_TX_TPS" \ + --duration "$gen_duration" \ + --weights '{"Payment": 100}' \ + --output "$logdir/$label-tx.json" \ + >"$logdir/$label-tx.log" 2>&1 & + WORKLOAD_PIDS+=("$!") + + sleep "$WORKLOAD_LEAD_SEC" +} + +# Reaps this arm's generators. A non-zero generator means the arms did not do +# the same work, so nothing is attributable: "cannot measure", never "too slow". +wait_workload() { + local label="$1" + local pid status=0 + for pid in ${WORKLOAD_PIDS[@]+"${WORKLOAD_PIDS[@]}"}; do + wait "$pid" || status=$? + done + WORKLOAD_PIDS=() + [ "$status" -eq 0 ] || + cannot_measure "$label workload generator exited $status; the arms did not do identical work -- see $RESULTS_DIR/workload-${TIMESTAMP}/" +} + +# Kills any generator still running, so an aborted arm leaves no python3 holding +# WebSocket connections. Guarded like stop_cluster's commands: this runs from the +# EXIT trap, where an unguarded failure would discard the real exit status. +stop_workload() { + local pid + for pid in ${WORKLOAD_PIDS[@]+"${WORKLOAD_PIDS[@]}"}; do + kill "$pid" 2>/dev/null || true + done + WORKLOAD_PIDS=() + return 0 +} + collect_metrics() { local label="$1" local out_file="$2" local status=0 + # An empty or short list would silently fall back to host-wide sampling, + # which is the outcome the pid argument exists to prevent. + local pids + pids=$(node_pids_csv) + local n_pids + n_pids=$(printf '%s' "$pids" | awk -F, '{print NF}') + [ "${n_pids:-0}" -eq "$NUM_NODES" ] || + cannot_measure "$label: found $n_pids of $NUM_NODES node pids, so the sampler cannot be scoped to this cluster" + bash "$SCRIPT_DIR/collect_system_metrics.sh" \ - "$(rpc_ports_csv)" "$DURATION" "$out_file" || status=$? + "$(rpc_ports_csv)" "$DURATION" "$out_file" "$pids" || status=$? [ "$status" -eq 0 ] || cannot_measure "$label metric collection failed (exit $status) — refusing to compare an incomplete run" @@ -371,13 +523,17 @@ log "=" # --- Baseline run --- BASELINE_FILE="$RESULTS_DIR/baseline-${TIMESTAMP}.json" start_cluster "0" "baseline" +start_workload "baseline" collect_metrics "baseline" "$BASELINE_FILE" +wait_workload "baseline" stop_cluster # --- Telemetry run --- TELEMETRY_FILE="$RESULTS_DIR/telemetry-${TIMESTAMP}.json" start_cluster "1" "telemetry" +start_workload "telemetry" collect_metrics "telemetry" "$TELEMETRY_FILE" +wait_workload "telemetry" stop_cluster # --------------------------------------------------------------------------- @@ -522,7 +678,7 @@ cat >"$REPORT_FILE" < +# ./collect_system_metrics.sh [pids_csv] +# +# pids_csv narrows process sampling to exactly those pids. Without it the scope +# is every xrpld on the host, which averages in any other cluster's nodes and +# reports the largest of them as the RSS peak. benchmark.sh passes its own pids +# because run-full-validation.sh leaves five validation nodes running while the +# benchmark's three start, and diluting the arms alike hides the delta. # # Example: # ./collect_system_metrics.sh "5005,5006,5007" 300 /tmp/metrics-baseline.json +# ./collect_system_metrics.sh "5020,5021,5022" 120 /tmp/m.json "8801,8802,8803" # # Output JSON format: # { @@ -54,6 +61,7 @@ usage() { echo " rpc_ports_csv Comma-separated RPC ports (e.g., 5005,5006,5007)" echo " duration_seconds How long to collect metrics" echo " output_file Path to write JSON results" + echo " pids_csv Optional: sample only these pids, not every host xrpld" exit 1 } @@ -64,16 +72,33 @@ fi RPC_PORTS_CSV="$1" DURATION="$2" OUTPUT_FILE="$3" +PIDS_CSV="${4:-}" + +# Reject a malformed pid list rather than silently sampling the whole host, +# which is the outcome this argument exists to prevent. +case "$PIDS_CSV" in + '') ;; + *[!0-9,]*) die "pids_csv must be a comma-separated list of pids, got '$PIDS_CSV'" ;; +esac IFS=',' read -ra RPC_PORTS <<<"$RPC_PORTS_CSV" -SAMPLE_INTERVAL=5 + +# consensus_round_mean_ms below counts DISTINCT ledger sequences seen by this +# loop, so it cannot resolve a close interval shorter than SAMPLE_INTERVAL. At +# 5 s it read back 5000 ms for every close time from 2 s to 5 s, so a 10% +# regression measured 0% and the benchmark's 1% consensus gate could never fire. +# 2 s is the close-time floor itself (ledgerMinClose, ConsensusParms.h:93), which +# is enough: measured against this file's own arithmetic, a 10% regression shows +# up as at least 9.3% anywhere in the 2 s to 5 s band. 1 s only doubles the +# probe load for slightly worse numbers. +SAMPLE_INTERVAL=2 # Hard ceiling on every RPC probe below. curl applies no overall timeout of its # own, so a node that accepts the connection and then stops answering — what a # stalled job queue looks like from outside — parks the sampling loop for the -# rest of the run. 5 s is one sample interval and some thousands of times a -# healthy server_info, so it bounds a wedged node's cost to one lost sample -# while never truncating a real reply. A probe that hits the ceiling exits +# rest of the run. 5 s is some thousands of times a healthy server_info, so it +# bounds a wedged node's cost to a couple of lost samples while never +# truncating a real reply. A probe that hits the ceiling exits # non-zero and is therefore skipped rather than recorded, which is the same # rule the latency loop already applies to a refused connection; if every # probe hits it, the empty file trips the placeholder warning below. @@ -175,21 +200,31 @@ for sample in $(seq 1 "$SAMPLES"); do # and -C matches nothing. "rippled" is accepted alongside "xrpld" so a # rename of the binary cannot silently zero the collector. # - # Scope is the whole host, as it always was: a second xrpld from another - # checkout is sampled too. Only run a benchmark on a box with one cluster. + # Scope is the pid list when one was given, and the whole host otherwise. + # Host scope averages in any other cluster's xrpld and reports the largest + # of them as the RSS peak, which reads as noise against a 5 MB threshold. # # A %cpu of exactly 0.0 is a real reading and is counted — dropping idle # samples would inflate the average — while non-numeric output is # rejected by the pattern. An RSS of 0 is not a live process, so it # contributes no memory sample; counting it would leave the file non-empty # and mark a dead cluster's 0 MB peak as a complete measurement. - ps -eo %cpu=,rss=,args= | - awk -v cpu_file="$CPU_FILE" -v mem_file="$MEM_FILE" ' - $3 !~ /(^|\/)(xrpld|rippled)$/ { next } - $1 ~ /^[0-9]+(\.[0-9]+)?$/ { cpu_sum += $1; cpu_n++ } - $2 ~ /^[0-9]+$/ && $2 + 0 > 0 { printf("%.2f\n", $2 / 1024) >> mem_file } - END { if (cpu_n > 0) printf("%.2f\n", cpu_sum / cpu_n) >> cpu_file } - ' || die "process sampling failed on sample $sample/$SAMPLES" + ps -eo pid=,%cpu=,rss=,args= | + awk -v cpu_file="$CPU_FILE" -v mem_file="$MEM_FILE" -v pids="$PIDS_CSV" ' + BEGIN { if (pids != "") { n = split(pids, a, ","); for (i = 1; i <= n; i++) want[a[i]] = 1 } } + pids != "" && !($1 in want) { next } + pids == "" && $4 !~ /(^|\/)(xrpld|rippled)$/ { next } + $2 ~ /^[0-9]+(\.[0-9]+)?$/ { cpu_sum += $2; cpu_n++ } + $3 ~ /^[0-9]+$/ && $3 + 0 > 0 { printf("%.2f\n", $3 / 1024) >> mem_file } + END { + # With an explicit pid list the expected count is known, so a + # dead node is detectable. Without it, exit 1 would fire on any + # host with no xrpld at all, which the empty-file check below + # already reports. + if (pids != "" && cpu_n != n) exit 1 + if (cpu_n > 0) printf("%.2f\n", cpu_sum / cpu_n) >> cpu_file + } + ' || die "process sampling failed on sample $sample/$SAMPLES: expected $(echo "$PIDS_CSV" | tr -cd , | wc -c)+1 live pids" # Collect RPC latency from each node. Only a successful call is a latency # measurement: a refused connection returns in well under a millisecond, @@ -310,12 +345,15 @@ else METRICS_COMPLETE=false fi -# Mean inter-ledger interval in ms: DURATION / (distinct ledgers - 1) * 1000. +# Mean inter-ledger interval in ms: ELAPSED / (distinct ledgers - 1) * 1000. # -# This is a MEAN, not a percentile — the JSON key says so. It is also aliased -# by the sample loop: LEDGER_FILE gets one sequence per sample, so at a -# SAMPLE_INTERVAL of 5 s the series cannot resolve a close interval faster -# than that (a ~4 s close is invisible). Read it as a coarse trend only. +# ELAPSED, not DURATION: DURATION is what the caller asked for, while the loop +# also spends time on its own probes, so it always runs longer. TPS above uses +# ELAPSED for the same reason. +# +# This is a MEAN, not a percentile — the JSON key says so. It still cannot +# resolve a close interval at or below SAMPLE_INTERVAL, because LEDGER_FILE +# gets at most one sequence per sample. Read it as a coarse trend. if [ -s "$LEDGER_FILE" ]; then UNIQUE_LEDGERS=$(sort -u "$LEDGER_FILE" | wc -l) # The > 1 test also keeps the divisor below at 1 or more. @@ -328,7 +366,7 @@ if [ -s "$LEDGER_FILE" ]; then # sampling loop and the CPU average, so computing this with awk removes # the failure path rather than reporting it. The divisor is >= 1 by the # test above. - CONSENSUS_MEAN=$(awk -v d="$DURATION" -v u="$UNIQUE_LEDGERS" \ + CONSENSUS_MEAN=$(awk -v d="$ELAPSED" -v u="$UNIQUE_LEDGERS" \ 'BEGIN { printf "%.0f", d * 1000 / (u - 1) }') else warn "Ledger seq never advanced ($UNIQUE_LEDGERS distinct); consensus_round_mean_ms is a 0 placeholder" diff --git a/docker/telemetry/workload/generate-validator-keys.sh b/docker/telemetry/workload/generate-validator-keys.sh index 48ebda8495..9c705197aa 100755 --- a/docker/telemetry/workload/generate-validator-keys.sh +++ b/docker/telemetry/workload/generate-validator-keys.sh @@ -4,6 +4,20 @@ # Uses a temporary standalone xrpld instance to call `validation_create` RPC # for each node. Outputs a JSON file mapping node index to seed + public key. # +# Production does this differently, and deliberately not the way this script +# does. There, `validator-keys-tool` runs `create_keys` once to make a master +# key that never leaves the operator's custody, then `create_token` to mint a +# revocable `[validator_token]` for the node; rotating a node means minting a +# new token, not moving the master key. This harness uses `validation_create` +# instead, which puts the seed itself in `[validation_seed]` on the node. +# +# That is safe here only because the cluster is disposable: the keys are made on +# the same machine that runs the nodes, into a temp workdir that the next run +# deletes, so there is no custody boundary for the two-step split to protect. +# The tool is also not present on this path -- it is built only with +# `-Dvalidator_keys=ON`, which the telemetry CI job does not pass. Do not copy +# this script's approach to a node holding a key you care about. +# # Usage: # ./generate-validator-keys.sh # @@ -58,6 +72,12 @@ TEMP_DIR="$(mktemp -d)" TEMP_PORT=5099 TEMP_CFG="$TEMP_DIR/xrpld.cfg" +# Hard ceiling on every RPC probe below. curl applies no overall timeout of its +# own, so a node that accepts the connection and then stops answering parks the +# poll loop for the rest of the run. The loops here count attempts, not seconds, +# so without this their stated timeouts are not bounds at all. +CURL_MAX_TIME="${CURL_MAX_TIME:-5}" + log "Starting temporary xrpld for key generation (port $TEMP_PORT)..." cat >"$TEMP_CFG" </dev/null 2>&1; then log "Temporary xrpld RPC ready (attempt $attempt)." break @@ -118,7 +138,7 @@ KEYS_JSON="[" VALIDATORS_TXT="[validators]" for i in $(seq 1 "$NUM_NODES"); do - result=$(curl -sf "http://localhost:$TEMP_PORT" \ + result=$(curl -sf --max-time "$CURL_MAX_TIME" "http://localhost:$TEMP_PORT" \ -d '{"method":"validation_create"}') seed=$(echo "$result" | jq -r '.result.validation_seed') pubkey=$(echo "$result" | jq -r '.result.validation_public_key') diff --git a/docker/telemetry/workload/run-full-validation.sh b/docker/telemetry/workload/run-full-validation.sh index f3c399440c..44222deee2 100755 --- a/docker/telemetry/workload/run-full-validation.sh +++ b/docker/telemetry/workload/run-full-validation.sh @@ -80,6 +80,12 @@ NUM_NODES=5 RPC_PORT_BASE=5005 WS_PORT_BASE=6006 PEER_PORT_BASE=51235 + +# Hard ceiling on every RPC probe below. curl applies no overall timeout of its +# own, so a node that accepts the connection and then stops answering parks the +# poll loop for the rest of the run. The loops here count attempts, not seconds, +# so without this their stated timeouts are not bounds at all. +CURL_MAX_TIME="${CURL_MAX_TIME:-5}" # Inert: parsed from --rpc-rate/--rpc-duration/--tx-tps/--tx-duration and never # read again. Load shape comes from the workload profile instead. Kept because # the CI workflow still passes the four flags. @@ -267,7 +273,7 @@ XRPLD_LOG_DIR="$WORKDIR" docker compose -f "$COMPOSE_FILE" up -d || log "Waiting for OTel Collector..." for attempt in $(seq 1 30); do - status=$(curl -so /dev/null -w '%{http_code}' http://localhost:4318/ 2>/dev/null || echo 000) + status=$(curl -so /dev/null -w '%{http_code}' --max-time "$CURL_MAX_TIME" http://localhost:4318/ 2>/dev/null || echo 000) if [ "$status" != "000" ]; then ok "OTel Collector ready (attempt $attempt)" break @@ -278,7 +284,7 @@ done log "Waiting for Tempo..." for attempt in $(seq 1 30); do - if curl -sf "http://localhost:3200/ready" >/dev/null 2>&1; then + if curl -sf --max-time "$CURL_MAX_TIME" "http://localhost:3200/ready" >/dev/null 2>&1; then ok "Tempo ready (attempt $attempt)" break fi @@ -288,7 +294,7 @@ done log "Waiting for Prometheus..." for attempt in $(seq 1 30); do - if curl -sf "http://localhost:9090/-/healthy" >/dev/null 2>&1; then + if curl -sf --max-time "$CURL_MAX_TIME" "http://localhost:9090/-/healthy" >/dev/null 2>&1; then ok "Prometheus ready (attempt $attempt)" break fi @@ -494,7 +500,7 @@ for attempt in $(seq 1 120); do laggards="" for i in $(seq 1 "$NUM_NODES"); do port=$((RPC_PORT_BASE + i - 1)) - state=$(curl -sf "http://localhost:$port" \ + state=$(curl -sf --max-time "$CURL_MAX_TIME" "http://localhost:$port" \ -d '{"method":"server_info"}' 2>/dev/null | jq -r '.result.info.server_state' 2>/dev/null || echo "") if [ "$state" = "proposing" ]; then @@ -552,7 +558,7 @@ echo "" # Wait for first validated ledger. log "Waiting for validated ledger..." for attempt in $(seq 1 60); do - val_seq=$(curl -sf "http://localhost:$RPC_PORT_BASE" \ + val_seq=$(curl -sf --max-time "$CURL_MAX_TIME" "http://localhost:$RPC_PORT_BASE" \ -d '{"method":"server_info"}' 2>/dev/null | jq -r '.result.info.validated_ledger.seq // 0' 2>/dev/null || echo 0) if [ "$val_seq" -gt 2 ] 2>/dev/null; then From 7f829a5929d7d0b052a1e7300bef1f2ce02bbd22 Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Wed, 9 Sep 2026 13:15:46 +0100 Subject: [PATCH 07/16] fix(telemetry): fail the regression gate on a unit change, and report what it gated compare_to_baseline took the unit from the baseline entry and dropped the current run's, and nothing compared the two, so a us -> ms change was scored as a numeric delta: four keys rewritten to the same physical durations reported 99.9% improvements and the gate exited 0. prom_queries.py says the baseline preserves the unit "so the comparator can sanity-check unit drift"; it never did. A unit mismatch now fails and names both units. The workflow's step summary printed total, regressions and improvements. total is every key in the report -- the union of baseline and current -- so it was neither the baseline count nor what was gated, and missing_in_current was computed and never printed. A run that gated 16 of 20 keys read as a full comparison. The comparator now reports a real "compared" count and the summary prints it beside the not-captured count, with a warning when any key was missed. The table also refused nothing on a truncated report; existence is not readability. check_regression_bounds told the operator to add max_abs_increase while reading max_abs_increase_ms / _us, so following the message added a key nothing reads and the gate kept failing with no explanation. The committed thresholds use only the suffixed spelling, so the message was the defect. Its three JSON inputs were also unchecked: a top-level null, list or number parsed and then died on the first .get, and a string "metrics" survived the placeholder test and reported its own characters as gated keys -- wrong advice rather than a crash. Four tests cover these; all four fail against the previous checker. --- .../telemetry/check_regression_bounds.py | 50 +++++++++++-- .../telemetry/test_check_regression_bounds.py | 73 +++++++++++++++++++ .github/workflows/telemetry-validation.yml | 61 ++++++++++++---- .../telemetry/workload/compare_to_baseline.py | 34 +++++++++ 4 files changed, 197 insertions(+), 21 deletions(-) diff --git a/.github/scripts/telemetry/check_regression_bounds.py b/.github/scripts/telemetry/check_regression_bounds.py index b07045ce70..4e89ac421c 100644 --- a/.github/scripts/telemetry/check_regression_bounds.py +++ b/.github/scripts/telemetry/check_regression_bounds.py @@ -87,6 +87,11 @@ UNIT_TO_MS = {"ms": 1.0, "s": 1000.0} REL_TOLERANCE = 1e-12 +def is_number(value): + """True for a real JSON number. bool is an int subclass, so exclude it.""" + return not isinstance(value, bool) and isinstance(value, (int, float)) + + def read_text_or_exit(path): """Read a required text input, or exit 1 naming the input that failed.""" try: @@ -96,11 +101,22 @@ def read_text_or_exit(path): def read_json_or_exit(path): - """Read and parse a required JSON input, or exit 1 naming what failed.""" + """Read and parse a required JSON object, or exit 1 naming what failed. + + All three JSON inputs are objects. A top-level null, list or number parses + fine and then dies on the first .get, so check the shape here rather than + report it as a traceback pointing into this script. + """ try: - return json.loads(read_text_or_exit(path)) + parsed = json.loads(read_text_or_exit(path)) except json.JSONDecodeError as exc: sys.exit(f"{path}: required input is not valid JSON -- {exc}") + if not isinstance(parsed, dict): + sys.exit( + f"{path}: required input is valid JSON but its top level is " + f"{type(parsed).__name__}, not an object -- nothing can be read from it" + ) + return parsed def span_edges_ms(): @@ -264,7 +280,7 @@ def _unusable_baseline(key, value, unit): A failure string, or None when the value is usable. """ # bool is a subclass of int; True would otherwise pass as the number 1. - if isinstance(value, bool) or not isinstance(value, (int, float)): + if not is_number(value): return ( f"{key}: baseline value {value!r} is not a number, so no bound can be " f"derived from it. Recapture the baseline from a CI run rather than " @@ -338,12 +354,24 @@ def check_key(key, entry, thresholds, ladders): if rule is None: failures.append( f"{key}: no per-metric override, so it falls back to the defaults and " - f"gates on the percentage bound alone. Add an override with " - f"max_abs_increase = {hi_next - value!r} (rule B)" + f"gates on the percentage bound alone. Add an override in " + f"{THRESHOLDS} with " + f"max_abs_increase_{unit} = {hi_next - value!r} (rule B)" ) return failures + if not isinstance(rule, dict): + return [ + f"{key}: threshold override {rule!r} is not an object carrying " + f"max_abs_increase_{unit} and max_pct_increase -- fix it in {THRESHOLDS}" + ] + bound = rule.get("max_abs_increase_ms", rule.get("max_abs_increase_us")) + if bound is not None and not is_number(bound): + return [ + f"{key}: max_abs_increase_{unit} is {bound!r}, not a number, so it " + f"cannot be compared with the derived bound -- fix it in {THRESHOLDS}" + ] expected = hi_next - value if bound is None or abs(bound - expected) > REL_TOLERANCE * expected: failures.append( @@ -354,6 +382,11 @@ def check_key(key, entry, thresholds, ladders): pct = rule.get("max_pct_increase") if pct is None: failures.append(f"{key}: no max_pct_increase, so the metric never gates") + elif not is_number(pct): + failures.append( + f"{key}: max_pct_increase is {pct!r}, not a number -- fix it in " + f"{THRESHOLDS}" + ) elif bound is not None and pct >= 100.0 * bound / value: failures.append( f"{key}: max_pct_increase {pct:g}% is at or above the absolute bound's " @@ -391,6 +424,13 @@ def main(): ladders = {"ms": span_edges_ms(), "us": microsecond_edges()} gated = baseline["metrics"] + # A list or a string is truthy, so it survives the placeholder test above + # and then either crashes or reports its characters as gated keys. + if not isinstance(gated, dict): + sys.exit( + f"{BASELINE}: 'metrics' is {type(gated).__name__}, not an object of " + f"key -> {{value, unit}} -- recapture the baseline from a CI run" + ) failures = [] declared = declared_keys(metrics_cfg) diff --git a/.github/scripts/telemetry/test_check_regression_bounds.py b/.github/scripts/telemetry/test_check_regression_bounds.py index ac66eb2f1b..44222e6576 100644 --- a/.github/scripts/telemetry/test_check_regression_bounds.py +++ b/.github/scripts/telemetry/test_check_regression_bounds.py @@ -156,6 +156,45 @@ class TestInputHandling(CheckerCase): self.assertEqual(code, 1, out) self.assertIn("valid JSON", out) + def test_non_object_top_level_fails_naming_the_input(self): + """Valid JSON of the wrong shape must be named, not raise a traceback. + + A top-level null, list or number parses, so it reaches the first .get + and dies pointing at a line in the checker rather than at the file the + operator has to fix. + """ + for rel, text in ( + (BASELINE, "null"), + (THRESHOLDS, "[]"), + (METRICS, "5"), + ): + with self.subTest(input=rel): + self.setUp() + (self.tree / rel).write_text(text) + code, out = self.run_checker() + self.assertEqual(code, 1, out) + self.assertIn(rel, out) + self.assertIn("not an object", out) + self.assertNotIn("Traceback", out) + + def test_non_object_metrics_map_fails_naming_the_input(self): + """A string 'metrics' is truthy, so it slips past the placeholder test. + + Left unchecked it reports the string's own characters as gated keys, + which is worse than a crash: the advice is wrong rather than absent. + An empty map still has to pass, because that is the bootstrap state. + """ + self.edit_json(BASELINE, lambda d: d.update(metrics="span.tx.process.p99")) + code, out = self.run_checker() + self.assertEqual(code, 1, out) + self.assertIn("'metrics' is str", out) + self.assertNotIn("Traceback", out) + + self.setUp() + self.edit_json(BASELINE, lambda d: d.update(metrics={})) + code, out = self.run_checker() + self.assertEqual(code, 0, out) + class TestRules(CheckerCase): """One case per rule, so a rule that stops flagging is caught.""" @@ -183,6 +222,40 @@ class TestRules(CheckerCase): self.assertEqual(code, 1, out) self.assertIn("(rule B)", out) + def test_rule_b_names_the_unit_suffixed_key(self): + """The key it tells the operator to add must be the key the code reads. + + The bound is stored as max_abs_increase_ms or _us. A message naming a + bare max_abs_increase sends the operator to add a key nothing reads, so + the gate keeps failing with no explanation. Both suffixes are covered, + because a test on the ms side alone passes on a hard-coded "_ms". + """ + for group, suffix in ( + ("span.ledger.build", "max_abs_increase_ms"), + ("job.transaction.queued", "max_abs_increase_us"), + ): + with self.subTest(group=group): + self.setUp() + self.edit_json(THRESHOLDS, lambda d: d["overrides"].pop(group)) + code, out = self.run_checker() + self.assertEqual(code, 1, out) + self.assertIn("(rule B)", out) + self.assertIn(suffix, out) + self.assertNotIn("max_abs_increase =", out) + + def test_non_numeric_threshold_is_reported_not_crashed(self): + """A hand-edited bound that is a string must be named, not raise.""" + self.edit_json( + THRESHOLDS, + lambda d: d["overrides"]["span.ledger.build"]["p99"].update( + max_abs_increase_ms="5.5" + ), + ) + code, out = self.run_checker() + self.assertEqual(code, 1, out) + self.assertIn("not a number", out) + self.assertNotIn("Traceback", out) + def test_rule_c_flags_rounded_bound(self): """A bound rounded for readability is still not the derived bound.""" _, exact = self.gated("span.tx.process.p99") diff --git a/.github/workflows/telemetry-validation.yml b/.github/workflows/telemetry-validation.yml index e5b20de529..e60b5399b8 100644 --- a/.github/workflows/telemetry-validation.yml +++ b/.github/workflows/telemetry-validation.yml @@ -5,7 +5,7 @@ # # This is a separate workflow from the main CI. It runs: # - On manual dispatch (workflow_dispatch) -# - On pushes to telemetry-related branches +# - On any push that touches one of the paths globs below # # The workflow is intentionally heavyweight (builds rippled, starts Docker # services, runs a multi-node cluster) — it validates the full telemetry @@ -420,22 +420,51 @@ jobs: cat "$TIMINGS" >>"$GITHUB_STEP_SUMMARY" echo '```' >>"$GITHUB_STEP_SUMMARY" elif [ -f "$REGRESSION" ]; then - REGR_COUNT=$(jq -e '.summary.regressions' "$REGRESSION") || REGR_COUNT=0 - IMPR_COUNT=$(jq -e '.summary.improvements' "$REGRESSION") || IMPR_COUNT=0 - TOTAL=$(jq -e '.summary.total' "$REGRESSION") || TOTAL=0 - echo "| Stat | Count |" >>"$GITHUB_STEP_SUMMARY" - echo "|------|-------|" >>"$GITHUB_STEP_SUMMARY" - echo "| Metrics compared | $TOTAL |" >>"$GITHUB_STEP_SUMMARY" - echo "| Regressions | $REGR_COUNT |" >>"$GITHUB_STEP_SUMMARY" - echo "| Improvements | $IMPR_COUNT |" >>"$GITHUB_STEP_SUMMARY" - echo "" >>"$GITHUB_STEP_SUMMARY" - if [ "$REGR_COUNT" -gt 0 ]; then - echo "### Regressions" >>"$GITHUB_STEP_SUMMARY" + # Existence is not readability: a truncated report satisfies -f, + # and the `|| =0` fallbacks below would then render a clean table + # of zeros for a run that compared nothing. Same trap the note + # above the baseline parse warns about, so check the shape first. + SUMMARY_OK=$(jq -r 'if (.summary | type) == "object" then "yes" else "no" end' \ + "$REGRESSION" 2>/dev/null) || SUMMARY_OK=no + if [ "$SUMMARY_OK" != "yes" ]; then + echo "## Regression Gate: report unreadable" >>"$GITHUB_STEP_SUMMARY" echo "" >>"$GITHUB_STEP_SUMMARY" - echo "| Metric | Baseline | Current | Δ | % | Unit |" >>"$GITHUB_STEP_SUMMARY" - echo "|--------|---------:|--------:|--:|--:|------|" >>"$GITHUB_STEP_SUMMARY" - jq -r '.metrics[] | select(.regressed) | "| \(.key) | \(.baseline) | \(.current) | \(.delta) | \(.pct_change)% | \(.unit) |"' \ - "$REGRESSION" >>"$GITHUB_STEP_SUMMARY" + echo "\`$REGRESSION\` exists but carries no \`summary\` object, so no" \ + "table can be rendered. The pass/fail above still comes from the" \ + "comparator's exit code." >>"$GITHUB_STEP_SUMMARY" + echo "::error::Regression report is present but has no summary object" + else + # No `jq -e`: it exits non-zero when a field is legitimately 0 or + # false, which the fallbacks would silently turn into 0 as well. + REGR_COUNT=$(jq -r '.summary.regressions // 0' "$REGRESSION") + IMPR_COUNT=$(jq -r '.summary.improvements // 0' "$REGRESSION") + TOTAL=$(jq -r '.summary.total // 0' "$REGRESSION") + MISSING_COUNT=$(jq -r '.summary.missing_in_current // 0' "$REGRESSION") + COMPARED=$(jq -r '.summary.compared // 0' "$REGRESSION") + # `total` is every key in the report, i.e. the union of the + # baseline and this run, so it is not what was gated. The + # comparator reports `compared` for that; do not derive it from + # total minus missing, because a key can also be skipped for + # being new or for having no data on either side. + echo "| Stat | Count |" >>"$GITHUB_STEP_SUMMARY" + echo "|------|-------|" >>"$GITHUB_STEP_SUMMARY" + echo "| Metrics in report | $TOTAL |" >>"$GITHUB_STEP_SUMMARY" + echo "| Metrics compared | $COMPARED |" >>"$GITHUB_STEP_SUMMARY" + echo "| Not captured this run | $MISSING_COUNT |" >>"$GITHUB_STEP_SUMMARY" + echo "| Regressions | $REGR_COUNT |" >>"$GITHUB_STEP_SUMMARY" + echo "| Improvements | $IMPR_COUNT |" >>"$GITHUB_STEP_SUMMARY" + echo "" >>"$GITHUB_STEP_SUMMARY" + if [ "$MISSING_COUNT" -gt 0 ]; then + echo "::warning::$MISSING_COUNT baseline metric(s) were not captured this run, so they were not gated" + fi + if [ "$REGR_COUNT" -gt 0 ]; then + echo "### Regressions" >>"$GITHUB_STEP_SUMMARY" + echo "" >>"$GITHUB_STEP_SUMMARY" + echo "| Metric | Baseline | Current | Δ | % | Unit |" >>"$GITHUB_STEP_SUMMARY" + echo "|--------|---------:|--------:|--:|--:|------|" >>"$GITHUB_STEP_SUMMARY" + jq -r '.metrics[] | select(.regressed) | "| \(.key) | \(.baseline) | \(.current) | \(.delta) | \(.pct_change)% | \(.unit) |"' \ + "$REGRESSION" >>"$GITHUB_STEP_SUMMARY" + fi fi fi diff --git a/docker/telemetry/workload/compare_to_baseline.py b/docker/telemetry/workload/compare_to_baseline.py index 5619400858..4ae452a460 100644 --- a/docker/telemetry/workload/compare_to_baseline.py +++ b/docker/telemetry/workload/compare_to_baseline.py @@ -282,6 +282,29 @@ def compute_delta( current = current_entry.get("value") if current_entry else None unit = (baseline_entry or current_entry or {}).get("unit", "") + # A unit change makes the two numbers incomparable, so subtracting them is + # meaningless: us -> ms reads as a 99.9% improvement and the gate passes. + # Fail instead, and name both units so the baseline can be refreshed. + baseline_unit = (baseline_entry or {}).get("unit", "") + current_unit = (current_entry or {}).get("unit", "") + if baseline_unit and current_unit and baseline_unit != current_unit: + pct_threshold, abs_threshold = resolve_thresholds(key, thresholds) + return MetricDelta( + key=key, + baseline=baseline, + current=current, + delta=None, + pct_change=None, + unit=f"{baseline_unit}->{current_unit}", + threshold_pct=pct_threshold, + threshold_abs=abs_threshold, + regressed=True, + note=( + f"unit changed: baseline is {baseline_unit}, current run is " + f"{current_unit} -- refresh the baseline instead of comparing" + ), + ) + if baseline is None and current is None: return _skip_delta( key, None, None, unit, thresholds, "no data (neither baseline nor current)" @@ -358,6 +381,12 @@ def print_summary(deltas: list[MetricDelta]) -> None: "absolute bound alone where the baseline is not positive):" ) _print_table(regressions) + # A regression can also be recorded with no delta at all -- a unit + # change makes the two numbers incomparable. That row prints as dashes, + # so name the reason here or the table looks like a bug. + for d in regressions: + if d.delta is None: + print(f" {d.key}: {d.note}") if improvements: top = improvements[:5] @@ -401,7 +430,12 @@ def write_report( "window": timings.get("window"), "profile": timings.get("profile"), "summary": { + # total is every key in the report, which is the UNION of the + # baseline and the current run -- not the baseline count. "compared" + # is the only number that says how much was actually gated: a delta + # exists only when both sides had a value. "total": len(deltas), + "compared": sum(1 for d in deltas if d.delta is not None), "regressions": len(regressions), "improvements": sum( 1 From ec308c6b0007e4a501636cf22b7e1e7c04e33d57 Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Wed, 9 Sep 2026 13:15:50 +0100 Subject: [PATCH 08/16] fix(telemetry): poll the parity queries instead of racing one instant query The four external-parity bounds checks each ran a single Prometheus instant query and failed on an empty result. The metric checks that run earlier poll /api/v1/series, which returns a series regardless of staleness, but a bounds check needs the sample value and so cannot use that endpoint. This file's own docstring records the consequence: a beast::insight gauge that stops changing can fall out of an instant query while /api/v1/series still returns it, so one attempt is not enough to call the series absent. Poll to the same deadline the metric checks use. A Prometheus error is raised rather than retried, because a rejected query never becomes valid and retrying it only burns the full timeout. --- .../telemetry/workload/validate_telemetry.py | 56 +++++++++++++++++-- 1 file changed, 50 insertions(+), 6 deletions(-) diff --git a/docker/telemetry/workload/validate_telemetry.py b/docker/telemetry/workload/validate_telemetry.py index 052d3827c1..2b7b79eb23 100644 --- a/docker/telemetry/workload/validate_telemetry.py +++ b/docker/telemetry/workload/validate_telemetry.py @@ -2300,6 +2300,48 @@ def _bounds_description(lo: float, hi: float | None, exclusive_lo: bool) -> str: return desc +async def _poll_instant_query( + session: aiohttp.ClientSession, + prometheus_url: str, + query: str, + deadline: float, +) -> list[dict[str, Any]]: + """Run an instant query, retrying until it returns series or time runs out. + + A bounds check needs the sample value, so it cannot use the /api/v1/series + endpoint the metric checks poll. An instant query answers from the last + scrape, and a gauge that stops changing can fall out of it, so one attempt + is not enough to call the series absent. + + Args: + session: aiohttp client session. + prometheus_url: Prometheus API base URL. + query: PromQL instant query. + deadline: Monotonic deadline. Never slept past. + + Returns: + The result list, empty if nothing appeared before the deadline. + """ + while True: + async with session.get( + f"{prometheus_url}/api/v1/query", params={"query": query} + ) as resp: + data = await resp.json() + # An error is not "not yet": a bad query never becomes good, so + # retrying it only burns the whole deadline. Raise instead, and let + # the caller report it against the check's own name. + if data.get("status") != "success": + raise RuntimeError( + "Prometheus rejected the query: " + f"{data.get('error') or data.get('status')}" + ) + results = data.get("data", {}).get("result", []) + remaining = deadline - time.monotonic() + if results or remaining <= 0: + return results + await asyncio.sleep(min(METRIC_POLL_INTERVAL_SEC, remaining)) + + async def _check_parity_value( session: aiohttp.ClientSession, prometheus_url: str, @@ -2323,18 +2365,20 @@ async def _check_parity_value( check_name = f"parity.value_sanity.{name}" try: - async with session.get( - f"{prometheus_url}/api/v1/query", params={"query": entry["query"]} - ) as resp: - data = await resp.json() - results = data.get("data", {}).get("result", []) + deadline = time.monotonic() + METRIC_POLL_TIMEOUT_SEC + results = await _poll_instant_query( + session, prometheus_url, entry["query"], deadline + ) if not results: return CheckResult( name=check_name, category="parity", passed=False, - message=f"{name}: no data returned from Prometheus", + message=( + f"{name}: no data returned from Prometheus after " + f"{METRIC_POLL_TIMEOUT_SEC:g}s" + ), ) values: list[float] = [] From 478b3e4b070944413a955dcaf56e60cd3eaf3e05 Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Wed, 9 Sep 2026 13:16:13 +0100 Subject: [PATCH 09/16] docs(telemetry): correct stale claims and citations in the harness docs The workload README contradicted itself on --skip-loki: one bullet said CI always passes it and so the two log-correlation checks are never exercised, another said the workflow no longer passes it. The workflow mentions the flag nowhere, so the first was the stale half. Other claims checked against the tree and corrected: - both the README and the plan doc described the push trigger as filtered on branch names. The workflow has no branches filter, deliberately, because GitHub ANDs branches with paths - the plan doc printed 6 of the workflow's 12 paths globs, and claimed the workflow was 367 lines against an actual 451. The glob block is now generated from the workflow, and the line count dropped rather than restated - rpcNOT_SUPPORTED does not exist anywhere in the tree. The symbol is RpcNotSupported, and the refusal sites are RipplePathFind.cpp:59-60 and PathFind.cpp:50-51, not :48-49 and :39 - RCLConsensus.cpp:666 and :663 are not log or event lines; the tx.included event is at :720 and the per-transaction debug log at :715 - LedgerMaster.cpp:463 is fixIndex, not the ledger.store span, which is at :470 - ServerHandler.cpp:705 is inside makeJsonError; processRequest is at :718 - file counts: docker/telemetry/workload/ is 25 files, include/xrpl/telemetry/ 13 - the optional-span bullet named five causes covering 10 of 16 entries, omitting the txq.* family and the WebSocket handshake - the /api/v1/series choice was attributed to stale StatsD gauges; this harness runs no StatsD A line number in run-full-validation.sh was cited in five places and drifts on every edit to that file, so those now name the file only. The keygen helper's header records what production does instead -- validator-keys-tool create_keys then create_token, keeping the master key off the node -- and why a disposable cluster does not. --- OpenTelemetryPlan/06-implementation-phases.md | 40 +++++++++++-------- docker/telemetry/workload/README.md | 26 +++++++----- docker/telemetry/workload/baselines/README.md | 8 ++-- .../telemetry/workload/expected_metrics.json | 6 +-- docker/telemetry/workload/expected_spans.json | 14 +++---- .../workload/regression-metrics.json | 4 +- docs/telemetry-runbook.md | 2 +- 7 files changed, 56 insertions(+), 44 deletions(-) diff --git a/OpenTelemetryPlan/06-implementation-phases.md b/OpenTelemetryPlan/06-implementation-phases.md index f1991d5f19..8ed289cb73 100644 --- a/OpenTelemetryPlan/06-implementation-phases.md +++ b/OpenTelemetryPlan/06-implementation-phases.md @@ -919,7 +919,7 @@ Alert Rules from External Dashboard**. ## 6.8.3 Phase 10: Synthetic Workload Generation & Telemetry Validation (Weeks 16-17) -> **Status**: Implemented on this branch — `docker/telemetry/workload/` (24 +> **Status**: Implemented on this branch — `docker/telemetry/workload/` (25 > files) and `.github/workflows/telemetry-validation.yml` are present here. > Upstream branches do not carry them, so the exit criteria below only hold from > `pratik/otel-phase10-workload-validation` onward. @@ -1003,7 +1003,7 @@ flowchart LR - **Transaction submitter and RPC load generator** both use xrpld's native WebSocket command format (`{"command": ...}`) — not JSON-RPC format. Response data lives inside `"result"` with `"status"` at the top level. - **Node config** requires `[signing_support] true` for server-side signing, and `[ips]` (not `[ips_fixed]`) to ensure peer connections count in `peer_finder_active_*` metrics. -- **Metric validation** uses the Prometheus `/api/v1/series` endpoint (not instant queries) to avoid false negatives from stale StatsD gauges. Every metric in `expected_metrics.json` must have > 0 series. +- **Metric validation** uses the Prometheus `/api/v1/series` endpoint (not instant queries) which polls for late-populating series and ignores Prometheus's staleness horizon. Every metric in `expected_metrics.json` must have > 0 series. - **Gauge visibility**: the harness sets `[insight] server=otel` (`run-full-validation.sh`), so `beast::insight` gauges become OTel observable gauges whose callback is invoked on every collection cycle. A gauge that sits at 0 and never changes (e.g. `jobq_job_count`) therefore still reports, and `/api/v1/series` sees it. - **I/O latency fix**: `io_latency_sampler` emits unconditionally on first sample, then applies the 10 ms threshold. This ensures `ios_latency` is registered in Prometheus even in low-load CI environments. - **tx.receive span**: attribute keys are bare, not dotted — `suppressed` and `tx_status` (`TxSpanNames.h:71,75`). `suppressed` is set on both outcomes (`false` on the accepted path, `true` when the HashRouter suppresses), but `tx_status` is set **only** on the reject/known-bad/dropped paths, so it is absent on a successful receive. Assert on the attribute, not on span status. @@ -1070,15 +1070,16 @@ See [Phase10_taskList.md](./Phase10_taskList.md) for the per-task breakdown. ### CI Deliverable (Task 10.6) The Phase 10 CI entry point is `.github/workflows/telemetry-validation.yml` -(367 lines, on the Phase 10 branch). It runs three jobs — `linux-image-tag`, +(on the Phase 10 branch). It runs three jobs — `linux-image-tag`, `build-xrpld`, `validate-telemetry` — and is triggered by `workflow_dispatch` -plus `push` on `pratik/otel-phase*`, `feature/otel-*` and -`feature/telemetry-*`. **There is no cron schedule**, so nothing runs this -workflow on a timer. +plus any `push` that touches one of the `paths` globs below. **There is no +branch filter**: GitHub ANDs `branches` with `paths`, so a branch glob would +decide validation by what a branch is called rather than by what it changed. +**There is no cron schedule**, so nothing runs this workflow on a timer. > **Fixed — the `push` trigger's `paths` filter now covers the C++ telemetry -> sources.** The branch filter is only half the trigger; `push` also carries a -> `paths` filter, and it previously read: +> sources.** The `push` trigger carries a `paths` filter, and it previously +> read: > > ```yaml > paths: @@ -1092,14 +1093,14 @@ workflow on a timer. > `include/xrpl/basics/Telemetry*.h` nor `src/xrpld/app/misc/Telemetry*` exists. > The telemetry code lives in `src/xrpld/telemetry/**` (9 files, including > `MetricsRegistry.cpp`), `src/libxrpl/telemetry/**` (7 files) and -> `include/xrpl/telemetry/**` (10 files), none of which were listed. +> `include/xrpl/telemetry/**` (13 files), none of which were listed. > Consequence at the time: a pure C++ telemetry change — new instrument, > renamed metric, changed span attribute — never triggered this workflow on > push; only edits under `docker/telemetry/**` or to the workflow file itself > did. > -> The two dead globs have been replaced with the three real module directories, -> so the filter now reads: +> The two dead globs have been replaced with the real module directories, the +> name-constant headers and the checkers, so the filter now reads: > > ```yaml > paths: @@ -1107,15 +1108,22 @@ workflow on a timer. > - "docker/telemetry/**" > - "include/xrpl/telemetry/**" > - "src/libxrpl/telemetry/**" -> - "src/libxrpl/beast/insight/**" > - "src/xrpld/telemetry/**" +> - "include/xrpl/beast/insight/**" +> - "src/libxrpl/beast/insight/**" +> - "**/*SpanNames.h" +> - "**/*MetricNames.h" +> - "src/tests/libxrpl/telemetry/**" +> - ".github/scripts/otel-naming/**" +> - ".github/scripts/telemetry/**" > ``` > > `src/libxrpl/beast/insight/**` is included because it holds `OTelCollector.cpp`, -> the `beast::insight` OTLP export path the harness depends on. Residual gap: the -> instrumented call sites scattered through `src/xrpld/app/` are not listed, so a -> change that only adds or moves a span at a call site does not trigger the -> workflow on push. Those are reachable by manual dispatch. +> the `beast::insight` OTLP export path the harness depends on. The `*SpanNames.h` +> and `*MetricNames.h` globs cover the name constants wherever they sit, including +> under `src/xrpld/app/`. Residual gap: an instrumented call site that adds or +> moves a span without touching a name header does not trigger the workflow on +> push. Those are reachable by manual dispatch. > **Caveat — four inert inputs (documented, not wired).** The workflow declares > five `workflow_dispatch` inputs, but only `run_benchmark` changes behaviour. diff --git a/docker/telemetry/workload/README.md b/docker/telemetry/workload/README.md index 5a212022b4..07c3bacb0a 100644 --- a/docker/telemetry/workload/README.md +++ b/docker/telemetry/workload/README.md @@ -213,9 +213,9 @@ python3 tx_submitter.py --endpoint ws://localhost:6006 \ Automated validation that all expected telemetry data exists. Every metric in `expected_metrics.json` is required — if it doesn't fire, the validation fails. Spans are required unless the entry carries `"optional": true`. -- **Span validation**: All span types from `expected_spans.json` with required attributes and parent-child hierarchies. Entries marked `"optional": true` only fire under traffic the harness may not produce (HTTP/JSON-RPC client, gRPC client, path-finding RPC — see [Pathfinding is not exercised](#pathfinding-is-not-exercised) — missing-ledger fetch, mode transitions); their absence is recorded as a passing skip, not a failure. +- **Span validation**: All span types from `expected_spans.json` with required attributes and parent-child hierarchies. The 16 entries marked `"optional": true` are the ones the harness cannot guarantee: no gRPC client, no path-finding RPC (see [Pathfinding is not exercised](#pathfinding-is-not-exercised)), no missing-ledger fetch, no mode transition, no WebSocket handshake, and the six `txq.*` spans only when fee escalation puts something in the queue. `rpc.http_request` and `rpc.process` are marked optional because the generator drives WebSocket rather than HTTP. Their absence is recorded as a passing skip, not a failure. - **Metric validation**: All metrics from `expected_metrics.json` — SpanMetrics, `beast::insight` gauges/counters/histograms, `MetricsRegistry` OTLP metrics. Every listed metric must have > 0 series. Uses the Prometheus `/api/v1/series` endpoint (not instant queries), polled until the metric appears or the poll window elapses, so a late-populating or quiet series is not a false negative. -- **Log-trace correlation**: trace_id/span_id in Loki logs (requires Loki). The two checks are `log.trace_id_present` and `log.trace_id_cross_reference`, and they exist only when `--skip-loki` is **not** passed — `run_validation()` builds them inside an `if not skip_loki` branch, so with the flag they are absent from the report rather than reported as skipped. **CI always passes `--skip-loki`, so these two are never exercised there** — see [CI Integration](#ci-integration). +- **Log-trace correlation**: trace_id/span_id in Loki logs (requires Loki). The two checks are `log.trace_id_present` and `log.trace_id_cross_reference`, and they exist only when `--skip-loki` is **not** passed — `run_validation()` builds them inside an `if not skip_loki` branch, so with the flag they are absent from the report rather than reported as skipped. The workflow passes no `--skip-loki`, so both checks are built and gated on every CI run — see [CI Integration](#ci-integration). - **Dashboard validation**: Every dashboard uid listed under `grafana_dashboards.uids` in `expected_metrics.json` loads with panels. That list currently covers **all 15** dashboards provisioned in `docker/telemetry/grafana/dashboards/`. Note the scope of this check: it asks the Grafana API whether the dashboard exists and returns a panel count — it does **not** run the panels' queries, so a dashboard can pass here while individual panels render empty. ```bash @@ -359,7 +359,7 @@ from the running nodes, and writes them as JSON. `benchmark.sh` calls it once per leg; it is rarely run by hand. ```bash -./collect_system_metrics.sh 5020,5021,5022 300 /tmp/metrics.json +./collect_system_metrics.sh 5020,5021,5022 300 /tmp/metrics.json [pids_csv] ``` Processes are selected by matching `argv[0]`'s basename against the daemon @@ -370,8 +370,12 @@ string, are not sampled — including them diluted the CPU average and attributed a foreign process's RSS to the node. `ps -C xrpld` is not usable for this: xrpld renames itself, so its `comm` is `xrpld-main`. -Selection covers the whole host, so a second xrpld from another checkout is -sampled as well. Benchmark on a machine running one cluster only. +A fourth argument narrows selection to an explicit pid list, and `benchmark.sh` +always passes its own nodes' pids. It has to: `run-full-validation.sh` leaves its +five validation nodes running while the benchmark's three start, so host-wide +selection would average eight processes in both arms and report the largest of +them as the RSS peak. Without the argument the scope is still the whole host, so +a second xrpld from another checkout is sampled as well. The output carries a `metrics_complete` flag. It is `false` when any measurement source came back empty — no matching process, no successful RPC @@ -433,7 +437,7 @@ Categories: The validation runs as a GitHub Actions workflow (`.github/workflows/telemetry-validation.yml`): -- Triggered manually (`workflow_dispatch`) or on pushes to telemetry branches. There is no cron schedule. +- Triggered manually (`workflow_dispatch`), or by any push touching the workflow's `paths` globs. There is no branch filter and no cron schedule. - Builds xrpld, starts the full stack, runs load, validates - Uploads reports as artifacts (and node logs when validation did not succeed) - Writes the validation summary and the regression-gate summary to the workflow **Step Summary** (`$GITHUB_STEP_SUMMARY`). It does **not** comment on the PR — the workflow declares no `permissions:` block and calls no GitHub API, so read the summary on the run page. @@ -446,10 +450,10 @@ them again — load shape comes entirely from `--profile` and ### Log-trace correlation in CI -The workflow no longer passes `--skip-loki`, so `log.trace_id_present` and +The workflow passes no `--skip-loki`, so `log.trace_id_present` and `log.trace_id_cross_reference` are constructed and gated on every CI run. A green -`Telemetry Validation` is now evidence that log lines carry trace context and -that a logged trace id resolves to an exported trace. `integration-test.sh` has +`Telemetry Validation` is evidence that log lines carry trace context and that a +logged trace id resolves to an exported trace. `integration-test.sh` has its own `check_log_correlation()`, but no workflow runs that script. Correlation depends on four independent legs, and a failed check on its own names @@ -515,8 +519,8 @@ Re-run it after any change to log formatting, span activation, the collector's **Why.** Pathfinding is disabled on every node this harness starts, so those calls could only ever fail: - `src/xrpld/core/detail/Config.cpp:725-726` sets `pathSearchMax = 0` whenever a `[validation_seed]` or `[validator_token]` section is present — "by default, validators don't have pathfinding enabled". -- `run-full-validation.sh:308` writes `[validation_seed]` into every generated node cfg, and that script carries no `[path_search]`, `[path_search_fast]` or `[path_search_max]` section to put the default back. -- `src/xrpld/rpc/handlers/orderbook/RipplePathFind.cpp:48-49` therefore returns `rpcNOT_SUPPORTED`; `PathFind.cpp:39` does the same for `path_find`. +- `run-full-validation.sh` writes `[validation_seed]` into every generated node cfg, and that script carries no `[path_search]`, `[path_search_fast]` or `[path_search_max]` section to put the default back. +- `src/xrpld/rpc/handlers/orderbook/RipplePathFind.cpp:59-60` therefore returns `RpcNotSupported`; `PathFind.cpp:50-51` does the same for `path_find`. **Why the refusals would not be harmless.** They are not silent. `pathfind.request` is opened at `RipplePathFind.cpp:35`, **above** that guard, so a refused call still exports a span, and the enclosing `rpc.command.ripple_path_find` span carries `rpc_status=error`. At a 3% weight that is a steady ~3% error floor in `span_calls_total{status_code="STATUS_CODE_ERROR"}` — a figure that reads as an xrpld error rate and is not one. **An error-rate threshold derived from a harness run that does issue path-finding load is measuring the harness, not xrpld.** diff --git a/docker/telemetry/workload/baselines/README.md b/docker/telemetry/workload/baselines/README.md index 684e7e640d..75a75c2561 100644 --- a/docker/telemetry/workload/baselines/README.md +++ b/docker/telemetry/workload/baselines/README.md @@ -142,7 +142,7 @@ needs a finer low-end ladder **as well as** a spread-aware baseline. `0.005` / `0.0095` / `0.0099` ms, which is `0.5` / `0.95` / `0.99 × 0.01` ms — the ladder's first edge times the quantile, the signature of every sample landing in the first bucket. Those numbers are interpolation arithmetic on the bucket floor, not latencies. It is physically plausible: -[`LedgerMaster.cpp:463`](../../../../src/xrpld/app/ledger/detail/LedgerMaster.cpp#L463) wraps an +[`LedgerMaster.cpp:470`](../../../../src/xrpld/app/ledger/detail/LedgerMaster.cpp#L470) wraps an in-memory `ledgerHistory_.insert`, which completes in single-digit microseconds. While all the mass stays under 10 us the reported quantile cannot move materially, so **no @@ -180,7 +180,7 @@ while the other quantile stayed well inside its bound in the same run — the si not of a regression. The mechanism is arrival timing, not slow code. The span opens only once a quorum-completing -validation arrives ([`LedgerMaster.cpp:987`](../../../../src/xrpld/app/ledger/detail/LedgerMaster.cpp#L987), +validation arrives ([`LedgerMaster.cpp:1003`](../../../../src/xrpld/app/ledger/detail/LedgerMaster.cpp#L1003), inside `checkAccept`, past the `tvc < minVal` early return) and wraps the promotion work that follows — `setValidated`, `setFull`, `setValidLedger`, `pendSaveValidated`. Its duration therefore tracks when peer validations arrive in a 5-node cluster and what promotion then schedules, so a @@ -323,7 +323,7 @@ each node's `[rpc_startup]` stanza. Logging is **synchronous**, and several of t spans contain log statements, so the configured level is part of the measurement: - `ledger.build` contains [`BuildLedger.cpp:81`](../../../../src/xrpld/app/ledger/detail/BuildLedger.cpp#L81) (debug). -- `consensus.accept` contains [RCLConsensus.cpp:655/663/686](../../../../src/xrpld/app/consensus/RCLConsensus.cpp#L663) (debug) — `:663` logs **once per transaction** in the canonical set. +- `consensus.accept` contains [RCLConsensus.cpp:683/687/698/715](../../../../src/xrpld/app/consensus/RCLConsensus.cpp#L715) (debug) — `:715` logs **once per transaction** in the canonical set. - `tx.apply` and the other `spans.names` entries in [`../regression-metrics.json`](../regression-metrics.json) are affected the same way. Raising the level admits more of those statements and inflates the p50/p95/p99 of the very @@ -415,7 +415,7 @@ the first produces metrics that look gated in the report but are not. `rpc.process` is deliberately absent from the `spans.names` list in `regression-metrics.json`, so no `span.rpc.process.*` key appears in this baseline. The span is created only in `ServerHandler::processRequest()` -(`src/xrpld/rpc/detail/ServerHandler.cpp:705`), which is reached only from the +(`src/xrpld/rpc/detail/ServerHandler.cpp:718`), which is reached only from the HTTP/JSON-RPC session path. The harness load generator is WebSocket-only and that path never calls `processRequest`, so the span is never emitted under any workload profile here — `expected_spans.json` marks it `"optional": true` for diff --git a/docker/telemetry/workload/expected_metrics.json b/docker/telemetry/workload/expected_metrics.json index 1c0534c750..34a4b55fad 100644 --- a/docker/telemetry/workload/expected_metrics.json +++ b/docker/telemetry/workload/expected_metrics.json @@ -199,7 +199,7 @@ "not_asserted": { "description": "Emitted-and-dashboarded metrics deliberately left unasserted because they are workload-gated or defect-gated: the harness workload cannot guarantee they appear, and a check that fails on a healthy run is worse than no check. This group has no \"metrics\" key, so validate_telemetry.py skips it (validate_metrics iterates category_data.get(\"metrics\", [])). Promote an entry into an asserted group only after the workload is changed to guarantee it.", "metrics_excluded": { - "rpc_method_errored_total": "MetricsRegistry.cpp:332-333, push counter. 'Errored' here means a thrown C++ exception, not an error status in the JSON reply: the only caller is PerfLogImp.cpp:409 under 'if (!finish)', reached only through PerfLogImp::rpcError (PerfLogImp.h:150-153), whose only call site is the catch (std::exception&) handler in RPCHandler.cpp:213. An RPC that returns an error status normally still takes the rpcFinish path at RPCHandler.cpp:190 and increments rpc_method_finished_total. That distinction mattered here while the generator still issued ripple_path_find: those calls were in fact refused — pathfinding is off on every harness node, so doRipplePathFind returns rpcNOT_SUPPORTED (see pathfind_fast_milliseconds below) — and it would have been easy to conclude from that alone that this counter must fire. It does not, because a refusal is a normal return, not a throw. The harness issues no path-finding command at all, so the question is moot here, but the distinction is kept on record because it is the one that decides this entry. Nothing in rpc_load_generator.py's remaining server_info / fee / account / ledger / tx / DEX mix is expected to throw either, so no series may ever be created.", + "rpc_method_errored_total": "MetricsRegistry.cpp:332-333, push counter. 'Errored' here means a thrown C++ exception, not an error status in the JSON reply: the only caller is PerfLogImp.cpp:409 under 'if (!finish)', reached only through PerfLogImp::rpcError (PerfLogImp.h:150-153), whose only call site is the catch (std::exception&) handler in RPCHandler.cpp:213. An RPC that returns an error status normally still takes the rpcFinish path at RPCHandler.cpp:190 and increments rpc_method_finished_total. That distinction mattered here while the generator still issued ripple_path_find: those calls were in fact refused — pathfinding is off on every harness node, so doRipplePathFind returns RpcNotSupported (see pathfind_fast_milliseconds below) — and it would have been easy to conclude from that alone that this counter must fire. It does not, because a refusal is a normal return, not a throw. The harness issues no path-finding command at all, so the question is moot here, but the distinction is kept on record because it is the one that decides this entry. Nothing in rpc_load_generator.py's remaining server_info / fee / account / ledger / tx / DEX mix is expected to throw either, so no series may ever be created.", "ledger_history_mismatch_total": "MetricsRegistry.cpp:377, incremented only from LedgerHistory.cpp:332 on a built-vs-validated ledger mismatch. On a healthy run it never fires — asserting it would mean asserting a defect.", "txq_expired_total": "MetricsRegistry.cpp:379, incremented only at TxQ.cpp:1428 when a queued tx expires past its LastLedgerSequence. CI does run a txq-burst phase (workload-profiles.json:41, 30 s of single-type Payment at 60 TPS), but that does not guarantee sustained fee escalation followed by expiry: a run in which every other check passed still exposed only txq_metrics and no txq_expired_total.", "txq_dropped_total": "MetricsRegistry.cpp:381, incremented only at TxQ.cpp:1302 / :1347 on queue-full admission refusal. Same reason as txq_expired_total.", @@ -210,8 +210,8 @@ "getobject_charge": "GetObjectMetricNames.h:105, PeerImp.cpp:2931. Same gate.", "rpc_size_bytes": "ServerHandler.cpp:191, group('rpc')->makeEvent('size', Unit::Bytes). The OTLP Prometheus exporter derives the metric-name suffix from the declared unit, so a byte unit yields rpc_size_bytes. The Unit::Bytes declaration itself landed earlier, in 76c9051203; what 24094e427b changed was the exporter finally consuming it, replacing a hardcoded CreateDoubleHistogram(name, 'Duration in ms', 'ms') with otelUnitDescription(unit)/otelUnitCode(unit), and that is what renamed the series off rpc_size_milliseconds and the millisecond bucket ladder. Neither name was ever recorded here, so the harness could confirm neither the rename nor a regression back onto that ladder. Notified from ServerHandler::processRequest:1133, the HTTP JSON-RPC path — it computes an HTTP status and appends a trailing newline — and the load generators are WebSocket-only, the same gate regression-metrics.json:4 records for rpc.process, so only the harness's handful of HTTP health polls reach it. Real coverage needs an HTTP JSON-RPC phase in rpc_load_generator.py; that is a workload change rather than a harness correction, and is deliberately out of scope here.", "rpc_time_milliseconds": "ServerHandler.cpp:192, group('rpc')->makeEvent('time') with the default millisecond unit. Notified from ServerHandler::processRequest:1129, the same HTTP JSON-RPC call site as rpc_size_bytes and behind the same WebSocket-only gate.", - "pathfind_fast_milliseconds": "PathRequestManager.h:35, makeEvent('pathfind_fast') with the default millisecond unit, so the exported form is the pathfind_fast_milliseconds_bucket/_count/_sum triple and there is no bare series — the same convention io_latency and rpc_method_us follow, and rpc-pathfinding queries the _bucket. THE OPERATIVE BLOCKER IS THE CONFIG, NOT THE CALL GRAPH: pathfinding is disabled outright on every harness node, so no PathRequest is ever constructed and no pathfind_* histogram can exist. Config.cpp:725-726 sets pathSearchMax to 0 whenever a [validation_seed] or [validator_token] section is present ('By default, validators don't have pathfinding enabled'); run-full-validation.sh writes [validation_seed] into every generated node cfg (:308) and contains no [path_search], [path_search_fast] or [path_search_max] section to put it back (grep count 0 for path_search in that file — the only [path_search*] sections in docker/telemetry/ are in xrpld-telemetry.cfg and xrpld-telemetry-mainnet.cfg, neither of which the harness uses); and doRipplePathFind returns rpcNOT_SUPPORTED at RipplePathFind.cpp:48-49, before context.loadType is set and before any branch on the ledger parameter. That config gate alone would keep the metric absent even if the generator did issue the command, because every such call is refused at the front door; and the generator issues no path-finding RPC in the first place. Two independent reasons. Read that first: the structural argument below is correct and matters if pathfinding is ever enabled, but it is not why the metric is missing today. STRUCTURAL ARGUMENT (verified, applies once pathSearchMax is non-zero): reportFast's only caller is PathRequest.cpp:852, inside the 'if (fast && quickReply_ == {})' branch of PathRequest::doUpdate. The only doUpdate call that passes fast=true is in PathRequest::doCreate (PathRequest.cpp:259), guarded by '!hasCompletion()'. Both ripple_path_find entry points construct the PathRequest with a completion function, so hasCompletion() (:161-164) is true and the fast pass is skipped: with no ledger specified doRipplePathFind goes to makeLegacyPathRequest, which passes the coroutine-post lambda (RipplePathFind.cpp:140-160); with a ledger specified it goes to doLegacyPathRequest, which passes an empty-body but non-null lambda (PathRequestManager.cpp:317). Only the path_find streaming subscription reaches reportFast, because makePathRequest builds the request from a subscriber with no completion (PathRequestManager.cpp:261). The load generator has never used path_find: it fires one request per send and awaits one reply, which a streaming subscription does not fit. Covering this metric therefore needs both a [path_search_max] override (or a non-validator node) in run-full-validation.sh and a path_find subscription phase in the generator. Both are harness/workload changes and out of scope here. Grafana Cloud shows zero series in 180 days on the devnet nodes, consistent with those nodes receiving no pathfinding RPC.", - "pathfind_full_milliseconds": "PathRequestManager.h:36, makeEvent('pathfind_full'); same histogram naming as pathfind_fast_milliseconds above. Notified from reportFull (:87-90) via PathRequest.cpp:857, the 'else if (!fast && fullReply_ == {})' branch. Blocked by exactly the same config gate as pathfind_fast_milliseconds, and the probability of emission under the harness workload is zero, not low: pathSearchMax is 0 on every harness node (Config.cpp:725-726 plus the [validation_seed] section at run-full-validation.sh:308, with no [path_search*] override anywhere in that file), so doRipplePathFind returns rpcNOT_SUPPORTED at RipplePathFind.cpp:48-49 and no PathRequest object is ever constructed for reportFull to fire from. An earlier revision of this entry described the path as reachable-but-probabilistic — emitting one ledger close behind the request via PathRequestManager::updateAll's one-shot branch (PathRequestManager.cpp:160-166) — and prescribed sending an explicit ledger_index from the generator so doLegacyPathRequest would call doUpdate(cache, false) synchronously (PathRequestManager.cpp:321). Both halves were wrong. The probability is zero rather than merely unreliable, and the prescribed remedy cannot work at all, because the pathSearchMax guard fires before the ledger parameter is read: adding ledger_index changes nothing while pathfinding is off. The generator also issues no path-finding RPC, so covering this metric needs a [path_search_max] override (or a non-validator node) in run-full-validation.sh AND path-finding load added — a harness-topology plus workload change, out of scope here. The workload README section 'Pathfinding is not exercised' holds the recipe. Grafana Cloud shows zero series in 180 days on the devnet nodes, consistent with those nodes receiving no pathfinding RPC rather than with a broken exporter.", + "pathfind_fast_milliseconds": "PathRequestManager.h:35, makeEvent('pathfind_fast') with the default millisecond unit, so the exported form is the pathfind_fast_milliseconds_bucket/_count/_sum triple and there is no bare series — the same convention io_latency and rpc_method_us follow, and rpc-pathfinding queries the _bucket. THE OPERATIVE BLOCKER IS THE CONFIG, NOT THE CALL GRAPH: pathfinding is disabled outright on every harness node, so no PathRequest is ever constructed and no pathfind_* histogram can exist. Config.cpp:725-726 sets pathSearchMax to 0 whenever a [validation_seed] or [validator_token] section is present ('By default, validators don't have pathfinding enabled'); run-full-validation.sh writes [validation_seed] into every generated node cfg and contains no [path_search], [path_search_fast] or [path_search_max] section to put it back (grep count 0 for path_search in that file — the only [path_search*] sections in docker/telemetry/ are in xrpld-telemetry.cfg and xrpld-telemetry-mainnet.cfg, neither of which the harness uses); and doRipplePathFind returns RpcNotSupported at RipplePathFind.cpp:59-60, before context.loadType is set and before any branch on the ledger parameter. That config gate alone would keep the metric absent even if the generator did issue the command, because every such call is refused at the front door; and the generator issues no path-finding RPC in the first place. Two independent reasons. Read that first: the structural argument below is correct and matters if pathfinding is ever enabled, but it is not why the metric is missing today. STRUCTURAL ARGUMENT (verified, applies once pathSearchMax is non-zero): reportFast's only caller is PathRequest.cpp:852, inside the 'if (fast && quickReply_ == {})' branch of PathRequest::doUpdate. The only doUpdate call that passes fast=true is in PathRequest::doCreate (PathRequest.cpp:259), guarded by '!hasCompletion()'. Both ripple_path_find entry points construct the PathRequest with a completion function, so hasCompletion() (:161-164) is true and the fast pass is skipped: with no ledger specified doRipplePathFind goes to makeLegacyPathRequest, which passes the coroutine-post lambda (RipplePathFind.cpp:140-160); with a ledger specified it goes to doLegacyPathRequest, which passes an empty-body but non-null lambda (PathRequestManager.cpp:317). Only the path_find streaming subscription reaches reportFast, because makePathRequest builds the request from a subscriber with no completion (PathRequestManager.cpp:261). The load generator has never used path_find: it fires one request per send and awaits one reply, which a streaming subscription does not fit. Covering this metric therefore needs both a [path_search_max] override (or a non-validator node) in run-full-validation.sh and a path_find subscription phase in the generator. Both are harness/workload changes and out of scope here. Grafana Cloud shows zero series in 180 days on the devnet nodes, consistent with those nodes receiving no pathfinding RPC.", + "pathfind_full_milliseconds": "PathRequestManager.h:36, makeEvent('pathfind_full'); same histogram naming as pathfind_fast_milliseconds above. Notified from reportFull (:87-90) via PathRequest.cpp:857, the 'else if (!fast && fullReply_ == {})' branch. Blocked by exactly the same config gate as pathfind_fast_milliseconds, and the probability of emission under the harness workload is zero, not low: pathSearchMax is 0 on every harness node (Config.cpp:725-726 plus the [validation_seed] section run-full-validation.sh writes, with no [path_search*] override anywhere in that file), so doRipplePathFind returns RpcNotSupported at RipplePathFind.cpp:59-60 and no PathRequest object is ever constructed for reportFull to fire from. An earlier revision of this entry described the path as reachable-but-probabilistic — emitting one ledger close behind the request via PathRequestManager::updateAll's one-shot branch (PathRequestManager.cpp:160-166) — and prescribed sending an explicit ledger_index from the generator so doLegacyPathRequest would call doUpdate(cache, false) synchronously (PathRequestManager.cpp:321). Both halves were wrong. The probability is zero rather than merely unreliable, and the prescribed remedy cannot work at all, because the pathSearchMax guard fires before the ledger parameter is read: adding ledger_index changes nothing while pathfinding is off. The generator also issues no path-finding RPC, so covering this metric needs a [path_search_max] override (or a non-validator node) in run-full-validation.sh AND path-finding load added — a harness-topology plus workload change, out of scope here. The workload README section 'Pathfinding is not exercised' holds the recipe. Grafana Cloud shows zero series in 180 days on the devnet nodes, consistent with those nodes receiving no pathfinding RPC rather than with a broken exporter.", "warn_total": "include/xrpl/resource/detail/Logic.h:41, makeMeter('warn'). makeMeter maps to CreateUInt64Counter (OTelCollector.cpp:878-881 -> :773-777), so the Prometheus exporter appends _total; the meter is created on the bare collector with no group, hence the unprefixed name. Incremented only at Logic.h:481, inside the 'if (notify)' branch reached when a consumer's balance crosses kWarningThreshold. A cooperating 5-node cluster plus a rate-limited load generator never charges a consumer that far, and Grafana Cloud confirms zero series in 180 days. Recorded explicitly because this was briefly mis-diagnosed as a phantom metric: the rpc-pathfinding panel that queries it is correct, and renders empty only because the condition has not occurred.", "drop_total": "include/xrpl/resource/detail/Logic.h:42, makeMeter('drop'); same CreateUInt64Counter mapping and same _total suffix as warn_total. Incremented only at Logic.h:505, when a consumer's balance is at or above kDropThreshold and the connection is dropped. Grafana Cloud shows 2 live series, so unlike warn_total this one does fire in the wild — but only on a genuinely abusive consumer, which the harness deliberately does not create, so it is condition-gated all the same. Its rpc-pathfinding panel is likewise correct rather than phantom.", "jobq_*_milliseconds, jobq_*_q_milliseconds": "This key is a pattern rather than a literal metric name — unlike every other entry in this map it stands for a whole family, one pair per job type. Created per job type in JobTypeData.h:97-98 from info.name() and info.name() + kSuffixQueued ('_q'), so the exported names are jobq__milliseconds and jobq__q_milliseconds with the job type lowercased by formatName. Which job types appear depends on which jobs a run happens to schedule, so no individual name is guaranteed. They are also rounded up to a whole millisecond at source (Event.h:47-51 applies ceil to a millisecond value type), which is why 6e2b2da772 moved the ledger-data-sync q-wait panels off jobq__q_milliseconds_bucket onto job_queued_us_bucket — they are poor assertion targets regardless." diff --git a/docker/telemetry/workload/expected_spans.json b/docker/telemetry/workload/expected_spans.json index 3309bf86cb..375c29be22 100644 --- a/docker/telemetry/workload/expected_spans.json +++ b/docker/telemetry/workload/expected_spans.json @@ -25,7 +25,7 @@ "required_attributes": [], "config_flag": "trace_rpc", "optional": true, - "note": "HTTP-only. Created solely in ServerHandler::processRequest() (ServerHandler.cpp:705), which is reached only from processSession(Session, coro) (ServerHandler.cpp:646) — the HTTP/JSON-RPC path that roots rpc.http_request at ServerHandler.cpp:640-641. The WebSocket path (processSession(WSSession, coro, jv), ServerHandler.cpp:467) never calls processRequest, so this span never appears under a WebSocket request. It does still appear under this harness, 5 traces on a normal run, because run-full-validation.sh polls each node over HTTP with curl (:449, :502) and those requests take the HTTP path. Note that the harness does speak HTTP: a reader concluding this span is unreachable here would go looking for a way to add HTTP traffic that already exists." + "note": "HTTP-only. Created solely in ServerHandler::processRequest() (ServerHandler.cpp:718), which is reached only from processSession(Session, coro) (ServerHandler.cpp:646) — the HTTP/JSON-RPC path that roots rpc.http_request at ServerHandler.cpp:640-641. The WebSocket path (processSession(WSSession, coro, jv), ServerHandler.cpp:467) never calls processRequest, so this span never appears under a WebSocket request. It does still appear under this harness, 5 traces on a normal run, because run-full-validation.sh polls each node over HTTP with curl (:449, :502) and those requests take the HTTP path. Note that the harness does speak HTTP: a reader concluding this span is unreachable here would go looking for a way to add HTTP traffic that already exists." }, { "name": "rpc.command.*", @@ -242,7 +242,7 @@ "resolution_direction" ], "config_flag": "trace_consensus", - "note": "Also carries close_time_correct, close_resolution_ms, consensus_state, proposing, round_time_ms, tx_count. Emits a tx.included span EVENT per transaction in the accepted set (RCLConsensus.cpp:666, with a tx_id attribute), which validate_telemetry.py cannot assert (no event support)." + "note": "Also carries close_time_correct, close_resolution_ms, consensus_state, proposing, round_time_ms, tx_count. Emits a tx.included span EVENT per transaction in the accepted set (RCLConsensus.cpp:720, with a tx_id attribute), which validate_telemetry.py cannot assert (no event support)." }, { "name": "consensus.validation.send", @@ -353,7 +353,7 @@ "required_attributes": ["pathfind_fast"], "config_flag": "trace_rpc", "optional": true, - "note": "Created by PathRequest::doUpdate (PathRequest.cpp:749-750), which the harness never reaches: pathfinding is disabled on every harness node, so no PathRequest is ever constructed. Config.cpp:725-726 sets pathSearchMax to 0 when a [validation_seed] or [validator_token] section is present, run-full-validation.sh writes [validation_seed] for every node (:308) and has no [path_search*] override, and doRipplePathFind then returns rpcNOT_SUPPORTED at RipplePathFind.cpp:48-49 — above the request-construction branches and below the pathfind.request span guard at :35. Liquidity is not the reason and never was: the call is refused before any path search is attempted, so the outcome does not depend on what the ledger holds. There is a second, independent reason: the harness sends no path-finding RPC at all, since rpc_load_generator.py carries no ripple_path_find weight. Enabling this span therefore needs BOTH a [path_search_max] override (or a non-validator node) in run-full-validation.sh AND the load restored — see the workload README section 'Pathfinding is not exercised'." + "note": "Created by PathRequest::doUpdate (PathRequest.cpp:749-750), which the harness never reaches: pathfinding is disabled on every harness node, so no PathRequest is ever constructed. Config.cpp:725-726 sets pathSearchMax to 0 when a [validation_seed] or [validator_token] section is present, run-full-validation.sh writes [validation_seed] for every node and has no [path_search*] override, and doRipplePathFind then returns RpcNotSupported at RipplePathFind.cpp:59-60 — above the request-construction branches and below the pathfind.request span guard at :35. Liquidity is not the reason and never was: the call is refused before any path search is attempted, so the outcome does not depend on what the ledger holds. There is a second, independent reason: the harness sends no path-finding RPC at all, since rpc_load_generator.py carries no ripple_path_find weight. Enabling this span therefore needs BOTH a [path_search_max] override (or a non-validator node) in run-full-validation.sh AND the load restored — see the workload README section 'Pathfinding is not exercised'." }, { "name": "pathfind.discover", @@ -371,7 +371,7 @@ "required_attributes": ["pathfind_ledger_index", "pathfind_num_requests"], "config_flag": "trace_rpc", "optional": true, - "note": "Async recomputation at ledger close. PathRequestManager::updateAll emits the span only when requests_ is non-empty (PathRequestManager.cpp:88-95), so it needs a live path_find subscription. On the harness requests_ can never become non-empty at all: the only two insertPathRequest call sites are makePathRequest (:268) and makeLegacyPathRequest (:296), and both handlers return rpcNOT_SUPPORTED first because pathfinding is disabled on every node (PathFind.cpp:39, RipplePathFind.cpp:48; see the pathfind.compute entry above for the config chain)." + "note": "Async recomputation at ledger close. PathRequestManager::updateAll emits the span only when requests_ is non-empty (PathRequestManager.cpp:88-95), so it needs a live path_find subscription. On the harness requests_ can never become non-empty at all: the only two insertPathRequest call sites are makePathRequest (:268) and makeLegacyPathRequest (:296), and both handlers return RpcNotSupported first because pathfinding is disabled on every node (PathFind.cpp:50-51, RipplePathFind.cpp:59; see the pathfind.compute entry above for the config chain)." }, { "name": "grpc.*", @@ -389,7 +389,7 @@ "child": "rpc.process", "description": "WebSocket message contains processing span", "skip": true, - "skip_reason": "This relationship does not exist in the code: rpc.process is created only in ServerHandler::processRequest() (ServerHandler.cpp:705), reached only from processSession(Session, coro) (ServerHandler.cpp:646) — the HTTP/JSON-RPC path. The WebSocket path (processSession(WSSession, coro, jv), ServerHandler.cpp:467) never calls processRequest, so rpc.process is never emitted at all under the WebSocket-only harness. The earlier diagnosis (cross-thread context loss needing a C++ fix) was wrong: rpc.ws_message is a deliberate freshRoot (ServerHandler.cpp:473-474) so each WS message is its own trace rather than nesting under a span leaked on a reused coroutine worker. Nothing to fix." + "skip_reason": "This relationship does not exist in the code: rpc.process is created only in ServerHandler::processRequest() (ServerHandler.cpp:718), reached only from processSession(Session, coro) (ServerHandler.cpp:646) — the HTTP/JSON-RPC path. The WebSocket path (processSession(WSSession, coro, jv), ServerHandler.cpp:467) never calls processRequest, so rpc.process is never emitted at all under the WebSocket-only harness. The earlier diagnosis (cross-thread context loss needing a C++ fix) was wrong: rpc.ws_message is a deliberate freshRoot (ServerHandler.cpp:473-474) so each WS message is its own trace rather than nesting under a span leaked on a reused coroutine worker. Nothing to fix." }, { "parent": "rpc.ws_message", @@ -421,7 +421,7 @@ "child": "pathfind.compute", "description": "Pathfind request contains the compute sub-span", "skip": true, - "skip_reason": "Real relationship (pathfind.compute is created inside PathRequest::doUpdate at PathRequest.cpp:749-750, under the pathfind.request scope), but the child never exists on the harness because pathfinding is disabled on every node: Config.cpp:725-726 zeroes pathSearchMax whenever a [validation_seed] or [validator_token] section is present, run-full-validation.sh writes [validation_seed] for all five nodes (:308) with no [path_search*] override, and doRipplePathFind returns rpcNOT_SUPPORTED at RipplePathFind.cpp:48-49 before constructing a PathRequest. The parent would appear anyway if the RPC were issued, because its ScopedSpanGuard is created at RipplePathFind.cpp:35, above that guard; but rpc_load_generator.py carries no ripple_path_find weight, so not even the parent appears and both ends of this relationship are absent. Liquidity has nothing to do with it — the earlier 'no liquidity, returns before computing' reason was wrong, because no path search is attempted at all. Asserting this relationship needs the load restored AND a [path_search_max] override (or a non-validator node) in run-full-validation.sh." + "skip_reason": "Real relationship (pathfind.compute is created inside PathRequest::doUpdate at PathRequest.cpp:749-750, under the pathfind.request scope), but the child never exists on the harness because pathfinding is disabled on every node: Config.cpp:725-726 zeroes pathSearchMax whenever a [validation_seed] or [validator_token] section is present, run-full-validation.sh writes [validation_seed] for all five nodes with no [path_search*] override, and doRipplePathFind returns RpcNotSupported at RipplePathFind.cpp:59-60 before constructing a PathRequest. The parent would appear anyway if the RPC were issued, because its ScopedSpanGuard is created at RipplePathFind.cpp:35, above that guard; but rpc_load_generator.py carries no ripple_path_find weight, so not even the parent appears and both ends of this relationship are absent. Liquidity has nothing to do with it — the earlier 'no liquidity, returns before computing' reason was wrong, because no path search is attempted at all. Asserting this relationship needs the load restored AND a [path_search_max] override (or a non-validator node) in run-full-validation.sh." }, { "parent": "rpc.command.*", @@ -436,7 +436,7 @@ "child": "pathfind.discover", "description": "The path computation contains the discovery pass.", "skip": true, - "skip_reason": "Real relationship with BOTH ends absent, for the reason given in full on the pathfind.compute entry above: pathfinding is disabled on every harness node because Config.cpp:725-726 zeroes pathSearchMax whenever a [validation_seed] section is present, run-full-validation.sh writes one for all five nodes (:308) with no [path_search_max] override, so doRipplePathFind returns rpcNOT_SUPPORTED before any PathRequest is constructed -- and the harness sends no path-finding RPC at all either. Asserting this needs both blockers lifted, which is a workload and node-config change rather than a harness one. Listed here so that the pathfinding family is fully accounted for rather than partly silent.", + "skip_reason": "Real relationship with BOTH ends absent, for the reason given in full on the pathfind.compute entry above: pathfinding is disabled on every harness node because Config.cpp:725-726 zeroes pathSearchMax whenever a [validation_seed] section is present, run-full-validation.sh writes one for all five nodes with no [path_search_max] override, so doRipplePathFind returns RpcNotSupported before any PathRequest is constructed -- and the harness sends no path-finding RPC at all either. Asserting this needs both blockers lifted, which is a workload and node-config change rather than a harness one. Listed here so that the pathfinding family is fully accounted for rather than partly silent.", "added": "Closes the declared-but-unlisted gap" }, diff --git a/docker/telemetry/workload/regression-metrics.json b/docker/telemetry/workload/regression-metrics.json index d53396e755..70508085ad 100644 --- a/docker/telemetry/workload/regression-metrics.json +++ b/docker/telemetry/workload/regression-metrics.json @@ -2,7 +2,7 @@ "_description": "Metric surface for the OTel-driven regression gate. Each entry names a metric, the quantiles to capture, and how to query Prometheus. The comparator compares current run against baseline-timings.json under these exact keys.", "_key_format": "{category}.{name}.p{quantile} (e.g. span.tx.process.p99, job.transaction.queued.p95). Only the categories defined below are captured; there is no rpc_methods group, so no rpc.* key is produced or gated (FU-4).", "_excluded_spans": "rpc.process is deliberately absent from spans.names. It is created only in ServerHandler::processRequest() on the HTTP/JSON-RPC path, which the workload load generators, being WebSocket-only, never reach, so its quantiles were captured as null every run and could never gate. (The harness shell scripts do issue a few HTTP JSON-RPC health polls, far too few to produce a meaningful quantile.) See baselines/README.md.", - "_excluded_ledger_store": "ledger.store is deliberately absent from spans.names too, for a different reason: it is below the ladder's resolution. The 2026-08-24 capture returned p50/p95/p99 of exactly 0.005/0.0095/0.0099 ms, which is 0.5/0.95/0.99 x the ladder's first edge of 0.01 ms — the signature of every sample landing in the first bucket, so the numbers are interpolation arithmetic on the bucket floor rather than latencies. That is physically plausible: LedgerMaster.cpp:463 wraps an in-memory ledgerHistory_.insert, which completes in single-digit microseconds. While all mass stays under 10 us the reported quantile cannot move materially, so NO absolute bound can gate it — every ledger.store slowing from 2 us to 9 us, 4.5x, leaves the reported value unchanged. Three keys that read as covered but cannot fire are worse than no keys (the same argument that excluded rpc.process), so they were removed rather than left in with a bound that looks derived. Restoring the key needs sub-10us edges on the collector's spanmetrics ladder (for example 0.001ms and 0.005ms) plus the matching entries in HistogramBuckets.h — that is the ladder's branch, not this file. ledger.store presence is still asserted by expected_spans.json and docker/telemetry/integration-test.sh, and its rate is still on the ledger-operations dashboard; only the latency gate drops it.", + "_excluded_ledger_store": "ledger.store is deliberately absent from spans.names too, for a different reason: it is below the ladder's resolution. The 2026-08-24 capture returned p50/p95/p99 of exactly 0.005/0.0095/0.0099 ms, which is 0.5/0.95/0.99 x the ladder's first edge of 0.01 ms — the signature of every sample landing in the first bucket, so the numbers are interpolation arithmetic on the bucket floor rather than latencies. That is physically plausible: LedgerMaster.cpp:470 wraps an in-memory ledgerHistory_.insert, which completes in single-digit microseconds. While all mass stays under 10 us the reported quantile cannot move materially, so NO absolute bound can gate it — every ledger.store slowing from 2 us to 9 us, 4.5x, leaves the reported value unchanged. Three keys that read as covered but cannot fire are worse than no keys (the same argument that excluded rpc.process), so they were removed rather than left in with a bound that looks derived. Restoring the key needs sub-10us edges on the collector's spanmetrics ladder (for example 0.001ms and 0.005ms) plus the matching entries in HistogramBuckets.h — that is the ladder's branch, not this file. ledger.store presence is still asserted by expected_spans.json and docker/telemetry/integration-test.sh, and its rate is still on the ledger-operations dashboard; only the latency gate drops it.", "_excluded_quantiles": "A THIRD KIND OF EXCLUSION, and the only one that deleting a name cannot express. spans.names lists span NAMES while _quantiles is shared across all of them, so the declared surface is the names x quantiles product and dropping ONE quantile of ONE span needs a subtraction. excluded_keys below is that subtraction: a flat {category}.{name}.p{quantile} key, exactly as _key_format defines it, mapped to the reason it is not gated. It can only ever remove a key, never add one, so a typo cannot silently start gating something new -- and check_regression_bounds.py rule F rejects an entry that would not otherwise be declared, an entry with an empty reason, and an entry that still carries a threshold override or a baseline value, so the exclusion cannot rot into dead config. Both prom_queries.py (which builds the capture plan) and check_regression_bounds.py (rule A) subtract it, so an excluded key is not queried, never reaches timings.json, and is not expected in the baseline. NOTHING ELSE CHANGES: the quantile is still computable from Prometheus with the _query_template above, the span is still asserted by expected_spans.json, and its rate is still on the ledger-operations dashboard. Only the latency gate drops it.", "_excluded_shape": "ALL FIVE ENTRIES BELOW SHARE ONE SHAPE, and it is worth naming because it will recur: the observed maximum across CI runs exceeds (baseline + bound), so an ordinary run clears the trip point with nothing having regressed. Two mechanisms produce that, and both are visible here. (1) A baseline that lands in the ladder's LOW buckets gets a tiny derived bound, because the bound IS the distance to the next edge up -- span.tx.apply.p50 at 0.0060 ms sits in the first bucket (0, 0.01] and gets 0.0440 ms of headroom, against a metric that has been measured at 2.3378 ms. (2) A spread so large that no bucket of headroom could absorb it -- span.ledger.validate.p99's 66.8x range reaches 25.8750 ms against a 10 ms trip point even though its bound is a comparatively generous 8.94 ms. The first mechanism is the one that excludes three keys here, and it is a property of WHERE THE CAPTURED RUN LANDED rather than of the metric: the same span.tx.apply.p50 has read 0.7917 ms, mid-distribution, where the identical rule produces a 4.21 ms bound that absorbs the whole range. Whether the gate functioned was therefore decided by luck of the draw. THE FOLLOW-UP THAT WOULD RESTORE COVERAGE, stated so it is not left implied: a baseline captured from a SINGLE run cannot support these keys, because one sample carries no information about spread and the bound is derived from that one sample alone. What would let them be gated again is a multi-run baseline -- or a spread measurement captured alongside the baseline, so a bound can be sized against observed variance instead of against the ladder only. That is not implemented; it is the design change these five exclusions are waiting on. Until then, do NOT re-gate any of them by re-baselining until a run happens to land favourably, which is the failure this note exists to prevent.", "excluded_keys": { @@ -10,7 +10,7 @@ "span.ledger.build.p50": "The same mechanism as span.consensus.ledger_close.p50, one bucket up. Baseline 0.1151 ms sits in (0.1, 0.25], so hi_next is 0.5 ms and the bound is 0.3849 ms -- a 4.34x trip point. Across three CI runs the value spans 0.1151 to 2.3826 ms, a 20.7x spread (25.3x over four runs), and the observed maximum is 4.77x the trip point. Note what the previous baseline hid: at 1.0612 ms the same rule gave a 8.94 ms bound and a 10 ms trip point, which absorbed the entire range, so this key read as gated purely because that capture landed mid-distribution. Ledger construction is the hot path this gate most wants to guard, which makes the loss real and worth fixing properly -- with a baseline that carries spread information, not with a wider bound.", "span.tx.apply.p50": "The most extreme case of the low-bucket mechanism, and the clearest evidence that a single-run baseline cannot size a bound for these keys. Baseline 0.00597 ms lands in the ladder's FIRST bucket (0, 0.01], so hi_next is 0.05 ms and the bound is 0.0440 ms. Across three CI runs the value spans 0.00597 to 2.3378 ms, a 391.8x spread (364x over four runs), putting the observed maximum at 46.76x its trip point -- by far the worst of the five. The previous baseline read 0.7917 ms for the same key on the same workload, a 132x difference between two runs, and at that value the identical rule produced a 4.21 ms bound whose 5 ms trip point absorbed the full range. Nothing about the metric changed between those two captures; only where the sampled run fell in its own distribution did. Separately, a baseline inside the first bucket means the reported figure is interpolation across that bucket and tracks the FRACTION of applies finishing under 10 us rather than a latency, which is the ledger.store problem in embryo -- so restoring this key needs a finer low-end ladder as well as a spread-aware baseline. Rule E does not flag it because the value is not quantile x first_edge exactly.", "span.ledger.validate.p95": "Run-to-run variance is larger than the bound this ladder can derive. Measured across four CI runs the value spans 0.1281 to 0.7500 ms, a 5.9x spread, against a baseline of 0.2404 ms whose trip point is the next ladder edge at 0.5 ms -- so an ordinary run clears the trip point with nothing having regressed. Run 32867433073 read 0.7500 ms, +212%, and turned CI red. The derived bound models QUANTIZATION noise only (hi_next - baseline is one bucket of headroom); the dominant noise term for this span is peer-validation arrival timing in a 5-node cluster, and that term was never measured before the key was gated. Widening is not available: a bound that tolerated 0.7500 ms would reach past the 1 ms edge and leave the key gating nothing. This is a variance limit, not a defect and not a missing bound -- do NOT re-gate it by widening.", - "span.ledger.validate.p99": "The same mechanism as p95, two orders of magnitude worse. Across the same four runs the value spans 0.3875 to 25.8750 ms, a 66.8x spread, against a baseline of 1.0600 ms and a 10 ms trip point; run 32862589645 read 25.8750 ms, +2341%. The span opens only once a quorum-completing validation arrives (LedgerMaster.cpp:987, inside checkAccept, past the tvc < minVal early return) and wraps the promotion work that follows -- setValidated, setFull, setValidLedger, pendSaveValidated -- so its duration tracks peer-validation arrival timing and what promotion then triggers. One slow consensus round therefore dominates the tail of a 3m rate window, and which round that is differs every run. A bound tolerating 25.8750 ms would be ~24.8 ms against a 1.0600 ms baseline, which gates nothing at all. Note that the two CI failures landed on DIFFERENT quantiles in different runs while the other quantile stayed well inside its bound in the same run: that asymmetry is the signature of variance, not of a regression." + "span.ledger.validate.p99": "The same mechanism as p95, two orders of magnitude worse. Across the same four runs the value spans 0.3875 to 25.8750 ms, a 66.8x spread, against a baseline of 1.0600 ms and a 10 ms trip point; run 32862589645 read 25.8750 ms, +2341%. The span opens only once a quorum-completing validation arrives (LedgerMaster.cpp:1003, inside checkAccept, past the tvc < minVal early return) and wraps the promotion work that follows -- setValidated, setFull, setValidLedger, pendSaveValidated -- so its duration tracks peer-validation arrival timing and what promotion then triggers. One slow consensus round therefore dominates the tail of a 3m rate window, and which round that is differs every run. A bound tolerating 25.8750 ms would be ~24.8 ms against a 1.0600 ms baseline, which gates nothing at all. Note that the two CI failures landed on DIFFERENT quantiles in different runs while the other quantile stayed well inside its bound in the same run: that asymmetry is the signature of variance, not of a regression." }, "spans": { "_query_template": "histogram_quantile({quantile}, sum by (le) (rate(span_duration_milliseconds_bucket{span_name=\"{name}\"}[{window}])))", diff --git a/docs/telemetry-runbook.md b/docs/telemetry-runbook.md index f881049dcd..2d85f4e99f 100644 --- a/docs/telemetry-runbook.md +++ b/docs/telemetry-runbook.md @@ -2853,7 +2853,7 @@ With all four satisfied, `info` is the minimum level at which the `log.trace_id_ `debug` does correlate strictly more: it additionally brings in [`BuildLedger.cpp:81`](../src/xrpld/app/ledger/detail/BuildLedger.cpp#L81) (inside the `ledger.build` `ScopedSpanGuard` at [:55](../src/xrpld/app/ledger/detail/BuildLedger.cpp#L55), once per ledger close) and [`RPCHandler.cpp:188`](../src/xrpld/rpc/detail/RPCHandler.cpp#L188) (inside the `rpc.command.*` `ScopedSpanGuard` at [:168](../src/xrpld/rpc/detail/RPCHandler.cpp#L168), once per RPC command), giving broader multi-subsystem coverage. -But raising the **base** level to `debug` puts synchronous log I/O inside `ledger.build`, `consensus.accept` (including [RCLConsensus.cpp:663](../src/xrpld/app/consensus/RCLConsensus.cpp#L663), which logs **per transaction**) and `tx.apply` — precisely the spans whose p50/p95/p99 latencies `regression-metrics.json` gates. A baseline captured at `debug` bakes that log I/O into the latency numbers permanently, turning the regression gate into a measurement of its own configuration. +But raising the **base** level to `debug` puts synchronous log I/O inside `ledger.build`, `consensus.accept` (including [RCLConsensus.cpp:715](../src/xrpld/app/consensus/RCLConsensus.cpp#L715), which logs **per transaction**) and `tx.apply` — precisely the spans whose p50/p95/p99 latencies `regression-metrics.json` gates. A baseline captured at `debug` bakes that log I/O into the latency numbers permanently, turning the regression gate into a measurement of its own configuration. So if you need the broader coverage, enable it **per partition** rather than globally, and only **after** a baseline has been captured at the harness's normal level: From db0eae1b3580328872bbc0f95aa7040b79188b88 Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Wed, 9 Sep 2026 14:05:33 +0100 Subject: [PATCH 10/16] fix(telemetry): pass StartOptions, not a string, in telemetry-off tests MetricsRegistry::start() takes a StartOptions aggregate. Six call sites in the #ifndef XRPL_ENABLE_TELEMETRY block still passed a std::string, so the block did not compile. Nothing in CI compiles it: telemetry defaults ON, and the block is skipped whenever the macro is defined. Add one shared kTestStartOptions carrying just the endpoint -- the other fields are never read on the no-op path -- and correct two comments that still described the old signature and a #else stub that does not exist. --- .../libxrpl/telemetry/MetricsRegistry.cpp | 35 ++++++++++++------- 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/src/tests/libxrpl/telemetry/MetricsRegistry.cpp b/src/tests/libxrpl/telemetry/MetricsRegistry.cpp index 11fbbfaed8..af3eb68ddf 100644 --- a/src/tests/libxrpl/telemetry/MetricsRegistry.cpp +++ b/src/tests/libxrpl/telemetry/MetricsRegistry.cpp @@ -603,13 +603,23 @@ using namespace xrpl; namespace { /** - * OTLP/HTTP endpoint passed to every start() call below. Nothing ever dials - * it -- these tests exercise the no-op path -- it just has to be a plausible - * URL. start() takes `std::string const&`, so call sites construct one from - * this view rather than repeating the literal. + * OTLP/HTTP endpoint used by every start() call below. Nothing ever dials it + * -- these tests exercise the no-op path -- it just has to be a plausible URL. + * It reaches start() through @ref kTestStartOptions. */ constexpr std::string_view kTestEndpoint{"http://localhost:4318/v1/metrics"}; +/** + * The only StartOptions field these tests need. + * + * start() takes the StartOptions aggregate, not a string. The other fields -- + * resource identity, network id, TLS paths -- are never read on the no-op + * path, and their defaults already mean "unset". One shared value keeps all + * six call sites on the same endpoint. + */ +telemetry::MetricsRegistry::StartOptions const kTestStartOptions{ + .endpoint = std::string{kTestEndpoint}}; + /** * Minimal mock ServiceRegistry for MetricsRegistry testing. * @@ -903,7 +913,7 @@ TEST_F(MetricsRegistryTest, disabled_start_stop) telemetry::MetricsRegistry registry(false, mockApp_, j_); // start() and stop() should be no-ops when disabled. - registry.start(std::string{kTestEndpoint}); + registry.start(kTestStartOptions); registry.stop(); // Double stop should be safe. @@ -924,8 +934,9 @@ TEST_F(MetricsRegistryTest, disabled_start_stop) // (src/tests/libxrpl/CMakeLists.txt:117-126 -- the `else()` branch; when it is // ON the .cpp needs concrete xrpld types such as LedgerMaster, TxQ, NetworkOPs, // Overlay and node_store::Database, which a standalone GTest binary cannot -// link). Both start() and startAsyncGauges() therefore compile here to their -// `#else` branch, which only (void)-casts its arguments. So these tests pin the +// link). Both start() and startAsyncGauges() have a single definition whose +// whole body sits inside #ifdef XRPL_ENABLE_TELEMETRY, so here they compile to +// an empty body with a [[maybe_unused]] parameter. So these tests pin the // API SURFACE -- that both entry points exist, are callable in either order, // and leave the object usable -- and NOT the gauge behaviour. Real coverage of // "gauges observe values only after startAsyncGauges()" is unreachable from @@ -943,7 +954,7 @@ TEST_F(MetricsRegistryTest, async_gauges_start_after_start_is_safe) telemetry::MetricsRegistry registry(false, mockApp_, j_); // The documented order: provider/sync instruments first, gauges second. - registry.start(std::string{kTestEndpoint}); + registry.start(kTestStartOptions); registry.startAsyncGauges(); // State: the enable flag is untouched by either phase. Exact value, not @@ -973,7 +984,7 @@ TEST_F(MetricsRegistryTest, async_gauges_before_start_does_not_break_start) EXPECT_EQ(registry.isEnabled(), false); // Phase 1 still works afterwards, so the bad call left no state behind. - registry.start(std::string{kTestEndpoint}); + registry.start(kTestStartOptions); registry.recordJobQueued("ledgerData", "ProcessLData"); EXPECT_EQ(registry.isEnabled(), false); @@ -994,7 +1005,7 @@ TEST_F(MetricsRegistryTest, async_gauges_respect_the_compile_time_guard) // return. EXPECT_EQ(registry.isEnabled(), true); - EXPECT_NO_THROW(registry.start(std::string{kTestEndpoint})); + EXPECT_NO_THROW(registry.start(kTestStartOptions)); EXPECT_NO_THROW(registry.startAsyncGauges()); EXPECT_NO_THROW(registry.stop()); @@ -1004,7 +1015,7 @@ TEST_F(MetricsRegistryTest, async_gauges_respect_the_compile_time_guard) TEST_F(MetricsRegistryTest, disabled_recording_methods) { telemetry::MetricsRegistry registry(false, mockApp_, j_); - registry.start(std::string{kTestEndpoint}); + registry.start(kTestStartOptions); // All recording methods should be no-ops (not crash). registry.recordRpcStarted("server_info"); @@ -1022,7 +1033,7 @@ TEST_F(MetricsRegistryTest, destructor_calls_stop) { // Let the destructor handle cleanup. telemetry::MetricsRegistry registry(false, mockApp_, j_); - registry.start(std::string{kTestEndpoint}); + registry.start(kTestStartOptions); } // If we get here without crash, the destructor handled stop. } From b6b7d468277608ab254eac39089d7b7694b848fd Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Wed, 9 Sep 2026 14:38:56 +0100 Subject: [PATCH 11/16] refactor(telemetry): make ValidationTracker lock-free The two writer paths run on the consensus and ledger-publish threads, and both took a single mutex that the nine window getters also held while scanning their deques. At a week of 4-second ledgers those deques hold about 151,000 records, the same record stored three times, so a read walked half a million entries and the writers waited behind it. Each writer now owns a 128-slot ring and shares nothing with the other, so a record is one timestamp read and one store. A full ring drops the event and bumps droppedEvents() rather than blocking a consensus thread; at 4-second ledgers that needs about eight and a half minutes with no drain, against a worst normal drain gap of one export interval plus one export timeout. The windows become a grid of one-minute buckets with a running agreed and total per span, so a getter reads a published snapshot instead of counting. Missed is derived from the two, which leaves a late repair touching only the agreed side. Steady-state footprint drops from about 8 MB to about 91 KB, and window edges quantise to a minute. Whichever thread is inside reconcile() owns the decision state; a second caller returns instead of waiting. Readers take a shared_ptr to the snapshot, so the values they read cannot be overwritten underneath them. The clock is now injected, which is what makes bucket expiry reachable in a test. The suite covers pairing, single-sided misses, the grace boundary, late repair across a bucket edge and past the window, each window edge exactly, two grid lengths of steady traffic, ring overflow, and three real-thread cases. It no longer sleeps: nine sleep_for calls totalling 81 seconds are gone. Every public method keeps its name and signature. --- .../libxrpl/telemetry/ValidationTracker.cpp | 880 ++++++++++++------ src/xrpld/telemetry/ValidationTracker.h | 600 +++++++++--- .../telemetry/detail/ValidationTracker.cpp | 450 +++++---- 3 files changed, 1301 insertions(+), 629 deletions(-) diff --git a/src/tests/libxrpl/telemetry/ValidationTracker.cpp b/src/tests/libxrpl/telemetry/ValidationTracker.cpp index 9a4d3d3b4b..e9b9558e9b 100644 --- a/src/tests/libxrpl/telemetry/ValidationTracker.cpp +++ b/src/tests/libxrpl/telemetry/ValidationTracker.cpp @@ -1,6 +1,10 @@ /** * @file ValidationTracker.cpp * Unit tests for xrpl::telemetry::ValidationTracker. + * + * The tracker reads time through an injected function, so every test places + * its events on a fake timeline. That is what makes a window edge, the grace + * period and a bucket boundary reachable without waiting for one. */ #include @@ -10,351 +14,637 @@ #include +#include #include #include #include #include +#include using namespace xrpl; using namespace xrpl::telemetry; +using Tracker = xrpl::telemetry::ValidationTracker; + +namespace { + /** - * Helper to create a unique uint256 from an integer seed. + * The fake current time. Every tracker built here reads it. */ -static uint256 +Tracker::TimePoint gNow{}; + +/** + * Time source handed to the tracker under test. + */ +Tracker::TimePoint +testNow() +{ + return gNow; +} + +/** + * Move the fake clock forward. + */ +void +advance(std::chrono::seconds by) +{ + gNow += by; +} + +/** + * Build a tracker on a fresh timeline. The start point is far from the clock + * epoch so subtracting a window cannot go negative. + */ +Tracker +makeTracker() +{ + gNow = Tracker::TimePoint{} + std::chrono::hours(1000); + return Tracker(&testNow); +} + +/** + * Push the clock past the grace period and reconcile, which is what turns a + * recorded event into an agreement or a miss. + */ +void +settle(Tracker& t) +{ + advance(Tracker::gracePeriod() + std::chrono::seconds(1)); + t.reconcile(); +} + +/** + * Distinct ledger hash per integer seed. + */ +uint256 makeHash(std::uint64_t n) { return uint256(n); } -/** - * Test fixture providing a fresh ValidationTracker per test. - */ -class ValidationTrackerTest : public ::testing::Test +} // namespace + +// ---- pairing --------------------------------------------------------------- + +TEST(ValidationTracker, both_sides_within_grace_counts_one_agreement) { -protected: - ValidationTracker tracker_; -}; + auto t = makeTracker(); + t.recordOurValidation(makeHash(1), 1); + t.recordNetworkValidation(makeHash(1), 1); + settle(t); -// --------------------------------------------------------------- -// 1. Normal agreement -// Record both our validation and network validation for the -// same hash, then reconcile after the grace period elapses. -// --------------------------------------------------------------- -TEST_F(ValidationTrackerTest, NormalAgreement) -{ - auto const hash = makeHash(1); - LedgerIndex const seq = 100; - - tracker_.recordOurValidation(hash, seq); - tracker_.recordNetworkValidation(hash, seq); - - // Immediately after recording, nothing is reconciled yet - // (grace period has not elapsed). - tracker_.reconcile(); - EXPECT_EQ(tracker_.totalValidationsSent(), 1u); - EXPECT_EQ(tracker_.totalValidationsChecked(), 1u); - - // Wait for the grace period (8 seconds) to elapse, then reconcile. - std::this_thread::sleep_for(std::chrono::seconds(9)); - tracker_.reconcile(); - - EXPECT_EQ(tracker_.totalAgreements(), 1u); - EXPECT_EQ(tracker_.totalMissed(), 0u); - EXPECT_EQ(tracker_.agreements1h(), 1u); - EXPECT_EQ(tracker_.missed1h(), 0u); - EXPECT_DOUBLE_EQ(tracker_.agreementPct1h(), 100.0); + EXPECT_EQ(t.agreements1h(), 1u); + EXPECT_EQ(t.missed1h(), 0u); + EXPECT_EQ(t.agreements24h(), 1u); + EXPECT_EQ(t.missed24h(), 0u); + EXPECT_EQ(t.agreements7d(), 1u); + EXPECT_EQ(t.missed7d(), 0u); + EXPECT_DOUBLE_EQ(t.agreementPct1h(), 100.0); + EXPECT_DOUBLE_EQ(t.agreementPct24h(), 100.0); + EXPECT_DOUBLE_EQ(t.agreementPct7d(), 100.0); } -// --------------------------------------------------------------- -// 2. Missed validation -// Only the network validates; we never do. After grace period -// the event should be reconciled as a miss. -// --------------------------------------------------------------- -TEST_F(ValidationTrackerTest, MissedValidation) +TEST(ValidationTracker, only_our_side_counts_one_miss) { - auto const hash = makeHash(2); - LedgerIndex const seq = 200; + auto t = makeTracker(); + t.recordOurValidation(makeHash(2), 2); + settle(t); - tracker_.recordNetworkValidation(hash, seq); - - // Wait for grace period then reconcile. - std::this_thread::sleep_for(std::chrono::seconds(9)); - tracker_.reconcile(); - - EXPECT_EQ(tracker_.totalAgreements(), 0u); - EXPECT_EQ(tracker_.totalMissed(), 1u); - EXPECT_EQ(tracker_.agreements1h(), 0u); - EXPECT_EQ(tracker_.missed1h(), 1u); - EXPECT_DOUBLE_EQ(tracker_.agreementPct1h(), 0.0); + EXPECT_EQ(t.agreements1h(), 0u); + EXPECT_EQ(t.missed1h(), 1u); + EXPECT_DOUBLE_EQ(t.agreementPct1h(), 0.0); } -// --------------------------------------------------------------- -// 3. Late repair -// Network validates first, grace period elapses (miss), then -// our validation arrives within the 5-minute repair window and -// the miss is flipped to an agreement. -// --------------------------------------------------------------- -TEST_F(ValidationTrackerTest, LateRepair) +TEST(ValidationTracker, only_network_side_counts_one_miss) { - auto const hash = makeHash(3); - LedgerIndex const seq = 300; + auto t = makeTracker(); + t.recordNetworkValidation(makeHash(3), 3); + settle(t); - // Network validates, but we do not (yet). - tracker_.recordNetworkValidation(hash, seq); - - // Grace period elapses -- reconciled as a miss. - std::this_thread::sleep_for(std::chrono::seconds(9)); - tracker_.reconcile(); - EXPECT_EQ(tracker_.totalMissed(), 1u); - EXPECT_EQ(tracker_.totalAgreements(), 0u); - EXPECT_EQ(tracker_.missed1h(), 1u); - - // Late arrival of our validation (within repair window). - tracker_.recordOurValidation(hash, seq); - tracker_.reconcile(); - - // Miss should be repaired to agreement. - EXPECT_EQ(tracker_.totalAgreements(), 1u); - EXPECT_EQ(tracker_.totalMissed(), 0u); - EXPECT_EQ(tracker_.agreements1h(), 1u); - EXPECT_EQ(tracker_.missed1h(), 0u); - EXPECT_DOUBLE_EQ(tracker_.agreementPct1h(), 100.0); + EXPECT_EQ(t.missed1h(), 1u); + EXPECT_EQ(t.agreements1h(), 0u); } -// --------------------------------------------------------------- -// 4. Empty window returns 0% -// When no events have been recorded the percentage methods -// must return 0.0, not NaN or any other value. -// --------------------------------------------------------------- -TEST_F(ValidationTrackerTest, EmptyWindowReturnsZero) +TEST(ValidationTracker, inside_the_grace_period_nothing_is_counted) { - EXPECT_DOUBLE_EQ(tracker_.agreementPct1h(), 0.0); - EXPECT_DOUBLE_EQ(tracker_.agreementPct24h(), 0.0); - EXPECT_EQ(tracker_.agreements1h(), 0u); - EXPECT_EQ(tracker_.missed1h(), 0u); - EXPECT_EQ(tracker_.agreements24h(), 0u); - EXPECT_EQ(tracker_.missed24h(), 0u); - EXPECT_EQ(tracker_.totalAgreements(), 0u); - EXPECT_EQ(tracker_.totalMissed(), 0u); - EXPECT_EQ(tracker_.totalValidationsSent(), 0u); - EXPECT_EQ(tracker_.totalValidationsChecked(), 0u); + auto t = makeTracker(); + t.recordOurValidation(makeHash(4), 4); + t.recordNetworkValidation(makeHash(4), 4); + + // One second short of the grace period, so the pair is still undecided. + advance(Tracker::gracePeriod() - std::chrono::seconds(1)); + t.reconcile(); + + EXPECT_EQ(t.agreements1h(), 0u); + EXPECT_EQ(t.missed1h(), 0u); + EXPECT_DOUBLE_EQ(t.agreementPct1h(), 0.0); + + // The message counters do not wait for a decision. + EXPECT_EQ(t.totalValidationsSent(), 1u); + EXPECT_EQ(t.totalValidationsChecked(), 1u); } -// --------------------------------------------------------------- -// 5. Grace period boundary -// Events recorded less than 8 seconds ago must NOT be -// reconciled. Verify that an immediate reconcile is a no-op. -// --------------------------------------------------------------- -TEST_F(ValidationTrackerTest, GracePeriodBoundary) +TEST(ValidationTracker, percentage_mixes_agreements_and_misses) { - auto const hash = makeHash(5); - LedgerIndex const seq = 500; - - tracker_.recordOurValidation(hash, seq); - tracker_.recordNetworkValidation(hash, seq); - - // Reconcile immediately -- grace period has not elapsed. - tracker_.reconcile(); - - // Nothing should be reconciled yet. - EXPECT_EQ(tracker_.totalAgreements(), 0u); - EXPECT_EQ(tracker_.totalMissed(), 0u); - EXPECT_EQ(tracker_.agreements1h(), 0u); - EXPECT_EQ(tracker_.missed1h(), 0u); - - // Lifetime send/check counters should still be incremented. - EXPECT_EQ(tracker_.totalValidationsSent(), 1u); - EXPECT_EQ(tracker_.totalValidationsChecked(), 1u); -} - -// --------------------------------------------------------------- -// 6. Max pending events -- trimming -// Add more than kMaxPendingEvents (1000) events. After -// reconciliation and a second reconcile pass the pending map -// should be trimmed. Lifetime totals must remain consistent. -// --------------------------------------------------------------- -TEST_F(ValidationTrackerTest, MaxPendingEventsTrimming) -{ - constexpr std::size_t kCount = 1100; - - for (std::size_t i = 0; i < kCount; ++i) + auto t = makeTracker(); + for (std::uint64_t i = 0; i < 3; ++i) { - auto const hash = makeHash(i + 1); - auto const seq = static_cast(i + 1); - tracker_.recordOurValidation(hash, seq); - tracker_.recordNetworkValidation(hash, seq); + t.recordOurValidation(makeHash(100 + i), static_cast(100 + i)); + t.recordNetworkValidation(makeHash(100 + i), static_cast(100 + i)); + } + t.recordOurValidation(makeHash(200), 200); + settle(t); + + EXPECT_EQ(t.agreements1h(), 3u); + EXPECT_EQ(t.missed1h(), 1u); + EXPECT_NEAR(t.agreementPct1h(), 75.0, 1e-9); +} + +// ---- an event is counted exactly once -------------------------------------- + +TEST(ValidationTracker, a_decided_ledger_is_not_counted_twice) +{ + auto t = makeTracker(); + t.recordOurValidation(makeHash(5), 5); + t.recordNetworkValidation(makeHash(5), 5); + settle(t); + EXPECT_EQ(t.agreements1h(), 1u); + + // The same hash arrives again long after it was counted and evicted. It + // must not open a fresh pending entry and be counted a second time. + advance(Tracker::lateRepairWindow() + std::chrono::seconds(1)); + t.reconcile(); + t.recordOurValidation(makeHash(5), 5); + t.recordNetworkValidation(makeHash(5), 5); + settle(t); + + EXPECT_EQ(t.agreements1h(), 1u); + EXPECT_EQ(t.missed1h(), 0u); +} + +TEST(ValidationTracker, duplicate_recording_of_one_ledger_counts_one_agreement) +{ + auto t = makeTracker(); + + // Our side reports the same ledger twice before anything is decided. + t.recordOurValidation(makeHash(30), 30); + t.recordOurValidation(makeHash(30), 30); + t.recordNetworkValidation(makeHash(30), 30); + settle(t); + + EXPECT_EQ(t.agreements1h(), 1u); + EXPECT_EQ(t.missed1h(), 0u); + EXPECT_EQ(t.totalAgreements(), 1u); + EXPECT_EQ(t.totalMissed(), 0u); +} + +TEST(ValidationTracker, send_and_check_counters_count_messages_not_ledgers) +{ + auto t = makeTracker(); + t.recordOurValidation(makeHash(31), 31); + t.recordNetworkValidation(makeHash(31), 31); + settle(t); + EXPECT_EQ(t.agreements1h(), 1u); + + // The same ledger is reported again once it is past repair. Neither total + // moves, but both message counters do. + advance(Tracker::lateRepairWindow() + std::chrono::seconds(1)); + t.reconcile(); + t.recordOurValidation(makeHash(31), 31); + t.recordNetworkValidation(makeHash(31), 31); + settle(t); + + EXPECT_EQ(t.agreements1h(), 1u); + EXPECT_EQ(t.missed1h(), 0u); + EXPECT_EQ(t.totalAgreements(), 1u); + EXPECT_EQ(t.totalMissed(), 0u); + EXPECT_EQ(t.totalValidationsSent(), 2u); + EXPECT_EQ(t.totalValidationsChecked(), 2u); +} + +// ---- late repair ----------------------------------------------------------- + +TEST(ValidationTracker, late_repair_turns_a_miss_into_an_agreement) +{ + auto t = makeTracker(); + t.recordNetworkValidation(makeHash(6), 6); + settle(t); + EXPECT_EQ(t.missed1h(), 1u); + EXPECT_EQ(t.agreements1h(), 0u); + + // Our own validation shows up inside the repair window. + advance(std::chrono::seconds(30)); + t.recordOurValidation(makeHash(6), 6); + t.reconcile(); + + EXPECT_EQ(t.missed1h(), 0u); + EXPECT_EQ(t.agreements1h(), 1u); + EXPECT_DOUBLE_EQ(t.agreementPct1h(), 100.0); +} + +TEST(ValidationTracker, late_repair_works_across_a_bucket_boundary) +{ + auto t = makeTracker(); + t.recordNetworkValidation(makeHash(7), 7); + settle(t); + EXPECT_EQ(t.missed1h(), 1u); + + // Cross into a later minute bucket before repairing, so the repair has to + // find the bucket the event was recorded in rather than the current one. + advance(std::chrono::seconds(120)); + t.recordOurValidation(makeHash(7), 7); + t.reconcile(); + + EXPECT_EQ(t.missed1h(), 0u); + EXPECT_EQ(t.agreements1h(), 1u); + EXPECT_EQ(t.agreements7d(), 1u); + EXPECT_EQ(t.missed7d(), 0u); +} + +TEST(ValidationTracker, repair_after_the_window_closes_is_ignored) +{ + auto t = makeTracker(); + t.recordNetworkValidation(makeHash(8), 8); + settle(t); + EXPECT_EQ(t.missed1h(), 1u); + + // Past the repair window, so the miss stands. + advance(Tracker::lateRepairWindow() + std::chrono::seconds(10)); + t.recordOurValidation(makeHash(8), 8); + t.reconcile(); + + EXPECT_EQ(t.missed1h(), 1u); + EXPECT_EQ(t.agreements1h(), 0u); +} + +// ---- window expiry --------------------------------------------------------- + +TEST(ValidationTracker, an_event_leaves_the_1h_window_but_stays_in_the_others) +{ + auto t = makeTracker(); + t.recordOurValidation(makeHash(9), 9); + t.recordNetworkValidation(makeHash(9), 9); + settle(t); + EXPECT_EQ(t.agreements1h(), 1u); + + advance(std::chrono::hours(1) + std::chrono::minutes(2)); + t.reconcile(); + + EXPECT_EQ(t.agreements1h(), 0u); + EXPECT_EQ(t.agreements24h(), 1u); + EXPECT_EQ(t.agreements7d(), 1u); + EXPECT_DOUBLE_EQ(t.agreementPct1h(), 0.0); + EXPECT_DOUBLE_EQ(t.agreementPct24h(), 100.0); +} + +TEST(ValidationTracker, an_event_leaves_the_24h_window_but_stays_in_7d) +{ + auto t = makeTracker(); + t.recordOurValidation(makeHash(10), 10); + t.recordNetworkValidation(makeHash(10), 10); + settle(t); + + advance(std::chrono::hours(24) + std::chrono::minutes(2)); + t.reconcile(); + + EXPECT_EQ(t.agreements1h(), 0u); + EXPECT_EQ(t.agreements24h(), 0u); + EXPECT_EQ(t.agreements7d(), 1u); +} + +TEST(ValidationTracker, an_event_leaves_every_window_after_7d) +{ + auto t = makeTracker(); + t.recordOurValidation(makeHash(11), 11); + t.recordNetworkValidation(makeHash(11), 11); + settle(t); + + advance(std::chrono::hours(168) + std::chrono::minutes(2)); + t.reconcile(); + + EXPECT_EQ(t.agreements7d(), 0u); + EXPECT_EQ(t.missed7d(), 0u); + EXPECT_DOUBLE_EQ(t.agreementPct7d(), 0.0); +} + +TEST(ValidationTracker, an_event_exactly_one_window_old_has_left_the_window) +{ + // The 1h window covers 60 minute buckets. An event 60 minutes old is out; + // 59 minutes old is still in. An off-by-one in the bound breaks one of + // these two checks. + auto t = makeTracker(); + t.recordOurValidation(makeHash(20), 20); + t.recordNetworkValidation(makeHash(20), 20); + settle(t); + EXPECT_EQ(t.agreements1h(), 1u); + + advance(std::chrono::minutes(59) - Tracker::gracePeriod()); + t.reconcile(); + EXPECT_EQ(t.agreements1h(), 1u); + + advance(std::chrono::minutes(1)); + t.reconcile(); + EXPECT_EQ(t.agreements1h(), 0u); + EXPECT_EQ(t.agreements24h(), 1u); +} + +TEST(ValidationTracker, an_event_exactly_one_day_old_has_left_the_day_window) +{ + auto t = makeTracker(); + t.recordOurValidation(makeHash(21), 21); + t.recordNetworkValidation(makeHash(21), 21); + settle(t); + + advance(std::chrono::minutes(24 * 60 - 1) - Tracker::gracePeriod()); + t.reconcile(); + EXPECT_EQ(t.agreements24h(), 1u); + + advance(std::chrono::minutes(1)); + t.reconcile(); + EXPECT_EQ(t.agreements24h(), 0u); + EXPECT_EQ(t.agreements7d(), 1u); +} + +TEST(ValidationTracker, a_gap_longer_than_the_7d_grid_does_not_resurrect_old_counts) +{ + auto t = makeTracker(); + t.recordOurValidation(makeHash(12), 12); + t.recordNetworkValidation(makeHash(12), 12); + settle(t); + EXPECT_EQ(t.agreements7d(), 1u); + + // Idle for more than one full trip around the bucket grid. Every bucket + // the new event reuses must be zeroed, not added to. + advance(std::chrono::hours(400)); + t.recordOurValidation(makeHash(13), 13); + t.recordNetworkValidation(makeHash(13), 13); + settle(t); + + EXPECT_EQ(t.agreements7d(), 1u); + EXPECT_EQ(t.missed7d(), 0u); + EXPECT_EQ(t.agreements1h(), 1u); +} + +TEST(ValidationTracker, steady_traffic_across_the_grid_boundary_keeps_recent_counts) +{ + // Events every minute for slightly more than 14 days, so bucket slots get + // reused while the tracker is continuously busy. The "idle longer than the + // grid" fast path must not fire here: recent minutes are still valid data. + // + // Two full trips around the grid, so a slot that was reused is later + // retired. One trip only proves reuse happened; the miscount from a bucket + // that was not cleared shows up when that bucket is subtracted. + auto t = makeTracker(); + + constexpr std::uint64_t kMinutes = 2 * 7 * 24 * 60 + 5; + for (std::uint64_t i = 0; i < kMinutes; ++i) + { + t.recordOurValidation(makeHash(i), static_cast(i)); + t.recordNetworkValidation(makeHash(i), static_cast(i)); + advance(std::chrono::seconds(9)); + t.reconcile(); + advance(std::chrono::seconds(51)); } - EXPECT_EQ(tracker_.totalValidationsSent(), kCount); - EXPECT_EQ(tracker_.totalValidationsChecked(), kCount); - - // Wait for grace period so all events can be reconciled. - std::this_thread::sleep_for(std::chrono::seconds(9)); - tracker_.reconcile(); - - // All events should be reconciled as agreements. - EXPECT_EQ(tracker_.totalAgreements(), kCount); - EXPECT_EQ(tracker_.totalMissed(), 0u); - - // Reconcile again to trigger pending eviction / trimming. - // The pending map should be trimmed, but totals remain correct. - tracker_.reconcile(); - EXPECT_EQ(tracker_.totalAgreements(), kCount); - EXPECT_EQ(tracker_.totalMissed(), 0u); + // One event per minute means each window holds exactly its own span, so + // these are exact, not "roughly". A bucket reused without being cleared + // would push them above the span. + EXPECT_EQ(t.agreements1h(), 60u); + EXPECT_EQ(t.missed1h(), 0u); + EXPECT_EQ(t.agreements24h(), 24u * 60u); + EXPECT_EQ(t.agreements7d(), 7u * 24u * 60u); + EXPECT_EQ(t.missed7d(), 0u); } -// --------------------------------------------------------------- -// 7. Multiple distinct ledgers -- mixed results -// Record a mix of agreements and misses to verify that window -// counts and percentages are computed correctly. -// --------------------------------------------------------------- -TEST_F(ValidationTrackerTest, MixedAgreementsAndMisses) +// ---- lifetime totals ------------------------------------------------------- + +TEST(ValidationTracker, lifetime_totals_survive_window_expiry) { - // 3 agreements: both sides validate. - for (int i = 1; i <= 3; ++i) + auto t = makeTracker(); + t.recordOurValidation(makeHash(14), 14); + t.recordNetworkValidation(makeHash(14), 14); + t.recordOurValidation(makeHash(15), 15); + settle(t); + EXPECT_EQ(t.totalAgreements(), 1u); + EXPECT_EQ(t.totalMissed(), 1u); + + advance(std::chrono::hours(200)); + t.reconcile(); + + EXPECT_EQ(t.agreements7d(), 0u); + EXPECT_EQ(t.totalAgreements(), 1u); + EXPECT_EQ(t.totalMissed(), 1u); +} + +TEST(ValidationTracker, a_repair_moves_a_lifetime_total_from_missed_to_agreed) +{ + auto t = makeTracker(); + t.recordNetworkValidation(makeHash(16), 16); + settle(t); + EXPECT_EQ(t.totalMissed(), 1u); + EXPECT_EQ(t.totalAgreements(), 0u); + + advance(std::chrono::seconds(20)); + t.recordOurValidation(makeHash(16), 16); + t.reconcile(); + + EXPECT_EQ(t.totalMissed(), 0u); + EXPECT_EQ(t.totalAgreements(), 1u); +} + +// ---- rings ----------------------------------------------------------------- + +TEST(ValidationTracker, an_empty_tracker_reports_zero_everywhere) +{ + auto t = makeTracker(); + EXPECT_EQ(t.agreements1h(), 0u); + EXPECT_EQ(t.missed1h(), 0u); + EXPECT_EQ(t.agreements24h(), 0u); + EXPECT_EQ(t.missed24h(), 0u); + EXPECT_EQ(t.agreements7d(), 0u); + EXPECT_EQ(t.missed7d(), 0u); + EXPECT_DOUBLE_EQ(t.agreementPct1h(), 0.0); + EXPECT_DOUBLE_EQ(t.agreementPct24h(), 0.0); + EXPECT_DOUBLE_EQ(t.agreementPct7d(), 0.0); + EXPECT_EQ(t.totalAgreements(), 0u); + EXPECT_EQ(t.totalMissed(), 0u); + EXPECT_EQ(t.totalValidationsSent(), 0u); + EXPECT_EQ(t.totalValidationsChecked(), 0u); + EXPECT_EQ(t.droppedEvents(), 0u); +} + +TEST(ValidationTracker, a_full_ring_drops_and_counts_instead_of_blocking) +{ + auto t = makeTracker(); + + // One more than the ring holds, with no drain in between. + for (std::size_t i = 0; i < Tracker::ringCapacity() + 1; ++i) + t.recordOurValidation(makeHash(1000 + i), static_cast(1000 + i)); + + EXPECT_EQ(t.droppedEvents(), 1u); + + settle(t); + EXPECT_EQ(t.missed1h(), Tracker::ringCapacity()); +} + +TEST(ValidationTracker, a_drop_on_one_ring_leaves_the_other_alone) +{ + auto t = makeTracker(); + for (std::size_t i = 0; i < Tracker::ringCapacity() + 4; ++i) + t.recordOurValidation(makeHash(2000 + i), static_cast(2000 + i)); + EXPECT_EQ(t.droppedEvents(), 4u); + + t.recordNetworkValidation(makeHash(3000), 3000); + settle(t); + + // The network ring was never full, so its event still missed normally: + // capacity misses from our ring, plus this one. + EXPECT_EQ(t.missed1h(), Tracker::ringCapacity() + 1); + EXPECT_EQ(t.droppedEvents(), 4u); +} + +TEST(ValidationTracker, draining_frees_ring_space_again) +{ + auto t = makeTracker(); + for (std::size_t i = 0; i < Tracker::ringCapacity(); ++i) + t.recordOurValidation(makeHash(4000 + i), static_cast(4000 + i)); + EXPECT_EQ(t.droppedEvents(), 0u); + + t.reconcile(); // drains, decides nothing yet + + for (std::size_t i = 0; i < Tracker::ringCapacity(); ++i) + t.recordOurValidation(makeHash(5000 + i), static_cast(5000 + i)); + + EXPECT_EQ(t.droppedEvents(), 0u); +} + +TEST(ValidationTracker, a_burst_larger_than_one_ring_is_counted_in_full_when_drained) +{ + // Pending events are bounded by the repair window, not by a count, so a + // burst several ring-loads long is counted in full as long as the reducer + // runs between loads. + auto t = makeTracker(); + + constexpr std::size_t kBatches = 4; + auto const perBatch = Tracker::ringCapacity(); + + for (std::size_t b = 0; b < kBatches; ++b) { - auto const hash = makeHash(static_cast(i)); - tracker_.recordOurValidation(hash, static_cast(i)); - tracker_.recordNetworkValidation(hash, static_cast(i)); + for (std::size_t i = 0; i < perBatch; ++i) + { + auto const n = b * perBatch + i + 1; + t.recordOurValidation(makeHash(n), static_cast(n)); + t.recordNetworkValidation(makeHash(n), static_cast(n)); + } + t.reconcile(); } - // 2 misses: only network validates. - for (int i = 4; i <= 5; ++i) + settle(t); + + auto const expected = kBatches * perBatch; + EXPECT_EQ(t.droppedEvents(), 0u); + EXPECT_EQ(t.agreements1h(), expected); + EXPECT_EQ(t.missed1h(), 0u); + EXPECT_EQ(t.totalAgreements(), expected); + EXPECT_EQ(t.totalValidationsSent(), expected); + EXPECT_EQ(t.totalValidationsChecked(), expected); +} + +// ---- concurrency ----------------------------------------------------------- + +TEST(ValidationTracker, two_producers_and_a_reducer_lose_nothing) +{ + // Real threads on the real code path. Each producer stays under the ring + // capacity between drains, so no drop is expected and every pair must be + // counted exactly once. + gNow = Tracker::TimePoint{} + std::chrono::hours(1000); + Tracker t(&testNow); + + constexpr std::uint64_t kLedgers = 64; + + std::thread ours([&t] { + for (std::uint64_t i = 0; i < kLedgers; ++i) + t.recordOurValidation(makeHash(6000 + i), static_cast(6000 + i)); + }); + std::thread theirs([&t] { + for (std::uint64_t i = 0; i < kLedgers; ++i) + t.recordNetworkValidation(makeHash(6000 + i), static_cast(6000 + i)); + }); + std::thread reader([&t] { + for (int i = 0; i < 200; ++i) + { + t.reconcile(); + static_cast(t.agreements1h()); + } + }); + + ours.join(); + theirs.join(); + reader.join(); + + settle(t); + EXPECT_EQ(t.droppedEvents(), 0u); + EXPECT_EQ(t.agreements1h(), kLedgers); + EXPECT_EQ(t.missed1h(), 0u); +} + +TEST(ValidationTracker, a_reader_runs_alongside_the_reducer_without_racing_it) +{ + // The production shape: one thread reconciles while another only reads the + // gauges. Every event is still counted once, and a reader never stops the + // reducer from finishing a cycle. + gNow = Tracker::TimePoint{} + std::chrono::hours(3000); + Tracker t(&testNow); + + constexpr std::uint64_t kLedgers = 500; + std::atomic stop{false}; + + std::thread reader([&t, &stop] { + std::uint64_t seen = 0; + while (!stop.load(std::memory_order_relaxed)) + seen += t.agreements1h() + t.missed1h() + t.agreements7d(); + static_cast(seen); + }); + + for (std::uint64_t i = 0; i < kLedgers; ++i) { - auto const hash = makeHash(static_cast(i)); - tracker_.recordNetworkValidation(hash, static_cast(i)); + t.recordOurValidation(makeHash(8000 + i), static_cast(8000 + i)); + t.recordNetworkValidation(makeHash(8000 + i), static_cast(8000 + i)); + advance(std::chrono::seconds(9)); + t.reconcile(); } - // Wait for grace period then reconcile. - std::this_thread::sleep_for(std::chrono::seconds(9)); - tracker_.reconcile(); + stop.store(true, std::memory_order_relaxed); + reader.join(); - EXPECT_EQ(tracker_.totalAgreements(), 3u); - EXPECT_EQ(tracker_.totalMissed(), 2u); - EXPECT_EQ(tracker_.agreements1h(), 3u); - EXPECT_EQ(tracker_.missed1h(), 2u); - - // 3 out of 5 = 60% - EXPECT_DOUBLE_EQ(tracker_.agreementPct1h(), 60.0); + EXPECT_EQ(t.droppedEvents(), 0u); + EXPECT_EQ(t.totalAgreements(), kLedgers); + EXPECT_EQ(t.totalMissed(), 0u); + EXPECT_EQ(t.missed1h(), 0u); } -// --------------------------------------------------------------- -// 8. Duplicate recording for same hash -// Recording the same hash multiple times should not create -// duplicate pending entries or double-count totals beyond the -// per-call increments. -// --------------------------------------------------------------- -TEST_F(ValidationTrackerTest, DuplicateRecordingSameHash) +TEST(ValidationTracker, concurrent_reducer_entry_does_not_deadlock_or_double_count) { - auto const hash = makeHash(42); - LedgerIndex const seq = 42; + gNow = Tracker::TimePoint{} + std::chrono::hours(2000); + Tracker t(&testNow); - // Record our validation twice for the same hash. - tracker_.recordOurValidation(hash, seq); - tracker_.recordOurValidation(hash, seq); - tracker_.recordNetworkValidation(hash, seq); - - // Each call increments the lifetime counter. - EXPECT_EQ(tracker_.totalValidationsSent(), 2u); - EXPECT_EQ(tracker_.totalValidationsChecked(), 1u); - - // But only one pending event exists, so only one agreement. - std::this_thread::sleep_for(std::chrono::seconds(9)); - tracker_.reconcile(); - - EXPECT_EQ(tracker_.totalAgreements(), 1u); - EXPECT_EQ(tracker_.totalMissed(), 0u); -} - -// --------------------------------------------------------------- -// 9. Only-we-validated scenario -// We validate but the network does not. After grace period -// this should be a miss (not an agreement). -// --------------------------------------------------------------- -TEST_F(ValidationTrackerTest, OnlyWeValidated) -{ - auto const hash = makeHash(99); - LedgerIndex const seq = 99; - - tracker_.recordOurValidation(hash, seq); - - std::this_thread::sleep_for(std::chrono::seconds(9)); - tracker_.reconcile(); - - EXPECT_EQ(tracker_.totalAgreements(), 0u); - EXPECT_EQ(tracker_.totalMissed(), 1u); - EXPECT_EQ(tracker_.missed1h(), 1u); - EXPECT_DOUBLE_EQ(tracker_.agreementPct1h(), 0.0); -} - -// --------------------------------------------------------------- -// 10. A counted ledger is never counted twice -// reconcile() drops the oldest reconciled events once the -// pending map passes kMaxPendingEvents. A validation arriving -// for one of those ledgers afterwards must not reach the -// agreement or missed totals a second time. -// -// Two hashes are made the oldest so the trim drops both: -// - evictedMiss (network only) reconciles as a miss. Our late -// validation cannot repair an entry the trim dropped, so -// totalMissed staying at 1 proves the trim really dropped -// it. A fixture where the trim did not run would repair it -// and report 0 misses. -// - evictedAgreed (both sides) reconciles as an agreement. -// Re-recording both sides is what double-counts an -// agreement. -// --------------------------------------------------------------- -TEST_F(ValidationTrackerTest, CountedLedgerNotCountedTwice) -{ - // The trim drops the oldest reconciled entries first. Each pause makes - // the next record time strictly larger, so these two are the oldest. - auto const evictedMiss = makeHash(1); - tracker_.recordNetworkValidation(evictedMiss, 1); - std::this_thread::sleep_for(std::chrono::milliseconds(5)); - - auto const evictedAgreed = makeHash(2); - tracker_.recordOurValidation(evictedAgreed, 2); - tracker_.recordNetworkValidation(evictedAgreed, 2); - std::this_thread::sleep_for(std::chrono::milliseconds(5)); - - // Fill to three over the bound so the trim drops three entries: the two - // above plus one filler. - constexpr std::size_t kFill = ValidationTracker::kMaxPendingEvents + 1; - for (std::size_t i = 0; i < kFill; ++i) + for (std::uint64_t i = 0; i < 32; ++i) { - auto const hash = makeHash(i + 3); - auto const seq = static_cast(i + 3); - tracker_.recordOurValidation(hash, seq); - tracker_.recordNetworkValidation(hash, seq); + t.recordOurValidation(makeHash(7000 + i), static_cast(7000 + i)); + t.recordNetworkValidation(makeHash(7000 + i), static_cast(7000 + i)); } + advance(Tracker::gracePeriod() + std::chrono::seconds(1)); - std::this_thread::sleep_for(std::chrono::seconds(9)); - tracker_.reconcile(); + std::vector readers; + for (int i = 0; i < 8; ++i) + readers.emplace_back([&t] { + for (int j = 0; j < 500; ++j) + t.reconcile(); + }); + for (auto& r : readers) + r.join(); - // Every filler plus evictedAgreed agrees; evictedMiss is the one miss. - EXPECT_EQ(tracker_.totalAgreements(), kFill + 1); - EXPECT_EQ(tracker_.totalMissed(), 1u); - EXPECT_EQ(tracker_.agreements1h(), kFill + 1); - EXPECT_EQ(tracker_.missed1h(), 1u); - - // Validations arrive again for the two dropped ledgers. - tracker_.recordOurValidation(evictedMiss, 1); - tracker_.recordOurValidation(evictedAgreed, 2); - tracker_.recordNetworkValidation(evictedAgreed, 2); - - // Long enough for a re-created pending entry to pass the grace period. - std::this_thread::sleep_for(std::chrono::seconds(9)); - tracker_.reconcile(); - - // Both ledgers were already counted, so every total is unchanged. - EXPECT_EQ(tracker_.totalAgreements(), kFill + 1); - EXPECT_EQ(tracker_.totalMissed(), 1u); - EXPECT_EQ(tracker_.agreements1h(), kFill + 1); - EXPECT_EQ(tracker_.missed1h(), 1u); - - // The send and check counters count messages, not ledgers, so the - // repeated validations do count towards them. - EXPECT_EQ(tracker_.totalValidationsSent(), kFill + 3); - EXPECT_EQ(tracker_.totalValidationsChecked(), kFill + 3); + EXPECT_EQ(t.agreements1h(), 32u); + EXPECT_EQ(t.missed1h(), 0u); + EXPECT_EQ(t.totalAgreements(), 32u); } diff --git a/src/xrpld/telemetry/ValidationTracker.h b/src/xrpld/telemetry/ValidationTracker.h index 3c4d26dc6e..30dad9061e 100644 --- a/src/xrpld/telemetry/ValidationTracker.h +++ b/src/xrpld/telemetry/ValidationTracker.h @@ -9,53 +9,50 @@ #include #include +#include #include #include #include #include #include -#include +#include namespace xrpl::telemetry { /** * Tracks whether this validator's validations agree with network consensus, - * maintaining rolling 1-hour and 24-hour windows plus lifetime totals. + * over rolling 1-hour, 24-hour and 7-day windows plus lifetime totals. * - * The tracker operates by recording two independent events per ledger: + * Two independent events are recorded per ledger: * 1. "We validated" -- our node published a validation for a ledger hash. - * 2. "Network validated" -- the network reached consensus on a ledger hash. + * 2. "Network validated" -- the network reached consensus on that hash. * - * After a configurable grace period (kGracePeriod), the reconcile() method - * compares the two flags. If both are set the ledger is counted as an - * "agreement"; otherwise it is a "miss". A late-repair mechanism allows a - * miss to be upgraded to an agreement if matching evidence arrives within - * kLateRepairWindow. + * reconcile() compares the two flags once the grace period has passed. Both + * flags set is an agreement, anything else is a miss. A miss becomes an + * agreement if its other half arrives inside the late-repair window. * - * Architecture / dependency diagram: + * The writer paths take no lock at all: each writer owns one ring, so the two + * never touch shared state. Everything a decision needs belongs to whichever + * thread is inside reconcile(), and a second caller returns instead of waiting. + * Readers take a copy of the snapshot the reducer published. + * + * Data flow: * @code - * +--------------------------+ - * | ConsensusAdapter / | - * | ValidatorSite | - * | (callers) | - * +---+-------------+-------+ - * | | - * | recordOur | recordNetwork - * | Validation | Validation - * v v - * +---------------------------+ - * | ValidationTracker | - * |---------------------------| - * | pending_ (hash_map) |----> LedgerEvent per hash - * | tallied_ (hash_set) |----> hashes already counted - * | window1h_ (deque) |----> WindowEvent sliding window - * | window24h_ (deque) |----> WindowEvent sliding window - * | atomic totals | - * +---------------------------+ - * | - * | reconcile() called periodically - * v - * agreement / miss counters updated + * RCLConsensus::Adaptor::validate LedgerMaster::setValidLedger + * | | + * recordOurValidation() recordNetworkValidation() + * v v + * +------------+ +--------------+ + * | ourRing_ | | networkRing_ | + * +------------+ +--------------+ + * \ / + * \--------- reconcile() ------------/ + * | + * pending_ --> one-minute buckets --> 1h / 24h / 7d counters + * | + * published_ snapshot + * | + * agreementPct1h() / agreements24h() / missed7d() / ... * @endcode * * Usage -- basic recording and querying: @@ -68,7 +65,7 @@ namespace xrpl::telemetry { * // On network consensus: * tracker.recordNetworkValidation(ledgerHash, seq); * - * // Periodically (e.g. every few seconds): + * // Periodically (e.g. every ten seconds): * tracker.reconcile(); * * // Query agreement percentage: @@ -81,16 +78,35 @@ namespace xrpl::telemetry { * * // Network validates first, our validation arrives late: * tracker.recordNetworkValidation(hash, seq); - * tracker.reconcile(); // initially counted as a miss + * tracker.reconcile(); // counted as a miss * - * // Late local validation arrives within repair window: + * // Our validation arrives inside the repair window: * tracker.recordOurValidation(hash, seq); - * tracker.reconcile(); // repaired to agreement + * tracker.reconcile(); // repaired to an agreement * @endcode * - * @note Thread-safety: all public methods are thread-safe. The pending_ - * map and sliding-window deques are protected by mutex_. Lifetime totals - * use std::atomic for lock-free reads. + * Usage -- a test drives the clock so a window edge is reachable at once: + * @code + * // A lambda with no captures converts to the function pointer NowFn wants. + * static xrpl::telemetry::ValidationTracker::TimePoint fakeNow{}; + * xrpl::telemetry::ValidationTracker tracker([] { return fakeNow; }); + * + * tracker.recordOurValidation(hash, seq); + * fakeNow += xrpl::telemetry::ValidationTracker::gracePeriod(); + * tracker.reconcile(); // decides the event without any waiting + * @endcode + * + * @note Thread-safety: every public method may be called concurrently. The + * two record methods each need a single writer thread, which is how consensus + * and the ledger master call them. reconcile() and the getters may be called + * from any thread and any number of threads. + * @note reconcile() and the getters share the published snapshot through an + * atomic shared_ptr, which libstdc++ guards with a short internal spin. No + * writer path touches it, so nothing a consensus thread calls can spin. + * @note A writer whose ring is full discards the event and bumps + * droppedEvents(). Counts are then low but never wrong. + * @note Window edges are rounded to whole minutes, because counts are kept in + * one-minute buckets. */ class ValidationTracker { @@ -106,11 +122,19 @@ public: using TimePoint = Clock::time_point; /** - * Maximum number of pending (unreconciled + recently reconciled) events. - * Once the pending map passes this size, reconcile() drops the oldest - * reconciled events. Public so a test can size a fixture against it. + * Time source. A test supplies its own so it can reach a window edge + * without waiting for one. */ - static constexpr std::size_t kMaxPendingEvents = 1000; + using NowFn = TimePoint (*)(); + + /** + * Construct a tracker reading time from the given source. + * @param now Function returning the current time. Defaults to the + * monotonic clock, so default construction works. + */ + explicit ValidationTracker(NowFn now = &ValidationTracker::steadyNow) : now_(now) + { + } /** * Record that this node sent a validation for the given ledger. @@ -129,10 +153,12 @@ public: recordNetworkValidation(uint256 const& ledgerHash, LedgerIndex seq); /** - * Reconcile pending ledger events whose grace period has elapsed. - * Should be called periodically (e.g. every few seconds). Moves - * reconciled events into the sliding windows and updates totals. - * Also performs late-repair and eviction of stale data. + * Drain both rings, decide every event past the grace period, retire + * expired buckets and publish a fresh snapshot, in that order. + * + * Call periodically, for example every ten seconds. Returns without doing + * anything if another thread is already inside, so a losing caller reads + * data at most one cycle old rather than blocking. */ void reconcile(); @@ -173,37 +199,37 @@ public: /** * Number of agreements in the 1-hour window. */ - [[nodiscard]] uint64_t + [[nodiscard]] std::uint64_t agreements1h() const; /** * Number of misses in the 1-hour window. */ - [[nodiscard]] uint64_t + [[nodiscard]] std::uint64_t missed1h() const; /** * Number of agreements in the 24-hour window. */ - [[nodiscard]] uint64_t + [[nodiscard]] std::uint64_t agreements24h() const; /** * Number of misses in the 24-hour window. */ - [[nodiscard]] uint64_t + [[nodiscard]] std::uint64_t missed24h() const; /** * Number of agreements in the 7-day window. */ - [[nodiscard]] uint64_t + [[nodiscard]] std::uint64_t agreements7d() const; /** * Number of misses in the 7-day window. */ - [[nodiscard]] uint64_t + [[nodiscard]] std::uint64_t missed7d() const; /** @} */ @@ -216,56 +242,81 @@ public: /** * Total agreements since process start. */ - [[nodiscard]] uint64_t + [[nodiscard]] std::uint64_t totalAgreements() const; /** * Total misses since process start. */ - [[nodiscard]] uint64_t + [[nodiscard]] std::uint64_t totalMissed() const; /** * Total validations this node sent. */ - [[nodiscard]] uint64_t + [[nodiscard]] std::uint64_t totalValidationsSent() const; /** * Total network validations observed for comparison. */ - [[nodiscard]] uint64_t + [[nodiscard]] std::uint64_t totalValidationsChecked() const; + /** + * Events a writer discarded because its ring was full. + * @return Lifetime count of discards across both rings. Non-zero means + * reconcile() is not being called often enough. + */ + [[nodiscard]] std::uint64_t + droppedEvents() const; + + /** @} */ + + /** + * @name Bounds, exposed so a test states the same bound as the code + */ + /** @{ */ + + /** + * Slots in each writer's ring. + */ + static constexpr std::size_t + ringCapacity() + { + return kRingCapacity; + } + + /** + * Delay before an event is decided, so both sides can arrive first. + */ + static constexpr std::chrono::seconds + gracePeriod() + { + return kGracePeriod; + } + + /** + * How long after a decision a miss can still become an agreement. + */ + static constexpr std::chrono::minutes + lateRepairWindow() + { + return kLateRepairWindow; + } + /** @} */ private: /** - * Per-ledger tracking state held in the pending map. + * Slots per ring. A power of two, so the index is a mask rather than a + * division. 128 slots is about 8.5 minutes of ledgers at one every four + * seconds, against a normal drain gap of well under a minute. */ - struct LedgerEvent - { - uint256 ledgerHash; ///< Ledger hash being tracked. - LedgerIndex seq{0}; ///< Ledger sequence number. - TimePoint recordTime; ///< Time the event was first recorded. - bool weValidated = false; ///< True if we sent a validation. - bool networkValidated = false; ///< True if network reached consensus. - bool reconciled = false; ///< True once grace period elapsed. - bool agreed = false; ///< True if both flags set at reconcile. - }; + static constexpr std::size_t kRingCapacity = 128; /** - * Lightweight event stored in the sliding-window deques. - */ - struct WindowEvent - { - TimePoint time; ///< When the event was reconciled. - uint256 ledgerHash; ///< Ledger hash for late-repair matching. - bool agreed{false}; ///< Whether this was an agreement. - }; - - /** - * Grace period before reconciling a ledger event. + * Grace period before deciding a ledger event. */ static constexpr auto kGracePeriod = std::chrono::seconds(8); @@ -274,6 +325,22 @@ private: */ static constexpr auto kLateRepairWindow = std::chrono::minutes(5); + /** + * One-minute buckets spanned by the short window. + */ + static constexpr std::size_t kBuckets1h = 60; + + /** + * One-minute buckets spanned by the long window. + */ + static constexpr std::size_t kBuckets24h = 24 * 60; + + /** + * One-minute buckets spanned by the extended window. Also the length of + * the bucket grid, so a slot is reused exactly seven days later. + */ + static constexpr std::size_t kBuckets7d = 7 * 24 * 60; + /** * Maximum number of ledger hashes remembered as already counted. * At one ledger every four seconds this spans about eleven hours. @@ -283,28 +350,281 @@ private: static constexpr std::size_t kMaxTalliedEvents = 10000; /** - * Duration of the short rolling window. + * Default time source. + * @return The monotonic clock's current time point. */ - static constexpr auto kWindow1h = std::chrono::hours(1); + static TimePoint + steadyNow(); /** - * Duration of the long rolling window. + * One recorded event as it travels from a writer to the reducer. */ - static constexpr auto kWindow24h = std::chrono::hours(24); + struct Slot + { + uint256 hash; ///< Ledger hash being reported. + + /** + * Ledger sequence number as the caller gave it. Carried for + * diagnostics: the counters key on the hash, not the sequence. + */ + LedgerIndex seq{0}; + + TimePoint at{}; ///< When the writer recorded it. + }; /** - * Duration of the extended rolling window (7 days). + * Single-producer, single-consumer ring of recorded events. + * + * The producer only advances head_ and the consumer only advances tail_, + * so a release store on one side and an acquire load on the other is + * enough: no compare-exchange, no retry loop, nothing to block on. + * + * @code + * Ring r; + * if (!r.push(hash, seq, now)) + * ; // full, the caller drops the event + * r.drain([](Slot const& s) { use(s); }); + * @endcode + * + * @note Exactly one thread may push and exactly one may drain. Two + * pushers corrupt the ring. */ - static constexpr auto kWindow7d = std::chrono::hours(168); + class Ring + { + public: + /** + * Add one event to the ring. + * @param hash Ledger hash to record. + * @param seq Ledger sequence number. + * @param at Time the producer observed the event. + * @return false when the ring is full, in which case nothing was + * stored and the caller must drop the event. + */ + [[nodiscard]] bool + push(uint256 const& hash, LedgerIndex seq, TimePoint at) + { + auto const head = head_.load(std::memory_order_relaxed); + if (head - tail_.load(std::memory_order_acquire) >= kRingCapacity) + return false; + + slots_[head & (kRingCapacity - 1)] = Slot{hash, seq, at}; + head_.store(head + 1, std::memory_order_release); + return true; + } + + /** + * Hand every stored event to fn, oldest first, and free their slots. + * @param fn Callable taking Slot const&. + */ + template + void + drain(Fn&& fn) + { + auto tail = tail_.load(std::memory_order_relaxed); + auto const head = head_.load(std::memory_order_acquire); + for (; tail != head; ++tail) + fn(slots_[tail & (kRingCapacity - 1)]); + tail_.store(tail, std::memory_order_release); + } + + private: + /** + * Storage, indexed by head_ or tail_ masked to the capacity. + */ + std::array slots_{}; + + /** + * Count of events ever pushed. Only the producer writes it. + */ + std::atomic head_{0}; + + /** + * Count of events ever drained. Only the consumer writes it. + */ + std::atomic tail_{0}; + }; /** - * Protects pending_, tallied_, talliedOrder_, window1h_, window24h_, - * and window7d_. + * Per-ledger tracking state held in the pending map. */ - mutable std::mutex mutex_; + struct LedgerEvent + { + TimePoint recordTime{}; ///< Time the event was first recorded. + std::uint64_t minute{0}; ///< Minute bucket the event belongs to. + bool weValidated{false}; ///< True if we sent a validation. + bool networkValidated{false}; ///< True if network reached consensus. + bool decided{false}; ///< True once the grace period elapsed. + bool agreed{false}; ///< True if both flags were set. + }; /** - * Pending ledger events indexed by ledger hash. + * Counts for one minute of the grid. + */ + struct Bucket + { + std::uint32_t agreed{0}; ///< Agreements decided in this minute. + std::uint32_t total{0}; ///< Events decided in this minute. + }; + + /** + * Running counts for one rolling window. + */ + struct WindowCount + { + std::uint64_t agreed{0}; ///< Agreements still inside the window. + std::uint64_t total{0}; ///< Events still inside the window. + + /** + * Misses still inside the window. + * @return total minus agreed. Derived, so a repair only has to move + * agreed. + */ + [[nodiscard]] std::uint64_t + missed() const + { + return total - agreed; + } + }; + + /** + * The nine numbers a reader wants, published as one value. + */ + struct Snapshot + { + WindowCount w1h; ///< 1-hour window counts. + WindowCount w24h; ///< 24-hour window counts. + WindowCount w7d; ///< 7-day window counts. + }; + + /** + * Convert a time point to its minute on the grid. + * @param t Time point to convert. + * @return Whole minutes since the clock's epoch. + */ + static std::uint64_t + minuteOf(TimePoint t); + + /** + * Agreement percentage for one window. + * @param w Window counts to divide. + * @return Percentage [0.0, 100.0], or 0.0 when the window is empty. + */ + static double + pct(WindowCount const& w); + + /** + * Oldest minute a window of the given length still covers. + * @param minute Newest minute recorded. + * @param span Window length in minutes. + * @return That window's tail minute, floored at zero. + */ + static std::uint64_t + oldestInWindow(std::uint64_t minute, std::size_t span); + + /** + * The snapshot readers are currently seeing. + * @return A copy of the published snapshot, so all nine numbers come from + * one reconcile. All zeroes before the first reconcile() publishes. + */ + [[nodiscard]] Snapshot + read() const; + + /** + * Put the running counters into a fresh snapshot and publish it. + */ + void + publish(); + + /** + * Move both rings' contents into pending_. + */ + void + drainRings(); + + /** + * Fold one drained event into pending_. + * @param s Slot the ring handed over. + * @param ours True if the event came from our own ring. + */ + void + note(Slot const& s, bool ours); + + /** + * Decide every pending event past the grace period, repair the ones whose + * other half arrived late, and drop entries too old to repair. + * @param now Current time point. + */ + void + decidePending(TimePoint now); + + /** + * Remember a ledger hash as counted, dropping the oldest remembered + * hash once kMaxTalliedEvents is reached. + * @param ledgerHash Hash of the ledger just counted into the totals. + */ + void + noteTallied(uint256 const& ledgerHash); + + /** + * Count one decided event in its own bucket and in all three windows. + * @param minute Bucket the event belongs to. + * @param agreed True to count it as an agreement. + */ + void + addToWindows(std::uint64_t minute, bool agreed); + + /** + * Turn one already-counted event from a miss into an agreement. + * @param minute Bucket the event was counted in. + */ + void + repairInWindows(std::uint64_t minute); + + /** + * Move each window's tail up to the given minute, subtracting whatever + * leaves. The 7-day tail also clears the bucket it passes, because that + * slot is about to be reused. + * @param minute Newest minute to account for. + */ + void + advanceWindows(std::uint64_t minute); + + /** + * Walk one window's tail forward, taking each passed bucket back out of + * that window's running counts. + * @param tail The window's tail minute, advanced in place. + * @param target Minute to stop at, the oldest the window still covers. + * @param count The window's running counts to subtract from. + * @param clear True to zero each passed bucket, which only the 7-day + * tail does because it is the tail whose slot gets reused. + */ + void + retireWindow(std::uint64_t& tail, std::uint64_t target, WindowCount& count, bool clear); + + /** + * Time source, read on every write and by the reducer. + */ + NowFn now_; + + /** + * Events from our own validations. Pushed by the consensus thread. + */ + Ring ourRing_; + + /** + * Events from network consensus. Pushed by the ledger master thread. + */ + Ring networkRing_; + + /** + * Set while a thread is inside reconcile(). A second caller sees it set + * and returns rather than waiting. + */ + std::atomic_flag reducing_{}; + + /** + * Pending ledger events indexed by ledger hash. Touched only inside + * reconcile(), so it needs no synchronisation. */ hash_map pending_; @@ -322,84 +642,82 @@ private: std::deque talliedOrder_; /** - * Sliding window of reconciled events (last 1 hour). + * One-minute counts, indexed by minute modulo kBuckets7d. */ - std::deque window1h_; + std::array buckets_{}; /** - * Sliding window of reconciled events (last 24 hours). + * Running counts for the 1-hour window. */ - std::deque window24h_; + WindowCount c1h_; /** - * Sliding window of reconciled events (last 7 days). + * Running counts for the 24-hour window. */ - std::deque window7d_; + WindowCount c24h_; + + /** + * Running counts for the 7-day window. + */ + WindowCount c7d_; + + /** + * Oldest minute the 1-hour window still counts. + */ + std::uint64_t tail1h_{0}; + + /** + * Oldest minute the 24-hour window still counts. + */ + std::uint64_t tail24h_{0}; + + /** + * Oldest minute the 7-day window still counts. + */ + std::uint64_t tail7d_{0}; + + /** + * Newest minute written to the grid. + */ + std::uint64_t newestMinute_{0}; + + /** + * False until the first minute is recorded, which is when the tails and + * newestMinute_ get their starting value. + */ + bool started_{false}; + + /** + * The snapshot readers see. The reducer swaps in a new one each cycle, and + * a reader that took the old one keeps it alive while it reads. Null until + * the first reconcile(). + */ + std::atomic> published_; /** * Lifetime count of agreements. */ - std::atomic totalAgreements_{0}; + std::atomic totalAgreements_{0}; /** * Lifetime count of misses. */ - std::atomic totalMissed_{0}; + std::atomic totalMissed_{0}; /** * Lifetime count of validations this node sent. */ - std::atomic totalValidationsSent_{0}; + std::atomic totalValidationsSent_{0}; /** * Lifetime count of network validations observed. */ - std::atomic totalValidationsChecked_{0}; + std::atomic totalValidationsChecked_{0}; /** - * Locate the pending event for a ledger, creating it on first sight. - * @param ledgerHash Hash of the ledger being recorded. - * @param seq Ledger sequence number, stored only on creation. - * @return Pointer to the event, or nullptr for a ledger that already - * reached the totals and left pending_. The caller records nothing in - * that case. - * @note Called with mutex_ held. + * Lifetime count of events dropped by a full ring. */ - [[nodiscard]] LedgerEvent* - pendingEvent(uint256 const& ledgerHash, LedgerIndex seq); - - /** - * Remember a ledger hash as counted, dropping the oldest remembered - * hash once kMaxTalliedEvents is reached. - * @param ledgerHash Hash of the ledger just counted into the totals. - * @note Called with mutex_ held. - */ - void - noteTallied(uint256 const& ledgerHash); - - /** - * Remove entries older than their respective window durations. - * @param now Current time point. - */ - void - evictStaleWindows(TimePoint now); - - /** - * Remove reconciled pending entries older than the late-repair window. - * Also trims the map if it exceeds kMaxPendingEvents. - * @param now Current time point. - */ - void - evictOldPending(TimePoint now); - - /** - * Scan a window deque and flip the first non-agreed entry matching - * the given ledger hash to agreed. - * @param window The sliding-window deque to repair. - * @param hash Ledger hash to match. - */ - static void - repairWindowEntry(std::deque& window, uint256 const& hash); + std::atomic droppedEvents_{0}; }; } // namespace xrpl::telemetry diff --git a/src/xrpld/telemetry/detail/ValidationTracker.cpp b/src/xrpld/telemetry/detail/ValidationTracker.cpp index 1609cf696c..297a6e88b8 100644 --- a/src/xrpld/telemetry/detail/ValidationTracker.cpp +++ b/src/xrpld/telemetry/detail/ValidationTracker.cpp @@ -8,31 +8,122 @@ #include #include -#include #include +#include +#include #include -#include -#include -#include +#include +#include namespace xrpl::telemetry { -ValidationTracker::LedgerEvent* -ValidationTracker::pendingEvent(uint256 const& ledgerHash, LedgerIndex seq) +ValidationTracker::TimePoint +ValidationTracker::steadyNow() { - if (auto const it = pending_.find(ledgerHash); it != pending_.end()) - return &it->second; + return Clock::now(); +} - // A hash in tallied_ already reached the totals and left pending_. - // Building a fresh record for it would count the same ledger twice. - if (tallied_.contains(ledgerHash)) - return nullptr; +// ---- writer side, called from consensus and ledger threads ----------------- - auto& evt = pending_[ledgerHash]; - evt.ledgerHash = ledgerHash; - evt.seq = seq; - evt.recordTime = Clock::now(); - return &evt; +void +ValidationTracker::recordOurValidation(uint256 const& ledgerHash, LedgerIndex seq) +{ + totalValidationsSent_.fetch_add(1, std::memory_order_relaxed); + + // The producer stamps the time. The grace period is measured from when the + // event happened, so stamping at drain time would smear it by a cycle. + if (!ourRing_.push(ledgerHash, seq, now_())) + droppedEvents_.fetch_add(1, std::memory_order_relaxed); +} + +void +ValidationTracker::recordNetworkValidation(uint256 const& ledgerHash, LedgerIndex seq) +{ + totalValidationsChecked_.fetch_add(1, std::memory_order_relaxed); + + if (!networkRing_.push(ledgerHash, seq, now_())) + droppedEvents_.fetch_add(1, std::memory_order_relaxed); +} + +// ---- reducer, called from the metrics reader ------------------------------- + +void +ValidationTracker::reconcile() +{ + if (reducing_.test_and_set(std::memory_order_acquire)) + return; + + auto const now = now_(); + drainRings(); + decidePending(now); + advanceWindows(minuteOf(now)); + publish(); + + reducing_.clear(std::memory_order_release); +} + +void +ValidationTracker::drainRings() +{ + ourRing_.drain([this](Slot const& s) { note(s, true); }); + networkRing_.drain([this](Slot const& s) { note(s, false); }); +} + +void +ValidationTracker::note(Slot const& s, bool ours) +{ + auto const it = pending_.find(s.hash); + if (it == pending_.end()) + { + // A hash already counted must not open a fresh entry, or the same + // ledger reaches the totals twice. + if (tallied_.contains(s.hash)) + return; + + auto& evt = pending_[s.hash]; + evt.recordTime = s.at; + evt.minute = minuteOf(s.at); + (ours ? evt.weValidated : evt.networkValidated) = true; + return; + } + + (ours ? it->second.weValidated : it->second.networkValidated) = true; +} + +void +ValidationTracker::decidePending(TimePoint now) +{ + for (auto& [hash, evt] : pending_) + { + if (!evt.decided) + { + if (now - evt.recordTime < kGracePeriod) + continue; + + evt.decided = true; + evt.agreed = evt.weValidated && evt.networkValidated; + noteTallied(hash); + addToWindows(evt.minute, evt.agreed); + (evt.agreed ? totalAgreements_ : totalMissed_).fetch_add(1, std::memory_order_relaxed); + } + else if ( + !evt.agreed && evt.weValidated && evt.networkValidated && + now - evt.recordTime <= kLateRepairWindow) + { + // A miss whose other half arrived late. Only `agreed` moves; the + // total was already counted in this event's bucket. + evt.agreed = true; + repairInWindows(evt.minute); + totalMissed_.fetch_sub(1, std::memory_order_relaxed); + totalAgreements_.fetch_add(1, std::memory_order_relaxed); + } + } + + // Nothing can be repaired past the window, so the entry is dead weight. + auto const cutoff = now - kLateRepairWindow; + for (auto it = pending_.begin(); it != pending_.end();) + it = (it->second.decided && it->second.recordTime < cutoff) ? pending_.erase(it) + : std::next(it); } void @@ -49,243 +140,216 @@ ValidationTracker::noteTallied(uint256 const& ledgerHash) } } -void -ValidationTracker::recordOurValidation(uint256 const& ledgerHash, LedgerIndex seq) -{ - std::scoped_lock const lock(mutex_); - totalValidationsSent_.fetch_add(1, std::memory_order_relaxed); - - // The counter above counts messages, so it also counts a ledger that is - // already tallied. Only the per-ledger record is skipped. - if (auto* const evt = pendingEvent(ledgerHash, seq)) - evt->weValidated = true; -} +// ---- buckets and window counters ------------------------------------------ void -ValidationTracker::recordNetworkValidation(uint256 const& ledgerHash, LedgerIndex seq) +ValidationTracker::addToWindows(std::uint64_t minute, bool agreed) { - std::scoped_lock const lock(mutex_); - totalValidationsChecked_.fetch_add(1, std::memory_order_relaxed); + advanceWindows(minute); - if (auto* const evt = pendingEvent(ledgerHash, seq)) - evt->networkValidated = true; -} + auto& b = buckets_[minute % kBuckets7d]; + ++b.total; + if (agreed) + ++b.agreed; -void -ValidationTracker::reconcile() -{ - std::scoped_lock const lock(mutex_); - auto const now = Clock::now(); - - for (auto& [hash, evt] : pending_) + for (auto* w : {&c1h_, &c24h_, &c7d_}) { - if (!evt.reconciled && (now - evt.recordTime) >= kGracePeriod) - { - // Initial reconciliation after grace period. - evt.reconciled = true; - evt.agreed = evt.weValidated && evt.networkValidated; - noteTallied(hash); + ++w->total; + if (agreed) + ++w->agreed; + } +} - if (evt.agreed) - { - totalAgreements_.fetch_add(1, std::memory_order_relaxed); - } - else - { - totalMissed_.fetch_add(1, std::memory_order_relaxed); - } +void +ValidationTracker::repairInWindows(std::uint64_t minute) +{ + // No window tail check is needed: the repair window is shorter than the + // shortest window, so a repairable event is still inside all three. + static_assert( + kLateRepairWindow < std::chrono::minutes(kBuckets1h), + "a repairable event must still be inside the shortest window"); - WindowEvent const we{.time = now, .ledgerHash = evt.ledgerHash, .agreed = evt.agreed}; - window1h_.push_back(we); - window24h_.push_back(we); - window7d_.push_back(we); - } - else if ( - evt.reconciled && !evt.agreed && evt.weValidated && evt.networkValidated && - (now - evt.recordTime) <= kLateRepairWindow) - { - // Late repair: was a miss, now both flags set. - evt.agreed = true; - totalMissed_.fetch_sub(1, std::memory_order_relaxed); - totalAgreements_.fetch_add(1, std::memory_order_relaxed); + ++buckets_[minute % kBuckets7d].agreed; + ++c1h_.agreed; + ++c24h_.agreed; + ++c7d_.agreed; +} - // Flip the corresponding window entries from miss to agreement. - repairWindowEntry(window1h_, evt.ledgerHash); - repairWindowEntry(window24h_, evt.ledgerHash); - repairWindowEntry(window7d_, evt.ledgerHash); - } +void +ValidationTracker::advanceWindows(std::uint64_t minute) +{ + if (!started_) + { + started_ = true; + tail1h_ = tail24h_ = tail7d_ = minute; + newestMinute_ = minute; + return; } - evictStaleWindows(now); - evictOldPending(now); -} + if (minute <= newestMinute_) + return; -void -ValidationTracker::evictStaleWindows(TimePoint now) -{ - auto const cutoff1h = now - kWindow1h; - while (!window1h_.empty() && window1h_.front().time < cutoff1h) - window1h_.pop_front(); + auto const previousNewest = newestMinute_; + newestMinute_ = minute; - auto const cutoff24h = now - kWindow24h; - while (!window24h_.empty() && window24h_.front().time < cutoff24h) - window24h_.pop_front(); - - auto const cutoff7d = now - kWindow7d; - while (!window7d_.empty() && window7d_.front().time < cutoff7d) - window7d_.pop_front(); -} - -void -ValidationTracker::evictOldPending(TimePoint now) -{ - auto const cutoff = now - kLateRepairWindow; - std::erase_if(pending_, [cutoff](auto const& entry) { - return entry.second.reconciled && entry.second.recordTime < cutoff; - }); - - // Hard trim if still over limit. The pass above already removed every - // reconciled entry older than the late-repair window, so every candidate - // here is still repairable. Drop the oldest first: it has the least repair - // time left, so it loses the least. pending_ is unordered, so the oldest - // has to be searched for rather than found at an end. - while (pending_.size() > kMaxPendingEvents) + // Nothing was recorded for longer than the whole grid, so every bucket + // still holding a count is stale. Clear them together instead of one at a + // time. The comparison is against the last minute WRITTEN, not against a + // window tail: under steady traffic a tail always sits exactly one + // grid-length back, and comparing to it would wipe live data every minute + // once the grid fills. + if (minute - previousNewest >= kBuckets7d) { - auto oldest = pending_.end(); - for (auto it = pending_.begin(); it != pending_.end(); ++it) - { - if (!it->second.reconciled) - continue; - if (oldest == pending_.end() || it->second.recordTime < oldest->second.recordTime) - oldest = it; - } - - // Only unreconciled entries left. Dropping one would lose its ledger - // from the totals entirely, so the bound gives way instead. - if (oldest == pending_.end()) - break; - - pending_.erase(oldest); + buckets_.fill(Bucket{}); + c1h_ = c24h_ = c7d_ = WindowCount{}; + tail1h_ = tail24h_ = tail7d_ = minute; + return; } + + retireWindow(tail1h_, oldestInWindow(minute, kBuckets1h), c1h_, false); + retireWindow(tail24h_, oldestInWindow(minute, kBuckets24h), c24h_, false); + retireWindow(tail7d_, oldestInWindow(minute, kBuckets7d), c7d_, true); +} + +void +ValidationTracker::retireWindow( + std::uint64_t& tail, + std::uint64_t target, + WindowCount& count, + bool clear) +{ + while (tail < target) + { + auto& b = buckets_[tail % kBuckets7d]; + count.total -= b.total; + count.agreed -= b.agreed; + if (clear) + b = Bucket{}; + ++tail; + } +} + +std::uint64_t +ValidationTracker::oldestInWindow(std::uint64_t minute, std::size_t span) +{ + return minute + 1 >= span ? minute + 1 - span : 0; +} + +std::uint64_t +ValidationTracker::minuteOf(TimePoint t) +{ + return static_cast( + std::chrono::duration_cast(t.time_since_epoch()).count()); +} + +// ---- snapshot publication and reads --------------------------------------- + +void +ValidationTracker::publish() +{ + published_.store( + std::make_shared(Snapshot{c1h_, c24h_, c7d_}), std::memory_order_release); +} + +ValidationTracker::Snapshot +ValidationTracker::read() const +{ + // Holding the shared_ptr keeps this snapshot alive for as long as the + // caller needs it, so the reducer can never write the values being read. + auto const s = published_.load(std::memory_order_acquire); + return s ? *s : Snapshot{}; +} + +double +ValidationTracker::pct(WindowCount const& w) +{ + if (w.total == 0) + return 0.0; + return (static_cast(w.agreed) / static_cast(w.total)) * 100.0; } double ValidationTracker::agreementPct1h() const { - std::scoped_lock const lock(mutex_); - if (window1h_.empty()) - return 0.0; - auto const agreed = static_cast( - std::ranges::count_if(window1h_, [](auto const& e) { return e.agreed; })); - return (agreed / static_cast(window1h_.size())) * 100.0; + return pct(read().w1h); } double ValidationTracker::agreementPct24h() const { - std::scoped_lock const lock(mutex_); - if (window24h_.empty()) - return 0.0; - auto const agreed = static_cast( - std::ranges::count_if(window24h_, [](auto const& e) { return e.agreed; })); - return (agreed / static_cast(window24h_.size())) * 100.0; -} - -uint64_t -ValidationTracker::agreements1h() const -{ - std::scoped_lock const lock(mutex_); - return static_cast( - std::ranges::count_if(window1h_, [](auto const& e) { return e.agreed; })); -} - -uint64_t -ValidationTracker::missed1h() const -{ - std::scoped_lock const lock(mutex_); - return static_cast( - std::ranges::count_if(window1h_, [](auto const& e) { return !e.agreed; })); -} - -uint64_t -ValidationTracker::agreements24h() const -{ - std::scoped_lock const lock(mutex_); - return static_cast( - std::ranges::count_if(window24h_, [](auto const& e) { return e.agreed; })); -} - -uint64_t -ValidationTracker::missed24h() const -{ - std::scoped_lock const lock(mutex_); - return static_cast( - std::ranges::count_if(window24h_, [](auto const& e) { return !e.agreed; })); + return pct(read().w24h); } double ValidationTracker::agreementPct7d() const { - std::scoped_lock const lock(mutex_); - if (window7d_.empty()) - return 0.0; - auto const agreed = static_cast( - std::ranges::count_if(window7d_, [](auto const& e) { return e.agreed; })); - return (agreed / static_cast(window7d_.size())) * 100.0; + return pct(read().w7d); } -uint64_t +std::uint64_t +ValidationTracker::agreements1h() const +{ + return read().w1h.agreed; +} + +std::uint64_t +ValidationTracker::missed1h() const +{ + return read().w1h.missed(); +} + +std::uint64_t +ValidationTracker::agreements24h() const +{ + return read().w24h.agreed; +} + +std::uint64_t +ValidationTracker::missed24h() const +{ + return read().w24h.missed(); +} + +std::uint64_t ValidationTracker::agreements7d() const { - std::scoped_lock const lock(mutex_); - return static_cast( - std::ranges::count_if(window7d_, [](auto const& e) { return e.agreed; })); + return read().w7d.agreed; } -uint64_t +std::uint64_t ValidationTracker::missed7d() const { - std::scoped_lock const lock(mutex_); - return static_cast( - std::ranges::count_if(window7d_, [](auto const& e) { return !e.agreed; })); + return read().w7d.missed(); } -uint64_t +std::uint64_t ValidationTracker::totalAgreements() const { return totalAgreements_.load(std::memory_order_relaxed); } -uint64_t +std::uint64_t ValidationTracker::totalMissed() const { return totalMissed_.load(std::memory_order_relaxed); } -uint64_t +std::uint64_t ValidationTracker::totalValidationsSent() const { return totalValidationsSent_.load(std::memory_order_relaxed); } -uint64_t +std::uint64_t ValidationTracker::totalValidationsChecked() const { return totalValidationsChecked_.load(std::memory_order_relaxed); } -void -ValidationTracker::repairWindowEntry(std::deque& window, uint256 const& hash) +std::uint64_t +ValidationTracker::droppedEvents() const { - // Scan backwards since late repairs target recently added entries. - for (auto& event : std::views::reverse(window)) - { - if (!event.agreed && event.ledgerHash == hash) - { - event.agreed = true; - return; - } - } + return droppedEvents_.load(std::memory_order_relaxed); } } // namespace xrpl::telemetry From ea4f7fd0d43494c150b282a37a7ff62ad1b77964 Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Wed, 9 Sep 2026 15:11:25 +0100 Subject: [PATCH 12/16] fix(telemetry): restore health_check and batch the metrics pipeline The health_check extension was present on the previous branch and dropped here with no replacement, while this branch's own TESTING.md still polls http://localhost:13133/ to decide the collector is ready. That check has had no listener since, so the documented readiness step cannot pass. Also add batch to the metrics pipeline. Without it the OTLP metric path exports one request per instrument; the added delay is bounded by the batch timeout, well under the Prometheus scrape interval. Both belong here rather than downstream: this branch owns the metrics pipeline and is the one that regressed the extension. --- docker/telemetry/otel-collector-config.yaml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docker/telemetry/otel-collector-config.yaml b/docker/telemetry/otel-collector-config.yaml index 5f23e3c932..e3429c676a 100644 --- a/docker/telemetry/otel-collector-config.yaml +++ b/docker/telemetry/otel-collector-config.yaml @@ -121,7 +121,12 @@ exporters: resource_to_telemetry_conversion: enabled: true +extensions: + health_check: + endpoint: 0.0.0.0:13133 + service: + extensions: [health_check] pipelines: traces: receivers: [otlp] @@ -129,5 +134,8 @@ service: exporters: [debug, otlp/tempo, spanmetrics] metrics: receivers: [otlp, spanmetrics] - processors: [resource/tier, resource/stripsdk] + # batch keeps the OTLP metric path from exporting one request per + # instrument. It delays a sample by at most the batch timeout, which + # is well under the Prometheus scrape interval. + processors: [resource/tier, resource/stripsdk, batch] exporters: [prometheus] From cbb85819975a5ebee439403d26b44a5835827433 Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Wed, 9 Sep 2026 15:11:49 +0100 Subject: [PATCH 13/16] fix(telemetry): make the log pipeline actually deliver, and fix its docs Addresses the open review findings on this branch. The log root was never delivered at all. Docker creates a missing bind-mount source as root, Config::getDebugLogFile() only warns when it cannot create the network subdirectory inside it, and Application carries on. The node therefore looked healthy while writing no debug.log, and Loki stayed empty with no error at any layer. docker/telemetry/data/logs has in fact been root-owned in a working checkout since it was first created. A one-shot xrpld-logdir-init service now creates the directory and hands it to XRPLD_UID/XRPLD_GID, following the pattern the storage-init service already uses. Ingested logs carried no node identity, so a multi-node stack collapsed into one indistinguishable stream while every dashboard filters on service_instance_id. The receiver now sets include_file_path and lifts the per-node directory onto the resource attribute service.instance.id, which is on the allow-list Loki promotes to an indexed stream label. A record attribute would only become structured metadata and could not be used in a selector. For that to join anything the directory name has to equal the emitter's service_instance_id, so the node directories are renamed to match: node$i becomes Node-$i, and the standalone config writes to logs/xrpld-standalone. The integration test aborted before reporting. Under set -o pipefail the grep | head -1 pipeline is killed by SIGPIPE once the log exceeds the pipe buffer, so the run exited 141 somewhere past a few hundred matching lines and read as a flaky test. grep -m1 stops on its own. The test also verified the local file and Tempo but never that a line reached Loki, which is the one hop this branch adds, so a bounded Loki assertion is added alongside a readiness wait. Documentation fixes: the Tempo cross-check counted .data, but Tempo returns OTLP shape so the array is batches and one trace can span several; the Loki step used the instant /query endpoint, which rejects a bare log selector with HTTP 400 and a text/plain body, so jq could never parse it and the step never printed a number even when ingestion worked. The filelog comment claimed six fractional digits where the node always emits nine. The two flowcharts used
, carried no legend, and advertised GetSpan(), which Log.cpp deliberately avoids in favour of reading the thread-local context directly. Finally, rename the deprecated collector component names: the pinned collector warns on every start that otlphttp and filelog are aliases for otlp_http and file_log. Alloy's otelcol.exporter.otlphttp and otelcol.receiver.filelog are that product's own component names and are not deprecated, so they are left alone. --- OpenTelemetryPlan/06-implementation-phases.md | 78 ++++++++++++++----- .../09-data-collection-reference.md | 8 +- docker/telemetry/TESTING.md | 51 ++++++++---- docker/telemetry/docker-compose.yml | 47 ++++++++--- .../provisioning/datasources/loki.yaml | 2 +- docker/telemetry/integration-test.sh | 77 ++++++++++++++++-- docker/telemetry/otel-collector-config.yaml | 56 +++++++++---- .../telemetry/otel-collector-filestorage.yaml | 10 +-- docker/telemetry/xrpld-telemetry.cfg | 11 ++- docs/telemetry-runbook.md | 12 +-- 10 files changed, 270 insertions(+), 82 deletions(-) diff --git a/OpenTelemetryPlan/06-implementation-phases.md b/OpenTelemetryPlan/06-implementation-phases.md index 080421a5c6..2a0d330a71 100644 --- a/OpenTelemetryPlan/06-implementation-phases.md +++ b/OpenTelemetryPlan/06-implementation-phases.md @@ -574,14 +574,14 @@ See [Phase7_taskList.md](./Phase7_taskList.md) for detailed per-task breakdown. ### Motivation -xrpld's `beast::Journal` logs and OpenTelemetry traces are currently two disjoint observability signals. When investigating an issue, operators must manually correlate timestamps between log files and Jaeger/Tempo traces. Phase 8 bridges this gap by injecting trace context (`trace_id`, `span_id`) into every log line emitted within an active, sampled span, and ingesting those logs into Grafana Loki via the OTel Collector's filelog receiver. +xrpld's `beast::Journal` logs and OpenTelemetry traces are currently two disjoint observability signals. When investigating an issue, operators must manually correlate timestamps between log files and Jaeger/Tempo traces. Phase 8 bridges this gap by injecting trace context (`trace_id`, `span_id`) into every log line emitted within an active, sampled span, and ingesting those logs into Grafana Loki via the OTel Collector's file_log receiver. #### Gains 1. **One-click trace-to-log navigation** — Click a trace in Tempo/Jaeger and immediately see the corresponding log lines in Loki, filtered by `trace_id`. 2. **Reverse lookup (log-to-trace)** — Loki derived fields make `trace_id` values clickable links back to Tempo. 3. **Unified observability** — All three pillars (traces, metrics, logs) flow through the same OTel Collector pipeline and are visible in a single Grafana instance. -4. **Zero new dependencies in xrpld** — Uses existing OTel SDK headers (`GetSpan`, `GetContext`) already linked in Phase 1. +4. **Zero new dependencies in xrpld** — Uses existing OTel SDK headers (`RuntimeContext`, `SpanContext`) already linked in Phase 1. 5. **Negligible overhead** — The implementation checks the thread-local context value directly, avoiding heap allocation on the no-span path (~15-20ns). On the active-span path, total cost is ~50ns per log call. At typical logging rates, overhead is negligible. #### Losses / Risks @@ -592,33 +592,54 @@ xrpld's `beast::Journal` logs and OpenTelemetry traces are currently two disjoin #### Decision -The correlation value far outweighs the risks. The log format change is backward-compatible (fields are appended only when a sampled span is active), and the filelog receiver regex is straightforward to maintain. +The correlation value far outweighs the risks. The log format change is backward-compatible (fields are appended only when a sampled span is active), and the file_log receiver regex is straightforward to maintain. ### Architecture Phase 8 has two independent sub-phases that can be developed in parallel: - **Phase 8a (code change)**: Modify `Logs::format()` in `src/libxrpl/basics/Log.cpp` to append `trace_id= span_id=` when the current thread has an active OTel span. Guarded by `#ifdef XRPL_ENABLE_TELEMETRY`. -- **Phase 8b (infra only)**: Add Loki to the Docker Compose stack, configure the OTel Collector's `filelog` receiver to tail xrpld's log file, parse out structured fields (timestamp, partition, severity, trace_id, span_id, message), and export to Loki via OTLP. Configure Grafana Tempo↔Loki bidirectional linking. +- **Phase 8b (infra only)**: Add Loki to the Docker Compose stack, configure the OTel Collector's `file_log` receiver to tail xrpld's log file, parse out structured fields (timestamp, partition, severity, trace_id, span_id, message), and export to Loki via OTLP. Configure Grafana Tempo↔Loki bidirectional linking. #### Trace ID Injection Flow ```mermaid flowchart LR subgraph xrpld["xrpld process"] - JLOG["JLOG(j.info())"] - Format["Logs::format()"] - OTelCtx["OTel Context
(thread-local)"] + JLOG["`**JLOG(j.info())** + a log call on some thread`"] + Format["`**Logs::format()** + builds the log line`"] + OTelCtx["`**OTel thread-local context** + RuntimeContext::GetCurrent() + GetValue(kSpanKey)`"] JLOG --> Format - OTelCtx -.->|"GetSpan()→GetContext()"| Format + OTelCtx -.->|"`GetContext() + if IsValid and IsSampled`"| Format end - subgraph output["Log Output"] - LogLine["2024-01-15T10:30:45.123Z
LedgerMaster:NFO
trace_id=abc123...
span_id=def456...
Validated ledger 42"] + subgraph output["Log output"] + LogLine["`2026-Jan-15 10:30:45.123456789 UTC + LedgerMaster:NFO + trace_id=abc123... span_id=def456... + Validated ledger 42`"] end Format --> LogLine + subgraph legend["Reading the diagram"] + direction LR + L1["`**Solid arrow** + happens on every log call`"] + L2["`**Dotted arrow** + only adds ids when a sampled span is active on this thread`"] + L3["`**kSpanKey lookup** + reads the context value directly, so the no-span path allocates nothing`"] + end + + L1 ~~~ L2 ~~~ L3 + output ~~~ legend + style xrpld fill:#1a237e,stroke:#0d1642,color:#fff style output fill:#1b5e20,stroke:#0d3d14,color:#fff style JLOG fill:#283593,stroke:#1a237e,color:#fff @@ -632,16 +653,35 @@ flowchart LR ```mermaid flowchart LR subgraph collector["OTel Collector"] - FR["filelog receiver
tails debug.log"] - RP["regex_parser
extracts trace_id,
span_id, severity"] - BP["batch processor"] - LE["otlp/loki exporter"] + FR["`**file_log receiver** + tails debug.log`"] + RP["`**regex_parser** + extracts timestamp, partition, + severity, trace_id, span_id`"] + BP["`**batch processor**`"] + LE["`**otlp_http/loki exporter**`"] FR --> RP --> BP --> LE end - LogFile["xrpld
debug.log"] --> FR - LE --> Loki["Grafana Loki
:3100"] - Loki <-->|"derivedFields ↔
tracesToLogs"| Tempo["Grafana Tempo"] + LogFile["`**xrpld** + debug.log`"] --> FR + LE --> Loki["`**Grafana Loki** + :3100`"] + Loki <-->|"`derivedFields + tracesToLogs`"| Tempo["`**Grafana Tempo**`"] + + subgraph legend["Reading the diagram"] + direction LR + L1["`**Solid arrow** + the path every log line takes`"] + L2["`**Double arrow** + Grafana links the two backends both ways: a trace jumps to its logs, a trace_id in a log jumps back to the trace`"] + L3["`**otlp_http, not otlp** + Loki is reached over OTLP/HTTP; the old dedicated loki exporter was removed upstream`"] + end + + L1 ~~~ L2 ~~~ L3 + collector ~~~ legend style collector fill:#e65100,stroke:#bf360c,color:#fff style FR fill:#f57c00,stroke:#e65100,color:#fff @@ -659,7 +699,7 @@ flowchart LR | ---- | ---------------------------------------------- | | 8.1 | Inject trace_id into Logs::format() | | 8.2 | Add Loki to Docker Compose stack | -| 8.3 | Add filelog receiver to OTel Collector | +| 8.3 | Add file_log receiver to OTel Collector | | 8.4 | Configure Grafana trace-to-log correlation | | 8.5 | Update integration tests | | 8.6 | Update documentation (runbook, reference docs) | @@ -670,7 +710,7 @@ flowchart LR - [ ] Log lines within active spans contain `trace_id= span_id=` - [ ] Log lines outside spans have no trace context (no empty fields) -- [ ] Loki ingests xrpld logs via OTel Collector filelog receiver +- [ ] Loki ingests xrpld logs via OTel Collector file_log receiver - [ ] Grafana Tempo → Loki one-click correlation works - [ ] Grafana Loki → Tempo reverse lookup works via derived field - [ ] Integration test verifies trace_id presence in logs diff --git a/OpenTelemetryPlan/09-data-collection-reference.md b/OpenTelemetryPlan/09-data-collection-reference.md index 0d67842bee..d2263d9e4e 100644 --- a/OpenTelemetryPlan/09-data-collection-reference.md +++ b/OpenTelemetryPlan/09-data-collection-reference.md @@ -956,7 +956,7 @@ Phase 8 injects OTel trace context into xrpld's `Logs::format()` output, enablin Example: ``` -2024-Jan-15 10:30:45.123456 UTC LedgerMaster:NFO trace_id=abc123def456789012345678abcdef01 span_id=0123456789abcdef Validated ledger 42 +2024-Jan-15 10:30:45.123456789 UTC LedgerMaster:NFO trace_id=abc123def456789012345678abcdef01 span_id=0123456789abcdef Validated ledger 42 ``` - **`trace_id=`** — 32-character lowercase hex trace identifier. Links to the distributed trace in Tempo/Jaeger. @@ -970,10 +970,10 @@ The trace context injection is implemented in `Logs::format()` (`src/libxrpl/bas ### Log Ingestion Pipeline ``` -xrpld debug.log -> OTel Collector filelog receiver -> regex_parser -> Loki exporter -> Grafana Loki +xrpld debug.log -> OTel Collector file_log receiver -> regex_parser -> Loki exporter -> Grafana Loki ``` -The OTel Collector's `filelog` receiver tails `debug.log` files and uses a `regex_parser` operator to extract structured fields: +The OTel Collector's `file_log` receiver tails `debug.log` files and uses a `regex_parser` operator to extract structured fields: | Field | Type | Description | | ----------- | -------- | -------------------------------------------------------- | @@ -993,7 +993,7 @@ Bidirectional linking between logs and traces is configured via Grafana datasour ### Loki Backend -Grafana Loki (v3.4.2) serves as the log storage backend. It receives log entries from the OTel Collector's `otlphttp/loki` exporter via the native OTLP endpoint at `http://loki:3100/otlp`. +Grafana Loki (v3.4.2) serves as the log storage backend. It receives log entries from the OTel Collector's `otlp_http/loki` exporter via the native OTLP endpoint at `http://loki:3100/otlp`. ### LogQL Query Examples diff --git a/docker/telemetry/TESTING.md b/docker/telemetry/TESTING.md index 01a3fc90cb..d3576562b4 100644 --- a/docker/telemetry/TESTING.md +++ b/docker/telemetry/TESTING.md @@ -45,6 +45,15 @@ the end of this test for which do and which do not. docker compose -f docker/telemetry/docker-compose.yml up -d ``` +The `xrpld-logdir-init` service creates `docker/telemetry/data/logs` and gives it +to uid/gid 1000. If `id -u` on this host is not 1000, xrpld cannot write its log +there and the log pipeline stays empty, so set the ids first: + +```bash +XRPLD_UID=$(id -u) XRPLD_GID=$(id -g) \ + docker compose -f docker/telemetry/docker-compose.yml up -d +``` + Wait for services to be ready: ```bash @@ -469,7 +478,7 @@ Expected: log lines with `trace_id=<32hex> span_id=<16hex>` between the severity code and the message. Example: ``` -2024-Jan-15 10:30:45.123456 UTC RPCHandler:NFO trace_id=abc123def456789012345678abcdef01 span_id=0123456789abcdef Calling server_info +2024-Jan-15 10:30:45.123456789 UTC RPCHandler:NFO trace_id=abc123def456789012345678abcdef01 span_id=0123456789abcdef Calling server_info ``` Lines emitted outside of an active span (background tasks, startup) will @@ -480,26 +489,42 @@ NOT have trace context — this is expected. Extract a `trace_id` from the log and verify it exists in Tempo: ```bash -TRACE_ID=$(grep -o 'trace_id=[a-f0-9]\{32\}' /path/to/debug.log | head -1 | cut -d= -f2) +TRACE_ID=$(grep -m1 -o 'trace_id=[a-f0-9]\{32\}' /path/to/debug.log | cut -d= -f2) echo "Checking trace: $TRACE_ID" -curl -s "http://localhost:3200/api/traces/$TRACE_ID" | jq '.data | length' +curl -s "http://localhost:3200/api/traces/$TRACE_ID" | jq '.batches | length' ``` -Expected result: `1` (the trace exists in Tempo). +Expected result: `> 0` (the trace exists in Tempo). +Tempo returns the trace in OTLP shape, so the array is `batches`, not `data`, +and one trace can arrive as several batches. ### Step 3: Verify Loki log ingestion -The OTel Collector's filelog receiver tails xrpld's debug.log and +The OTel Collector's file_log receiver tails xrpld's debug.log and exports parsed entries to Loki. Verify Loki has received entries: ```bash -# Query Loki for any xrpld logs -curl -sG "http://localhost:3100/loki/api/v1/query" \ +# Query Loki for any xrpld logs in the last 10 minutes +NOW_NS=$(($(date +%s) * 1000000000)) +curl -sG "http://localhost:3100/loki/api/v1/query_range" \ --data-urlencode 'query={service_name="xrpld"}' \ - --data-urlencode 'limit=5' | jq '.data.result | length' + --data-urlencode "start=$((NOW_NS - 600000000000))" \ + --data-urlencode "end=${NOW_NS}" \ + --data-urlencode 'limit=5' \ + --data-urlencode 'direction=backward' | + jq '[.data.result[].values | length] | add // 0' ``` -Expected: > 0 results. +Expected: > 0 log lines. + +Use `query_range`, not `query`. Loki rejects a bare log selector on the +instant `/query` endpoint with HTTP 400 and a `text/plain` body +("log queries are not supported as an instant query type"), so `jq` fails to +parse it and the step never prints a number — even when ingestion is working. +Only metric queries such as `sum(count_over_time(...))` are allowed there, +which is why the validation scripts can use the instant endpoint. +Timestamps are unix nanoseconds, matching `workload/validate_telemetry.py`. +Counting `.data.result | length` would count streams, not log lines. ### Step 4: Verify Grafana Tempo-to-Loki correlation @@ -555,7 +580,7 @@ Expected: > 0 results. ``` 2. Verify `[ips_fixed]` lists all 6 peer ports 3. Verify `validators.txt` has all 6 public keys -4. Check node debug logs: `tail -50 /tmp/xrpld-integration/node1/debug.log` +4. Check node debug logs: `tail -50 /tmp/xrpld-integration/Node-1/debug.log` 5. Ensure `[peer_private]` is set to `1` (prevents reaching out to public network) ### Transaction not processing @@ -588,15 +613,15 @@ Expected: > 0 results. The mount source defaults to the repo-relative `docker/telemetry/data/logs` (where the telemetry configs write). Override `XRPLD_LOG_DIR` to tail logs from another root. -2. Check OTel Collector logs for filelog receiver errors: +2. Check OTel Collector logs for file_log receiver errors: ```bash - docker compose -f docker/telemetry/docker-compose.yml logs otel-collector | grep -i "filelog\|loki\|error" + docker compose -f docker/telemetry/docker-compose.yml logs otel-collector | grep -i "file_log\|loki\|error" ``` 3. Verify Loki is running: ```bash curl -s http://localhost:3100/ready ``` -4. Verify the filelog receiver glob pattern matches your log files: +4. Verify the file_log receiver glob pattern matches your log files: The default pattern is `/var/log/xrpld/*/debug.log` ### Grafana trace-log links not working diff --git a/docker/telemetry/docker-compose.yml b/docker/telemetry/docker-compose.yml index 98f6965c3f..5665f9d56d 100644 --- a/docker/telemetry/docker-compose.yml +++ b/docker/telemetry/docker-compose.yml @@ -3,7 +3,7 @@ # Provides services for local development: # - otel-collector: receives OTLP traces from xrpld, batches and # forwards them to Tempo. Also tails xrpld log files -# via filelog receiver and exports to Loki. Listens on ports +# via file_log receiver and exports to Loki. Listens on ports # 4317 (gRPC) and 4318 (HTTP). # - tempo: Grafana Tempo tracing backend, queryable via Grafana Explore # on port 3000. Recommended for production (S3/GCS storage, TraceQL). @@ -40,11 +40,34 @@ services: networks: - xrpld-telemetry + # One-shot init for the xrpld log root. Docker creates a missing bind-mount + # source as root, and xrpld then cannot create the subdirectory + # inside it. Config::getDebugLogFile() only warns on that failure and carries + # on, so the node looks healthy while writing no debug.log at all and the + # whole log pipeline stays empty with no error at any layer. Create the + # directory here and hand it to the host user instead. + # + # XRPLD_UID/XRPLD_GID default to 1000, the first non-root user on a typical + # Linux host. Set them if `id -u` differs, or xrpld still cannot write. + # Reuses the Prometheus image for the same reason otelcol-storage-init does. + xrpld-logdir-init: + image: prom/prometheus:v3.13.2 + user: "0:0" + entrypoint: ["sh", "-c"] + command: + [ + "mkdir -p /data/logs && chown ${XRPLD_UID:-1000}:${XRPLD_GID:-1000} /data /data/logs", + ] + volumes: + - ./data:/data + networks: + - xrpld-telemetry + # OpenTelemetry Collector: receives spans from xrpld via OTLP protocol, # batches them for efficiency, and forwards to Tempo for storage. otel-collector: image: otel/opentelemetry-collector-contrib:0.158.0 - # Second --config layers filelog offset persistence on top of the shared + # Second --config layers file_log offset persistence on top of the shared # base config; the collector deep-merges them. Only this stack keeps its # logs across restarts, so only this stack needs it. command: @@ -62,16 +85,18 @@ services: volumes: # Mount collector pipeline config (receivers → processors → exporters) - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml:ro - # Dev-only overlay: persist filelog read offsets across restarts + # Dev-only overlay: persist file_log read offsets across restarts - ./otel-collector-filestorage.yaml:/etc/otel-collector-filestorage.yaml:ro - # Mount the xrpld log root for the filelog receiver. The telemetry + # Mount the xrpld log root for the file_log receiver. The telemetry # configs write to docker/telemetry/data/logs//debug.log, so - # the default source is the repo-relative ./data/logs — user-owned and - # needing no root, so `docker compose up` works with no setup. Override - # XRPLD_LOG_DIR to point at another root (e.g. the integration test sets - # it to its own workdir). Mounted read-only so the collector only tails. + # the default source is the repo-relative ./data/logs, which + # xrpld-logdir-init has already created and handed to the host user. + # Override XRPLD_LOG_DIR to point at another root (e.g. the integration + # test sets it to its own workdir; that root is created by the test, so + # the init service is a no-op there). Mounted read-only so the collector + # only tails. - ${XRPLD_LOG_DIR:-./data/logs}:/var/log/xrpld:ro - # Persisted filelog read offsets, so a collector restart resumes + # Persisted file_log read offsets, so a collector restart resumes # instead of re-reading every debug.log from the top. - otelcol-storage:/var/lib/otelcol depends_on: @@ -81,6 +106,8 @@ services: condition: service_started otelcol-storage-init: condition: service_completed_successfully + xrpld-logdir-init: + condition: service_completed_successfully networks: - xrpld-telemetry @@ -101,7 +128,7 @@ services: # Grafana Loki for centralized log ingestion and log-trace # correlation. Loki 3.x supports native OTLP ingestion, so the OTel - # Collector exports via otlphttp to Loki's /otlp endpoint. + # Collector exports via otlp_http to Loki's /otlp endpoint. # Query logs via Grafana Explore -> Loki at http://localhost:3000. loki: image: grafana/loki:3.4.2 diff --git a/docker/telemetry/grafana/provisioning/datasources/loki.yaml b/docker/telemetry/grafana/provisioning/datasources/loki.yaml index 0a6b73a575..a70ac9deb3 100644 --- a/docker/telemetry/grafana/provisioning/datasources/loki.yaml +++ b/docker/telemetry/grafana/provisioning/datasources/loki.yaml @@ -1,6 +1,6 @@ # Grafana Loki data source provisioning for rippled log-trace correlation. # -# Loki ingests rippled logs via OTel Collector's filelog receiver. +# Loki ingests rippled logs via OTel Collector's file_log receiver. # The derivedFields config links trace_id values in log lines back to # Tempo traces, enabling one-click log-to-trace navigation in Grafana. diff --git a/docker/telemetry/integration-test.sh b/docker/telemetry/integration-test.sh index e3981d4e4a..6c930dcee9 100755 --- a/docker/telemetry/integration-test.sh +++ b/docker/telemetry/integration-test.sh @@ -37,6 +37,10 @@ GENESIS_SEED="snoPBrXtMeMyMHUVTgbuqAfg1SUTb" DEST_ACCOUNT="" # Generated dynamically via wallet_propose TEMPO="http://localhost:3200" PROM="http://localhost:9090" +LOKI="http://localhost:3100" +# How long to wait for a log line to travel file -> file_log receiver -> batch +# processor -> Loki. The batch timeout is 1s, so this is mostly ingestion slack. +LOKI_INGEST_TIMEOUT=30 # Counters for pass/fail PASS=0 @@ -88,7 +92,7 @@ check_log_correlation() { local sample_trace_id="" for i in $(seq 1 "$NUM_NODES"); do - local logfile="$WORKDIR/node$i/debug.log" + local logfile="$WORKDIR/Node-$i/debug.log" if [ ! -f "$logfile" ]; then continue fi @@ -97,12 +101,17 @@ check_log_correlation() { matches=$(grep -c 'trace_id=[a-f0-9]\{32\} span_id=[a-f0-9]\{16\}' "$logfile") || matches=0 total_matches=$((total_matches + matches)) if [ -z "$sample_trace_id" ] && [ "$matches" -gt 0 ]; then - sample_trace_id=$(grep -o 'trace_id=[a-f0-9]\{32\}' "$logfile" | head -1 | cut -d= -f2) + # -m1 makes grep stop after the first match and exit normally. + # Piping into `head -1` instead closes the pipe under grep, and + # under `set -o pipefail` the resulting SIGPIPE (141) aborts the + # whole run. It only bites once the log is bigger than the pipe + # buffer, so it reads as a flaky test. + sample_trace_id=$(grep -m1 -o 'trace_id=[a-f0-9]\{32\}' "$logfile" | cut -d= -f2) fi done if [ "$files_scanned" -eq 0 ]; then - fail "Log correlation: no debug.log files found in $WORKDIR/node*/" + fail "Log correlation: no debug.log files found in $WORKDIR/Node-*/" return fi @@ -123,14 +132,54 @@ check_log_correlation() { else fail "Log-Tempo cross-check: trace_id=$sample_trace_id NOT found in Tempo" fi + + check_loki_ingestion "$sample_trace_id" fi } +# Verify the log line actually reached Loki, not just the local file. +# +# Without this the log-correlation check passes on a stack whose log mount is +# wrong or whose Loki exporter is broken, because reading the file and reading +# Tempo both still work. This is the only assertion that exercises the +# file_log -> Loki hop, so it is what makes the log pipeline tested rather than +# merely configured. +# +# Uses /query_range, not /query: Loki rejects a bare log selector on the instant +# endpoint with HTTP 400 and a text/plain body, so jq could never parse it. +# Bounds are unix nanoseconds, matching workload/validate_telemetry.py. +check_loki_ingestion() { + local trace_id="$1" + local lines=0 + local start_ns end_ns + + for attempt in $(seq 1 "$LOKI_INGEST_TIMEOUT"); do + end_ns=$(($(date +%s) * 1000000000)) + # Look back over the whole run, not a fixed window: the entry carries + # the timestamp parsed out of the log line, not its ingestion time. + start_ns=$((end_ns - 86400000000000)) + lines=$(curl -sfG "$LOKI/loki/api/v1/query_range" \ + --data-urlencode "query={service_name=\"xrpld\"} |= \"$trace_id\"" \ + --data-urlencode "start=$start_ns" \ + --data-urlencode "end=$end_ns" \ + --data-urlencode "limit=5" \ + --data-urlencode "direction=backward" | + jq '[.data.result[].values | length] | add // 0' 2>/dev/null) || lines=0 + if [ "${lines:-0}" -gt 0 ]; then + ok "Loki ingestion: trace_id=$trace_id found in Loki ($lines lines, attempt $attempt)" + return + fi + sleep 1 + done + + fail "Loki ingestion: trace_id=$trace_id never reached Loki after ${LOKI_INGEST_TIMEOUT}s" +} + cleanup() { log "Cleaning up..." # Kill xrpld nodes for i in $(seq 1 "$NUM_NODES"); do - local pidfile="$WORKDIR/node$i/xrpld.pid" + local pidfile="$WORKDIR/Node-$i/xrpld.pid" if [ -f "$pidfile" ]; then kill "$(cat "$pidfile")" 2>/dev/null || true rm -f "$pidfile" @@ -171,7 +220,7 @@ log "All prerequisites met." # --------------------------------------------------------------------------- log "Cleaning previous run data..." for i in $(seq 1 "$NUM_NODES"); do - pidfile="$WORKDIR/node$i/xrpld.pid" + pidfile="$WORKDIR/Node-$i/xrpld.pid" if [ -f "$pidfile" ]; then kill "$(cat "$pidfile")" 2>/dev/null || true fi @@ -237,6 +286,18 @@ for attempt in $(seq 1 30); do sleep 1 done +log "Waiting for Loki to be ready..." +for attempt in $(seq 1 60); do + if curl -sf "$LOKI/ready" >/dev/null 2>&1; then + log "Loki ready (attempt $attempt)." + break + fi + if [ "$attempt" -eq 60 ]; then + die "Loki not ready after 60s" + fi + sleep 1 +done + # --------------------------------------------------------------------------- # Step 3: Generate validator keys # --------------------------------------------------------------------------- @@ -326,7 +387,7 @@ VALIDATORS_FILE="$WORKDIR/validators.txt" # Create per-node configs for i in $(seq 1 "$NUM_NODES"); do - NODE_DIR="$WORKDIR/node$i" + NODE_DIR="$WORKDIR/Node-$i" mkdir -p "$NODE_DIR/nudb" "$NODE_DIR/db" RPC_PORT=$((RPC_PORT_BASE + i - 1)) @@ -419,7 +480,7 @@ done log "Starting $NUM_NODES xrpld nodes..." for i in $(seq 1 "$NUM_NODES"); do - NODE_DIR="$WORKDIR/node$i" + NODE_DIR="$WORKDIR/Node-$i" "$XRPLD" --conf "$NODE_DIR/xrpld.cfg" --start >"$NODE_DIR/stdout.log" 2>&1 & echo $! >"$NODE_DIR/xrpld.pid" log " Node $i started (PID $(cat "$NODE_DIR/xrpld.pid"))" @@ -719,7 +780,7 @@ echo " xrpld nodes (6) are running:" for i in $(seq 1 "$NUM_NODES"); do RPC_PORT=$((RPC_PORT_BASE + i - 1)) PEER_PORT=$((PEER_PORT_BASE + i - 1)) - echo " Node $i: RPC=localhost:$RPC_PORT Peer=:$PEER_PORT PID=$(cat "$WORKDIR/node$i/xrpld.pid" 2>/dev/null || echo 'unknown')" + echo " Node $i: RPC=localhost:$RPC_PORT Peer=:$PEER_PORT PID=$(cat "$WORKDIR/Node-$i/xrpld.pid" 2>/dev/null || echo 'unknown')" done echo "" echo " To tear down:" diff --git a/docker/telemetry/otel-collector-config.yaml b/docker/telemetry/otel-collector-config.yaml index 618eb94d5a..8ad4545dea 100644 --- a/docker/telemetry/otel-collector-config.yaml +++ b/docker/telemetry/otel-collector-config.yaml @@ -3,7 +3,7 @@ # Pipelines: # traces: OTLP receiver -> batch processor -> debug + Tempo + spanmetrics # metrics: OTLP receiver + spanmetrics connector -> Prometheus exporter -# logs: filelog receiver -> batch processor -> otlphttp/Loki +# logs: file_log receiver -> batch processor -> otlp_http/Loki # # xrpld sends traces via OTLP/HTTP to port 4318. The collector batches # them, forwards to Tempo, and derives RED metrics via the spanmetrics @@ -14,16 +14,12 @@ # metrics pipelines. Metrics are exported to Prometheus alongside # span-derived metrics. # -# The filelog receiver tails xrpld's debug.log files under +# The file_log receiver tails xrpld's debug.log files under # /var/log/xrpld/ (mounted from the host). A regex_parser operator # extracts timestamp, partition, severity, and optional trace_id/span_id # fields injected by Logs::format(). Parsed logs are exported to Grafana # Loki for log-trace correlation. -extensions: - health_check: - endpoint: 0.0.0.0:13133 - receivers: otlp: protocols: @@ -35,8 +31,13 @@ receivers: # correlation. Extracts structured fields (timestamp, partition, severity, # trace_id, span_id, message) via regex. The trace_id and span_id are # optional — only present when the log was emitted within an active span. - filelog: + file_log: include: [/var/log/xrpld/*/debug.log] + # Needed to recover which node a line came from. The subdirectory name is + # the only per-node signal in the log stream: Logs::format() writes + # trace_id and span_id but no node identity. Emitters name the directory + # after their own service_instance_id so the two agree. + include_file_path: true # Read each file from the start. The upstream default is `end`, which # skips everything written before the receiver's first poll — so any log # line a node emitted before the collector got to it would be lost, and @@ -49,16 +50,36 @@ receivers: start_at: beginning operators: # Log format emitted by Logs::format() is: - # YYYY-Mmm-DD HH:MM:SS.ffffff UTC : [trace_id=... span_id=...] + # YYYY-Mmm-DD HH:MM:SS.fffffffff UTC : [trace_id=... span_id=...] # The `partition:` prefix is omitted when partition is empty, so the - # capture group is non-capturing optional. Fractional seconds up to 6 - # digits are parsed via the `%f` strptime directive. + # capture group is non-capturing optional. The node emits nanosecond + # precision (9 digits); `%f` accepts any number of fractional digits. - type: regex_parser regex: '^(?P\S+\s+\S+)\s+\S+\s+(?:(?P\S+):)?(?P\S+)\s+(?:trace_id=(?P[a-f0-9]+)\s+span_id=(?P[a-f0-9]+)\s+)?(?P.*)$' timestamp: parse_from: attributes.timestamp layout: "%Y-%b-%d %H:%M:%S.%f" location: UTC + # Lift the per-node directory out of the file path and onto the + # RESOURCE. include_file_path alone is not enough: it produces a log + # RECORD attribute, and on OTLP ingest Loki promotes only an allow-list + # of RESOURCE attributes to indexed stream labels. A record attribute + # becomes structured metadata, which cannot be used in a {...} selector. + # service.instance.id is on that allow-list and arrives as the LogQL + # label service_instance_id, which is the label the dashboards filter on. + # Dotted keys need bracket syntax; dot notation would be read as a + # nested traversal and match nothing. + - type: regex_parser + parse_from: attributes["log.file.path"] + parse_to: attributes + regex: "^/var/log/xrpld/(?P[^/]+)/" + - type: move + from: attributes.node_dir + to: resource["service.instance.id"] + # Drop the raw path once the node name is on the resource. Keeping it + # would add a structured-metadata field to every line for no benefit. + - type: remove + field: attributes["log.file.path"] processors: batch: @@ -179,7 +200,7 @@ exporters: # Export logs to Grafana Loki via OTLP/HTTP. Loki 3.x supports # native OTLP ingestion on its /otlp endpoint, replacing the removed # loki exporter (dropped in otel-collector-contrib v0.147.0). - otlphttp/loki: + otlp_http/loki: endpoint: http://loki:3100/otlp prometheus: endpoint: 0.0.0.0:8889 @@ -190,6 +211,10 @@ exporters: resource_to_telemetry_conversion: enabled: true +extensions: + health_check: + endpoint: 0.0.0.0:13133 + service: extensions: [health_check] pipelines: @@ -199,11 +224,14 @@ service: exporters: [debug, otlp/tempo, spanmetrics] metrics: receivers: [otlp, spanmetrics] + # batch keeps the OTLP metric path from exporting one request per + # instrument. It delays a sample by at most the batch timeout, which + # is well under the Prometheus scrape interval. processors: [resource/tier, resource/stripsdk, batch] exporters: [prometheus] - # Log pipeline ingests xrpld debug.log via filelog receiver, + # Log pipeline ingests xrpld debug.log via file_log receiver, # batches entries, and exports to Loki for log-trace correlation. logs: - receivers: [filelog] + receivers: [file_log] processors: [resource/logs, resource/tier, resource/stripsdk, batch] - exporters: [otlphttp/loki] + exporters: [otlp_http/loki] diff --git a/docker/telemetry/otel-collector-filestorage.yaml b/docker/telemetry/otel-collector-filestorage.yaml index 5362431725..fec6c7ddab 100644 --- a/docker/telemetry/otel-collector-filestorage.yaml +++ b/docker/telemetry/otel-collector-filestorage.yaml @@ -1,4 +1,4 @@ -# Collector overlay that persists filelog read offsets. Applied ONLY by the +# Collector overlay that persists file_log read offsets. Applied ONLY by the # developer stack (docker/telemetry/docker-compose.yml), as a second --config # after otel-collector-config.yaml; the collector deep-merges the two. # @@ -15,14 +15,14 @@ # instead of re-reading debug.log from the top. extensions: - file_storage/filelog: + file_storage/file_log: directory: /var/lib/otelcol/file_storage create_directory: true receivers: - filelog: - storage: file_storage/filelog + file_log: + storage: file_storage/file_log # Lists are replaced rather than merged, so this must repeat the base entry. service: - extensions: [health_check, file_storage/filelog] + extensions: [health_check, file_storage/file_log] diff --git a/docker/telemetry/xrpld-telemetry.cfg b/docker/telemetry/xrpld-telemetry.cfg index 64c59f4577..3294630bc9 100644 --- a/docker/telemetry/xrpld-telemetry.cfg +++ b/docker/telemetry/xrpld-telemetry.cfg @@ -35,10 +35,15 @@ advisory_delete=0 docker/telemetry/data # Path is resolved relative to this config file's directory (docker/telemetry), -# so this writes to docker/telemetry/data/logs/devnet/debug.log — the same -# dir the compose stack bind-mounts into the collector as /var/log/xrpld. +# so this writes to docker/telemetry/data/logs/xrpld-standalone/debug.log — the +# same dir the compose stack bind-mounts into the collector as /var/log/xrpld. +# +# The subdirectory name must equal [telemetry] service_instance_id below. The +# collector reads it off the file path and stamps it as the Loki label +# service_instance_id, so a mismatch here means log lines carry a node name +# that no trace or metric shares, and nothing joins. [debug_logfile] -data/logs/devnet/debug.log +data/logs/xrpld-standalone/debug.log [rpc_startup] { "command": "log_level", "severity": "debug" } diff --git a/docs/telemetry-runbook.md b/docs/telemetry-runbook.md index e90b30171c..adbeda9ffb 100644 --- a/docs/telemetry-runbook.md +++ b/docs/telemetry-runbook.md @@ -857,7 +857,7 @@ Requires `trace_peer=1` in the `[telemetry]` config section. When xrpld is built with `telemetry=ON`, log lines emitted within an active, sampled OpenTelemetry span automatically include `trace_id` and `span_id` fields: ``` -2024-Jan-15 10:30:45.123456 UTC LedgerMaster:NFO trace_id=abc123def456789012345678abcdef01 span_id=0123456789abcdef Validated ledger 42 +2024-Jan-15 10:30:45.123456789 UTC LedgerMaster:NFO trace_id=abc123def456789012345678abcdef01 span_id=0123456789abcdef Validated ledger 42 ``` This enables bidirectional navigation between logs and traces in Grafana: @@ -867,9 +867,11 @@ This enables bidirectional navigation between logs and traces in Grafana: ### Log Ingestion Pipeline -Log files are ingested by the OTel Collector's `filelog` receiver, which tails `debug.log` files and parses them with a regex that extracts `timestamp`, `partition`, `severity`, `trace_id`, `span_id`, and `message` fields. Parsed entries are exported to Grafana Loki. +Log files are ingested by the OTel Collector's `file_log` receiver, which tails `debug.log` files and parses them with a regex that extracts `timestamp`, `partition`, `severity`, `trace_id`, `span_id`, and `message` fields. Parsed entries are exported to Grafana Loki. -The receiver tails `/var/log/xrpld/*/debug.log` inside the collector container. docker-compose bind-mounts the host log root there; the source defaults to the repo-relative `docker/telemetry/data/logs`, which the telemetry configs write to (`data/logs//debug.log`) and which needs no root. To tail logs from elsewhere, set `XRPLD_LOG_DIR` before `docker compose up` (the integration test does this to point at its own workdir). The single trailing `*` matches one per-network or per-node subdirectory. +The receiver tails `/var/log/xrpld/*/debug.log` inside the collector container. docker-compose bind-mounts the host log root there; the source defaults to the repo-relative `docker/telemetry/data/logs`, which the telemetry configs write to (`data/logs//debug.log`). To tail logs from elsewhere, set `XRPLD_LOG_DIR` before `docker compose up` (the integration test does this to point at its own workdir). The single trailing `*` matches one per-node subdirectory. + +That subdirectory is load-bearing, not cosmetic. Docker creates a missing bind-mount source as root, and `Config::getDebugLogFile()` only warns when it cannot create the log directory, so a root-owned log root produces a healthy-looking node that writes no `debug.log` and an empty Loki with no error at any layer. The `xrpld-logdir-init` service creates the directory and hands it to `XRPLD_UID`/`XRPLD_GID` (default 1000) to prevent that. The receiver also lifts the subdirectory name onto the resource attribute `service.instance.id`, which Loki indexes as the label `service_instance_id`, so each emitter must name its log directory after its own `[telemetry] service_instance_id` or log lines carry a node name that no trace or metric shares. Each file is read from the beginning, because the receiver's own default (`end`) would skip anything a node wrote before the collector's first poll and would never read a log that has stopped being written to. Read offsets are held in memory by default, so a restarted collector re-reads the files it already ingested. The developer stack avoids that by layering `otel-collector-filestorage.yaml` as a second `--config`, which adds a `file_storage` extension that keeps the offsets on a named volume; a one-shot init service prepares that volume, because the collector runs as a non-root user and a fresh Docker volume is owned by root. Ephemeral stacks such as the workload validation harness create a fresh log directory per run, so they have nothing to resume from and deliberately omit the overlay. @@ -973,9 +975,9 @@ count_over_time({service_name="xrpld"} |= "trace_id=" [5m]) ### No logs in Loki - Verify the log file mount in docker-compose.yml points to the correct xrpld log directory (default source `docker/telemetry/data/logs`, or the `XRPLD_LOG_DIR` override) and that xrpld actually writes `debug.log` there -- Check OTel Collector logs for filelog receiver errors: `docker compose logs otel-collector` +- Check OTel Collector logs for file_log receiver errors: `docker compose logs otel-collector` - Verify Loki is running: `curl http://localhost:3100/ready` -- Check the filelog receiver glob `/var/log/xrpld/*/debug.log` matches your log layout — the log file must sit one subdirectory below the mount root +- Check the file_log receiver glob `/var/log/xrpld/*/debug.log` matches your log layout — the log file must sit one subdirectory below the mount root ## Performance Tuning From 7f8f5b4be3cc24ee8b2214f2f442ed13ac9a980f Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Wed, 9 Sep 2026 15:12:18 +0100 Subject: [PATCH 14/16] feat(telemetry): ship logs through Alloy, alongside traces and metrics Alloy carried no log pipeline at all: no loki.source, loki.write or loki.process anywhere in the config, against a working metrics and OTLP path. Any deployment fed through Alloy rather than the reference collector therefore sent traces and metrics but no logs, so log-to-trace correlation was unavailable there even though both ends of the link were configured. Logs now leave through the same otlphttp exporter as the other two signals, so all three carry one resource identity. Alloy's own filelog receiver would have been the closest match to the reference collector, but it is still public-preview and refuses to load unless the service is started with --stability.level=public-preview, which would mean editing the unit on a box reachable only by RunCommand. The Loki source components are generally available, so they are used and bridged into OTLP by otelcol.receiver.loki; the service needs no extra flag. That bridge hands over an empty resource and puts everything on the log record, so the transform sets the resource attributes in log context. service.instance.id is concatenated in from XRPLD_HOST_LABEL because OTTL has no env() converter, and it is a resource attribute rather than a record one because only resource attributes are promoted to indexed Loki labels. It must equal the node's own service_instance_id or the logs join nothing. devnet writes one flat file rather than a per-node directory, so identity cannot be read off the path the way the docker collector does it; XRPLD_LOG_GLOB overrides the path for other layouts. The line is parsed for its own timestamp, severity and trace context, and trace_id/span_id are set on the first-class OTLP record fields so Grafana links a log to its trace without re-parsing the body. Lines emitted outside a sampled span keep an empty trace id rather than an invalid one. Also carry the node-identity operators into the Grafana Cloud collector variant, align this config's log directory with its service_instance_id, and rename the deprecated otlphttp/filelog collector component names. Alloy's otelcol.exporter.otlphttp is that product's own name and is unchanged. --- OpenTelemetryPlan/02-design-decisions.md | 6 +- .../05-configuration-reference.md | 30 ++-- .../07-observability-backends.md | 12 +- docker/telemetry/alloy/config.alloy | 129 +++++++++++++++++- .../dashboards/log-derived-insights.json | 2 +- .../otel-collector-config.grafanacloud.yaml | 41 ++++-- docker/telemetry/xrpld-telemetry.cfg | 9 +- docs/telemetry-runbook.md | 16 +-- 8 files changed, 200 insertions(+), 45 deletions(-) diff --git a/OpenTelemetryPlan/02-design-decisions.md b/OpenTelemetryPlan/02-design-decisions.md index e32ecc4b4e..f7a7ee7022 100644 --- a/OpenTelemetryPlan/02-design-decisions.md +++ b/OpenTelemetryPlan/02-design-decisions.md @@ -505,7 +505,7 @@ The following data is explicitly **excluded** from telemetry collection: > 1. `PeerImp`'s constructor logs the peer's `remoteAddress_` — an `IP:port` — > at `info` severity (`PeerImp.h:837-842`), and other overlay call sites log > addresses too. These land in the ordinary `debug.log` stream. -> 2. The collector's `filelog` receiver tails exactly that file +> 2. The collector's `file_log` receiver tails exactly that file > (`otel-collector-config.yaml:38-47`, `include: [/var/log/xrpld/*/debug.log]`) > and the `logs` pipeline exports it to Loki (`:236-239`). > @@ -515,7 +515,7 @@ The following data is explicitly **excluded** from telemetry collection: > fields — a `delete` action on an attribute key would not touch them. > > **The control points are therefore log-side, not trace-side:** Loki -> retention and access control on the log store; the `filelog` receiver's +> retention and access control on the log store; the `file_log` receiver's > `include` list (dropping it disables log↔trace correlation entirely); or a > collector-side transform on the log body. Do not describe the telemetry > pipeline as IP-free without qualifying it to traces. @@ -875,7 +875,7 @@ rather than calling `GetSpan()`, so the common no-span path costs no heap allocation. Because the IDs land in the ordinary `debug.log` stream, correlation is -end-to-end without touching PerfLog: the collector's `filelog` receiver parses +end-to-end without touching PerfLog: the collector's `file_log` receiver parses `trace_id`/`span_id` as optional capture groups and ships the lines to Loki, and Grafana links both directions (Tempo `tracesToLogs` → Loki, Loki derived fields → Tempo). Details in [05 §5.8.5](./05-configuration-reference.md). diff --git a/OpenTelemetryPlan/05-configuration-reference.md b/OpenTelemetryPlan/05-configuration-reference.md index ab473e667c..13078518ea 100644 --- a/OpenTelemetryPlan/05-configuration-reference.md +++ b/OpenTelemetryPlan/05-configuration-reference.md @@ -251,12 +251,12 @@ local stack and by CI. It carries **three** pipelines, not one: | --------- | --------------------- | ---------------------------------------------------------------- | ------------------------------------ | | `traces` | `otlp` | `resource/tier`, `resource/stripsdk`, `attributes/hash`, `batch` | `debug`, `otlp/tempo`, `spanmetrics` | | `metrics` | `otlp`, `spanmetrics` | `resource/tier`, `resource/stripsdk`, `batch` | `prometheus` | -| `logs` | `filelog` | `resource/logs`, `resource/tier`, `resource/stripsdk`, `batch` | `otlphttp/loki` | +| `logs` | `file_log` | `resource/logs`, `resource/tier`, `resource/stripsdk`, `batch` | `otlp_http/loki` | Component detail: - **Receivers.** `otlp` on gRPC `0.0.0.0:4317` and HTTP `0.0.0.0:4318` (both - traces and native metrics arrive on 4318). `filelog` tails + traces and native metrics arrive on 4318). `file_log` tails `/var/log/xrpld/*/debug.log` and runs a `regex_parser` that lifts `timestamp`, `partition`, `severity` and the optional `trace_id`/`span_id` emitted by the journal sink (§5.8.5). @@ -282,7 +282,7 @@ Component detail: dimensions are promoted to labels (`command`, `rpc_status`, `tx_type`, `ter_result`, `stage`, `consensus_mode`, `outcome`, …). - **Exporters.** `debug` (console, `verbosity: detailed`), `otlp/tempo` - (`tempo:4317`, `tls.insecure: true`), `otlphttp/loki` + (`tempo:4317`, `tls.insecure: true`), `otlp_http/loki` (`http://loki:3100/otlp` — Loki 3.x native OTLP; the old `loki` exporter was removed in collector-contrib v0.147.0), and `prometheus` on `0.0.0.0:8889` with `resource_to_telemetry_conversion.enabled: true` so the @@ -307,7 +307,7 @@ graph. The full delta: | `basicauth/grafanacloud` | `:29` | Extension; instance id / API token from the container environment | | `tail_sampling` | `:60` | One `probabilistic` policy at **0.5%**, `decision_wait: 10s` | | `transform/cloudlabels` | `:119` | Copies three resource attrs onto datapoint labels for Cloud (OTLP) ingest | -| `otlphttp/grafanacloud` | `:236` | Single OTLP/HTTP exporter fanning all three signals to Grafana Cloud | +| `otlp_http/grafanacloud` | `:236` | Single OTLP/HTTP exporter fanning all three signals to Grafana Cloud | | `metrics_flush_interval` | `:136` | `spanmetrics` flushes every 15s instead of the 60s default | | Removed by the overlay | Consequence | @@ -346,14 +346,14 @@ NetworkPolicy, peer trace-context validation) is covered in The authoritative development stack lives in the repo at `docker/telemetry/docker-compose.yml`. It brings up **six** services on a shared `xrpld-telemetry` bridge network. All images are pinned to exact tags. -| Service | Image | Published ports | Role | -| ---------------- | ---------------------------------------------- | ---------------------- | ---------------------------------------------------------------- | -| `otel-collector` | `otel/opentelemetry-collector-contrib:0.158.0` | `4317`, `4318`, `8889` | OTLP ingest, spanmetrics, filelog tail, Prometheus scrape target | -| `tempo` | `grafana/tempo:2.9.4` | `3200` | Trace storage and TraceQL | -| `loki` | `grafana/loki:3.7.6` | `3100` | Log storage for log↔trace correlation | -| `prometheus` | `prom/prometheus:v3.13.2` | `9090` | Scrapes the collector's `:8889` | -| `grafana` | `grafana/grafana:13.1.2` | `3000` | Dashboards + provisioned datasources/alerts, anonymous admin | -| `renderer` | `grafana/grafana-image-renderer:v5.12.0` | `8081` | Panel→PNG rendering for image export and alert screenshots | +| Service | Image | Published ports | Role | +| ---------------- | ---------------------------------------------- | ---------------------- | ----------------------------------------------------------------- | +| `otel-collector` | `otel/opentelemetry-collector-contrib:0.158.0` | `4317`, `4318`, `8889` | OTLP ingest, spanmetrics, file_log tail, Prometheus scrape target | +| `tempo` | `grafana/tempo:2.9.4` | `3200` | Trace storage and TraceQL | +| `loki` | `grafana/loki:3.7.6` | `3100` | Log storage for log↔trace correlation | +| `prometheus` | `prom/prometheus:v3.13.2` | `9090` | Scrapes the collector's `:8889` | +| `grafana` | `grafana/grafana:13.1.2` | `3000` | Dashboards + provisioned datasources/alerts, anonymous admin | +| `renderer` | `grafana/grafana-image-renderer:v5.12.0` | `8081` | Panel→PNG rendering for image export and alert screenshots | Two corrections to earlier drafts: @@ -366,7 +366,7 @@ Two corrections to earlier drafts: port mapping or run `docker compose exec`. The collector also bind-mounts the xrpld log root read-only -(`${XRPLD_LOG_DIR:-./data/logs}` → `/var/log/xrpld`) for the `filelog` +(`${XRPLD_LOG_DIR:-./data/logs}` → `/var/log/xrpld`) for the `file_log` receiver, and the `grafana` service reads Slack/email alert secrets from an optional gitignored `.env.alerting`. @@ -570,12 +570,12 @@ Fluentd or PerfLog change. Two pieces: allocation on the (common) no-span path. This is the ordinary `debug.log` stream — PerfLog is not involved, and the `setTraceId` hook described in earlier drafts was never built. -2. **The collector ingests them.** The `filelog` receiver tails +2. **The collector ingests them.** The `file_log` receiver tails `/var/log/xrpld/*/debug.log` and its `regex_parser` lifts `trace_id` and `span_id` as optional capture groups (§5.5.1). `resource/logs` applies an `upsert` of `service.name=xrpld`, which Loki promotes to the stream label `service_name`, so the canonical selector is **`{service_name="xrpld"}`**. - Logs land in Loki via `otlphttp/loki`. + Logs land in Loki via `otlp_http/loki`. > **Known issue — the collector's `job` upsert is ineffective for stream > selection.** `resource/logs` also applies an `upsert` of a `job=xrpld` attribute diff --git a/OpenTelemetryPlan/07-observability-backends.md b/OpenTelemetryPlan/07-observability-backends.md index 8dcfe61982..993d5b994a 100644 --- a/OpenTelemetryPlan/07-observability-backends.md +++ b/OpenTelemetryPlan/07-observability-backends.md @@ -410,7 +410,7 @@ How to correlate OpenTelemetry traces with existing xrpld observability. There is **one** collection agent, not three. Earlier drafts of this diagram routed logs through "Promtail/Fluentd" and metrics through a "StatsD Exporter"; neither exists in this stack. Logs are read by the OTel Collector's own -`filelog` receiver, and `beast::insight` metrics arrive at the same collector +`file_log` receiver, and `beast::insight` metrics arrive at the same collector over OTLP (`[insight] server=otel`). The single-agent shape is the point: one process, one config file, one place to add redaction or tier tagging. @@ -422,7 +422,7 @@ flowchart TB insight["Beast Insight + XRPL_METRIC_*
native OTLP metrics"] end - otelc["OTel Collector
receivers: otlp, filelog
connector: spanmetrics
3 pipelines"] + otelc["OTel Collector
receivers: otlp, file_log
connector: spanmetrics
3 pipelines"] subgraph storage["Storage"] tempo[("Tempo")] @@ -433,11 +433,11 @@ flowchart TB dashboards["Grafana
Tempo to Loki via tracesToLogs
Loki to Tempo via derived fields"] otel -->|"OTLP/HTTP :4318"| otelc - journal -->|"filelog tails
/var/log/xrpld"| otelc + journal -->|"file_log tails
/var/log/xrpld"| otelc insight -->|"OTLP/HTTP :4318"| otelc otelc -->|"otlp/tempo"| tempo - otelc -->|"otlphttp/loki"| loki + otelc -->|"otlp_http/loki"| loki otelc -->|"prometheus :8889"| prom tempo --> dashboards @@ -459,7 +459,7 @@ flowchart TB **Reading the diagram:** - **xrpld Node (three signals, one transport)**: spans and metrics both leave over OTLP/HTTP on port 4318. Logs do not leave the node at all — the node just writes `debug.log`, and the journal sink prefixes `trace_id=`/`span_id=` whenever a span is active (`Log.cpp:304-338`). -- **OTel Collector (single agent)**: an `otlp` receiver takes spans and metrics; a `filelog` receiver tails `/var/log/xrpld/*/debug.log` and regex-parses the trace/span IDs out of each line. A `spanmetrics` connector derives RED metrics from the trace stream and feeds them into the metrics pipeline. Three pipelines, three exporters — see [05 §5.5.1](./05-configuration-reference.md). +- **OTel Collector (single agent)**: an `otlp` receiver takes spans and metrics; a `file_log` receiver tails `/var/log/xrpld/*/debug.log` and regex-parses the trace/span IDs out of each line. A `spanmetrics` connector derives RED metrics from the trace stream and feeds them into the metrics pipeline. Three pipelines, three exporters — see [05 §5.5.1](./05-configuration-reference.md). - **PerfLog is not in this picture.** It still writes `perf.log`, but nothing collects it and it carries no trace ID; the `setTraceId` hook once planned for it was never built ([02 §2.6.5](./02-design-decisions.md)). - **StatsD is not in this picture either.** It remains a supported `[insight] server=` choice, but selecting it takes metrics _out_ of this pipeline and requires a StatsD receiver you would have to add yourself — the compose file's StatsD port mapping is commented out. - **Grafana**: correlation is bidirectional and configured in the datasources, not in a bespoke panel — Tempo's `tracesToLogs` (`filterByTraceID: true`) jumps trace → logs, and `loki.yaml`'s derived fields jump log → trace. @@ -471,7 +471,7 @@ flowchart TB | **Trace** | `trace_id` | Logs | **Live.** Tempo `tracesToLogs`, `filterByTraceID: true` | | **Trace** | `tx_hash` | — | Live as a span attribute for search; **not** used as a cross-signal join key (`tags: []`) | | **Trace** | `ledger_seq` | — | Live as a span attribute; not a join key | -| **Journal log** | `trace_id`, `span_id` | Traces | **Live.** Emitted by `Log.cpp:304-338` into `debug.log`, parsed by the collector's `filelog` receiver, jumped via `loki.yaml` derived fields | +| **Journal log** | `trace_id`, `span_id` | Traces | **Live.** Emitted by `Log.cpp:304-338` into `debug.log`, parsed by the collector's `file_log` receiver, jumped via `loki.yaml` derived fields | | **PerfLog** | `trace_id` | Traces | **Not implemented.** PerfLog output has no trace ID; the planned `setTraceId` hook was never built. Use the journal log instead | | **Insight** | `exemplar.trace_id` | Traces | **Not implemented.** No exemplar configuration exists anywhere in the code or collector config — no `exemplar_filter` on the SDK side, no `exemplarTraceIdDestinations` on the Prometheus datasource. Metric spike → trace jumps must be done by time range today | diff --git a/docker/telemetry/alloy/config.alloy b/docker/telemetry/alloy/config.alloy index 788134fedf..3135b6149a 100644 --- a/docker/telemetry/alloy/config.alloy +++ b/docker/telemetry/alloy/config.alloy @@ -17,6 +17,15 @@ // See docker/telemetry/otel-collector-config.grafanacloud.yaml for the // authoritative collector equivalent; keep the dimension list in sync with it. // +// Logs are tailed here too, so all three signals leave a node through one +// exporter with one resource identity. The reference collector reads the log +// file with otelcol's own filelog receiver; Alloy's equivalent +// (otelcol.receiver.filelog) is still public-preview and refuses to load +// without --stability.level=public-preview on the alloy service, so the Loki +// source components are used instead and bridged into OTLP by +// otelcol.receiver.loki. Every component below is generally-available, so the +// service needs no extra flag. +// // PIPELINE // // HOST / SYSTEMD METRICS: @@ -30,6 +39,15 @@ // ▼ // exporter.otlphttp (GC OTLP gateway) // +// xrpld LOGS (debug.log): +// file_match ─▶ loki.source.file ─▶ loki.process ─▶ receiver.loki +// (parse+label) │ OTLP logs +// ▼ +// processor.transform.tier ─▶ processor.batch +// │ +// ▼ +// exporter.otlphttp (GC OTLP gateway) +// // The Grafana Cloud OTLP gateway converts OTLP resource attributes to // Prometheus labels server-side, so no otelcol.exporter.prometheus is needed. // @@ -44,11 +62,20 @@ // GRAFANACLOUD_OTLP_URL OTLP/HTTP gateway URL, including the /otlp path // GRAFANACLOUD_OTLP_USER OTLP basic-auth username (numeric stack id) // GRAFANACLOUD_OTLP_KEY OTLP basic-auth password (access token) -// XRPLD_HOST_LABEL host label for this node's scraped metrics +// XRPLD_HOST_LABEL host label for this node's scraped metrics, and +// the service.instance.id stamped on its logs. It +// must equal the node's [telemetry] +// service_instance_id, or logs carry a node name +// that no trace or metric shares and nothing joins. +// Optional: +// XRPLD_LOG_GLOB debug.log path to tail. Defaults to +// /space/xrpld/log/debug.log, the devnet layout. // // PER-DEPLOYMENT EDITS: the deployment.environment and xrpl.network.type tier // values in otelcol.processor.transform are literals (OTTL cannot read env -// vars) -- edit them to match this node's tier and network. +// vars) -- edit them to match this node's tier and network. The log path is the +// exception: it is built in River, where sys.env does expand, and concatenated +// into the OTTL statement. logging { level = "info" @@ -134,6 +161,68 @@ otelcol.receiver.otlp "xrpld" { // * xrpl.network.type -> set only when absent (don't overwrite the node's // own value). OTTL `where ... == nil` gives insert (not upsert) semantics. // * telemetry.sdk.* -> deleted (SDK noise). +// =========================================================================== +// xrpld LOGS (debug.log --> OTLP gateway) +// =========================================================================== + +// Tail the node's debug.log. devnet writes a single flat file +// (/space/xrpld/log/debug.log) rather than the per-node subdirectory the docker +// stack uses, so identity cannot be read off the path here -- it comes from +// XRPLD_HOST_LABEL below. Override XRPLD_LOG_GLOB for a different layout. +local.file_match "xrpld_logs" { + path_targets = [{ + __path__ = coalesce(sys.env("XRPLD_LOG_GLOB"), "/space/xrpld/log/debug.log"), + }] +} + +loki.source.file "xrpld_logs" { + targets = local.file_match.xrpld_logs.targets + forward_to = [loki.process.xrpld_logs.receiver] +} + +// Parse the line Logs::format() emits: +// YYYY-Mmm-DD HH:MM:SS.fffffffff UTC : [trace_id=... span_id=...] +// The `partition:` prefix is omitted when the partition is empty, so that group +// is optional, and trace_id/span_id are present only for a sampled span. +loki.process "xrpld_logs" { + forward_to = [otelcol.receiver.loki.xrpld.receiver] + + stage.regex { + expression = "^(?P\\S+\\s+\\S+)\\s+\\S+\\s+(?:(?P\\S+):)?(?P\\S+)\\s+(?:trace_id=(?P[a-f0-9]+)\\s+span_id=(?P[a-f0-9]+)\\s+)?" + } + + // Use the node's own timestamp, not ingest time, or a log line cannot be + // lined up with the span it belongs to. The node emits nanosecond precision; + // the fractional part of this layout accepts any number of digits. + stage.timestamp { + source = "log_time" + format = "2006-Jan-02 15:04:05.999999999" + location = "UTC" + } + + // A regex capture stays in the pipeline's extracted map and never reaches the + // entry unless a stage attaches it. Structured metadata keeps these queryable + // without making any of them an indexed label. + stage.structured_metadata { + values = { + partition = "", + severity = "", + trace_id = "", + span_id = "", + } + } +} + +// Bridge the Loki entries into OTLP so logs use the same exporter, and get the +// same resource identity, as traces and metrics. Note this receiver produces an +// EMPTY resource and puts everything on the log record, so the resource +// attributes are set in processor.transform below. +otelcol.receiver.loki "xrpld" { + output { + logs = [otelcol.processor.transform.tier.input] + } +} + otelcol.processor.transform "tier" { error_mode = "ignore" @@ -166,6 +255,40 @@ otelcol.processor.transform "tier" { ] } + // Logs use `log` context, not `resource`, for two reasons: receiver.loki + // hands over an empty resource, and the fields to promote (severity, trace + // ids) live on the record. Each Alloy instance serves one node, so writing a + // resource attribute from a record is unambiguous here. + // + // service.instance.id is concatenated in from XRPLD_HOST_LABEL because OTTL + // has no env() converter. Loki promotes only an allow-listed set of RESOURCE + // attributes to indexed stream labels, and service.instance.id is on that + // list, so it arrives as the label service_instance_id that the dashboards + // filter on. Setting it as a record attribute instead would make it + // structured metadata, which cannot be used in a {...} stream selector. + log_statements { + context = "log" + statements = [ + "set(resource.attributes[\"service.instance.id\"], \"" + coalesce(sys.env("XRPLD_HOST_LABEL"), "unknown") + "\")", + `set(resource.attributes["service.name"], "xrpld")`, + `set(resource.attributes["deployment.environment"], "prod")`, + `set(resource.attributes["xrpl.network.type"], "mainnet") where resource.attributes["xrpl.network.type"] == nil`, + // Promote the parsed fields onto the first-class OTLP record fields, so + // Grafana links a log line to its trace natively instead of re-parsing + // the body. The guards matter: most lines are emitted outside a sampled + // span and must keep an empty trace id rather than an invalid one. + `set(severity_text, attributes["severity"]) where attributes["severity"] != nil`, + `set(trace_id.string, attributes["trace_id"]) where attributes["trace_id"] != nil`, + `set(span_id.string, attributes["span_id"]) where attributes["span_id"] != nil`, + // Drop the bridge's own bookkeeping so it does not become structured + // metadata on every line. + `delete_key(attributes, "filename")`, + `delete_key(attributes, "loki.attribute.labels")`, + `delete_key(attributes, "log.file.path")`, + `delete_key(attributes, "log.file.name")`, + ] + } + output { // Traces fan out: to the batch/gateway path AND into the spanmetrics // connector so the RED metrics are derived from the same tagged spans. @@ -175,6 +298,7 @@ otelcol.processor.transform "tier" { ] // Native metrics go straight to the batch/gateway path. metrics = [otelcol.processor.batch.xrpld.input] + logs = [otelcol.processor.batch.xrpld.input] } } @@ -252,6 +376,7 @@ otelcol.processor.batch "xrpld" { output { traces = [otelcol.exporter.otlphttp.grafanacloud.input] metrics = [otelcol.exporter.otlphttp.grafanacloud.input] + logs = [otelcol.exporter.otlphttp.grafanacloud.input] } } diff --git a/docker/telemetry/grafana/dashboards/log-derived-insights.json b/docker/telemetry/grafana/dashboards/log-derived-insights.json index 2fa1b84eb3..dad20fc542 100644 --- a/docker/telemetry/grafana/dashboards/log-derived-insights.json +++ b/docker/telemetry/grafana/dashboards/log-derived-insights.json @@ -1035,7 +1035,7 @@ { "type": "timeseries", "title": "Log Line Rate By Severity", - "description": "###### What this is:\n*Rate of log lines emitted by xrpld, split by severity.*\n\n###### How it's computed:\n*Per-second count of matching log lines grouped by the severity field parsed out of each line.*\n\n###### Reading it:\n*Use this to confirm the log pipeline is alive, and to see at a glance whether DBG lines are being collected at all.*\n\n###### Healthy range:\n*Workload-dependent. If the DBG series is absent, every panel in a [DBG] row on this dashboard will be empty.*\n\n###### Watch for:\n*A sudden collapse to only WRN and ERR, which means debug logging was turned off and the [DBG] rows have gone blind rather than quiet.*\n\n###### Keywords:\n- **Severity** *(per line)* — xrpld log level: DBG, NFO, WRN, ERR, FTL.\n- **Structured metadata** *(per line)* — Loki fields parsed from the line, filtered with `|` rather than in the stream selector.\n\n###### Computation boundary:\n*Result: Per node per severity — a count of log lines, not of events in the node.*\n*Derived in the Grafana query; the collector's filelog receiver parses severity, xrpld itself exports no such metric.*\n\n###### Source:\n[Log.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/basics/Log.cpp)\n\n###### Function:\n`Logs::Sink::write`\n\n###### References:\n[Loki structured metadata](https://grafana.com/docs/loki/latest/get-started/labels/structured-metadata/)", + "description": "###### What this is:\n*Rate of log lines emitted by xrpld, split by severity.*\n\n###### How it's computed:\n*Per-second count of matching log lines grouped by the severity field parsed out of each line.*\n\n###### Reading it:\n*Use this to confirm the log pipeline is alive, and to see at a glance whether DBG lines are being collected at all.*\n\n###### Healthy range:\n*Workload-dependent. If the DBG series is absent, every panel in a [DBG] row on this dashboard will be empty.*\n\n###### Watch for:\n*A sudden collapse to only WRN and ERR, which means debug logging was turned off and the [DBG] rows have gone blind rather than quiet.*\n\n###### Keywords:\n- **Severity** *(per line)* — xrpld log level: DBG, NFO, WRN, ERR, FTL.\n- **Structured metadata** *(per line)* — Loki fields parsed from the line, filtered with `|` rather than in the stream selector.\n\n###### Computation boundary:\n*Result: Per node per severity — a count of log lines, not of events in the node.*\n*Derived in the Grafana query; the collector's file_log receiver parses severity, xrpld itself exports no such metric.*\n\n###### Source:\n[Log.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/basics/Log.cpp)\n\n###### Function:\n`Logs::Sink::write`\n\n###### References:\n[Loki structured metadata](https://grafana.com/docs/loki/latest/get-started/labels/structured-metadata/)", "gridPos": { "h": 10, "w": 12, diff --git a/docker/telemetry/otel-collector-config.grafanacloud.yaml b/docker/telemetry/otel-collector-config.grafanacloud.yaml index 84fd440849..21ea1804c7 100644 --- a/docker/telemetry/otel-collector-config.grafanacloud.yaml +++ b/docker/telemetry/otel-collector-config.grafanacloud.yaml @@ -38,9 +38,14 @@ receivers: endpoint: 0.0.0.0:4317 http: endpoint: 0.0.0.0:4318 - filelog: + file_log: include: - /var/log/xrpld/*/debug.log + # Needed to recover which node a line came from. The subdirectory name is + # the only per-node signal in the log stream: Logs::format() writes + # trace_id and span_id but no node identity. Emitters name the directory + # after their own service_instance_id so the two agree. + include_file_path: true operators: - type: regex_parser regex: '^(?P\S+\s+\S+)\s+\S+\s+(?:(?P\S+):)?(?P\S+)\s+(?:trace_id=(?P[a-f0-9]+)\s+span_id=(?P[a-f0-9]+)\s+)?(?P.*)$' @@ -48,6 +53,26 @@ receivers: parse_from: attributes.timestamp layout: "%Y-%b-%d %H:%M:%S.%f" location: UTC + # Lift the per-node directory out of the file path and onto the + # RESOURCE. include_file_path alone is not enough: it produces a log + # RECORD attribute, and on OTLP ingest Loki promotes only an allow-list + # of RESOURCE attributes to indexed stream labels. A record attribute + # becomes structured metadata, which cannot be used in a {...} selector. + # service.instance.id is on that allow-list and arrives as the LogQL + # label service_instance_id, which is the label the dashboards filter on. + # Dotted keys need bracket syntax; dot notation would be read as a + # nested traversal and match nothing. + - type: regex_parser + parse_from: attributes["log.file.path"] + parse_to: attributes + regex: "^/var/log/xrpld/(?P[^/]+)/" + - type: move + from: attributes.node_dir + to: resource["service.instance.id"] + # Drop the raw path once the node name is on the resource. Keeping it + # would add a structured-metadata field to every line for no benefit. + - type: remove + field: attributes["log.file.path"] processors: batch: @@ -222,7 +247,7 @@ exporters: endpoint: tempo:4317 tls: insecure: true - otlphttp/loki: + otlp_http/loki: endpoint: http://loki:3100/otlp prometheus: endpoint: 0.0.0.0:8889 @@ -235,7 +260,7 @@ exporters: # Single OTLP/HTTP exporter to Grafana Cloud. The gateway fans the three # signals out to hosted Tempo (traces), Mimir/Prometheus (metrics), and # Loki (logs). Retry + queue guard against transient gateway errors. - otlphttp/grafanacloud: + otlp_http/grafanacloud: endpoint: ${env:GRAFANA_CLOUD_OTLP_ENDPOINT} auth: authenticator: basicauth/grafanacloud @@ -248,7 +273,7 @@ service: extensions: [health_check, basicauth/grafanacloud] pipelines: # Each pipeline keeps its local exporter(s) AND adds Grafana Cloud. - # For cloud-only, drop debug/otlp/tempo, prometheus, and otlphttp/loki + # For cloud-only, drop debug/otlp/tempo, prometheus, and otlp_http/loki # from the respective exporter lists. # 100% of spans feed the spanmetrics connector so span-derived RED # metrics stay exact. No tail sampling on this branch. @@ -261,7 +286,7 @@ service: traces/store: receivers: [otlp] processors: [tail_sampling, resource/tier, resource/stripsdk, batch] - exporters: [otlp/tempo, otlphttp/grafanacloud] + exporters: [otlp/tempo, otlp_http/grafanacloud] # The local Prometheus scrape promotes tier/instance resource attrs to # labels via resource_to_telemetry_conversion; Grafana Cloud (OTLP) does # not, so it runs a separate pipeline that copies them onto datapoint @@ -275,8 +300,8 @@ service: receivers: [otlp, spanmetrics] processors: [resource/tier, resource/stripsdk, transform/cloudlabels, batch] - exporters: [otlphttp/grafanacloud] + exporters: [otlp_http/grafanacloud] logs: - receivers: [filelog] + receivers: [file_log] processors: [resource/logs, resource/tier, resource/stripsdk, batch] - exporters: [otlphttp/loki, otlphttp/grafanacloud] + exporters: [otlp_http/loki, otlp_http/grafanacloud] diff --git a/docker/telemetry/xrpld-telemetry.cfg b/docker/telemetry/xrpld-telemetry.cfg index 76800019a7..2c88ec6bc0 100644 --- a/docker/telemetry/xrpld-telemetry.cfg +++ b/docker/telemetry/xrpld-telemetry.cfg @@ -93,10 +93,15 @@ docker/telemetry/data # --- Logging ---------------------------------------------------------------- # Path is resolved relative to this config file's directory (docker/telemetry), -# so this writes to docker/telemetry/data/logs/devnet/debug.log — the same +# so this writes to docker/telemetry/data/logs/xrpld-devnet/debug.log — the same # dir the compose stack bind-mounts into the collector as /var/log/xrpld. +# +# The subdirectory name must equal [telemetry] service_instance_id below. The +# collector reads it off the file path and stamps it as the Loki label +# service_instance_id, so a mismatch here means log lines carry a node name +# that no trace or metric shares, and nothing joins. [debug_logfile] -data/logs/devnet/debug.log +data/logs/xrpld-devnet/debug.log [rpc_startup] { "command": "log_level", "severity": "debug" } diff --git a/docs/telemetry-runbook.md b/docs/telemetry-runbook.md index e1dc0ef7ad..20e37bfdbf 100644 --- a/docs/telemetry-runbook.md +++ b/docs/telemetry-runbook.md @@ -109,7 +109,7 @@ Both set `[insight] server=otel` (native metrics → collector → Prometheus, w drives the dashboards) and `service_instance_id`, exposed by Prometheus as the `service_instance_id` label that the `$node` dashboard variable filters on. The mainnet config logs to `/var/log/xrpld/mainnet/debug.log` — the path -the collector's filelog receiver tails for log-trace correlation. +the collector's file_log receiver tails for log-trace correlation. Metrics begin flowing as soon as the node connects to peers (`server_state` ≥ `connected`); full ledger and consensus panels populate after sync @@ -208,12 +208,12 @@ To return to local-only export, bring the stack up with just the base The prepared config **dual-exports**: data goes to both the local stack and Grafana Cloud, so the on-box backends remain a fallback. For cloud-only, remove the local exporters (`debug`, `otlp/tempo`, `prometheus`, -`otlphttp/loki`) from the respective pipelines in +`otlp_http/loki`) from the respective pipelines in `otel-collector-config.grafanacloud.yaml`, leaving only -`otlphttp/grafanacloud`. +`otlp_http/grafanacloud`. > **Note**: shipping logs to Grafana Cloud requires keeping xrpld file -> logging on (at least `warning` level) so the collector's filelog receiver +> logging on (at least `warning` level) so the collector's file_log receiver > has a `debug.log` to tail. Traces and metrics are unaffected by log level. ### Importing dashboards to Grafana Cloud @@ -2815,7 +2815,7 @@ This enables bidirectional navigation between logs and traces in Grafana: ### Log Ingestion Pipeline -Log files are ingested by the OTel Collector's `filelog` receiver, which tails `debug.log` files and parses them with a regex that extracts `timestamp`, `partition`, `severity`, `trace_id`, `span_id`, and `message` fields. Parsed entries are exported to Grafana Loki. +Log files are ingested by the OTel Collector's `file_log` receiver, which tails `debug.log` files and parses them with a regex that extracts `timestamp`, `partition`, `severity`, `trace_id`, `span_id`, and `message` fields. Parsed entries are exported to Grafana Loki. The receiver tails `/var/log/xrpld/*/debug.log` inside the collector container. docker-compose bind-mounts the host log root there; the source defaults to the repo-relative `docker/telemetry/data/logs`, which the telemetry configs write to (`data/logs//debug.log`) and which needs no root. To tail logs from elsewhere, set `XRPLD_LOG_DIR` before `docker compose up` (the integration test does this to point at its own workdir). The single trailing `*` matches one per-network or per-node subdirectory. @@ -2850,7 +2850,7 @@ after the selector and cannot be discovered by `label_values()`. # Logs from the last hour containing trace context. `partition`, `severity`, and # `trace_id` are already parsed into structured metadata by the collector's -# filelog receiver, so re-extracting them with regexp is unnecessary work. +# file_log receiver, so re-extracting them with regexp is unnecessary work. {service_name="xrpld"} | trace_id != "" # Count of traced vs untraced log lines @@ -3563,9 +3563,9 @@ not a sign the cache is working. ### No logs in Loki - Verify the log file mount in docker-compose.yml points to the correct xrpld log directory (default source `docker/telemetry/data/logs`, or the `XRPLD_LOG_DIR` override) and that xrpld actually writes `debug.log` there -- Check OTel Collector logs for filelog receiver errors: `docker compose logs otel-collector` +- Check OTel Collector logs for file_log receiver errors: `docker compose logs otel-collector` - Verify Loki is running: `curl http://localhost:3100/ready` -- Check the filelog receiver glob `/var/log/xrpld/*/debug.log` matches your log layout — the log file must sit one subdirectory below the mount root +- Check the file_log receiver glob `/var/log/xrpld/*/debug.log` matches your log layout — the log file must sit one subdirectory below the mount root ## Performance Tuning From ac07e1345f765446a353ac74376bbcdc8e62301e Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Wed, 9 Sep 2026 15:12:36 +0100 Subject: [PATCH 15/16] fix(telemetry): name the harness log directories after their instance ids The collector reads the per-node directory off the log file path and stamps it as the Loki label service_instance_id, so the directory name has to equal the node's own [telemetry] service_instance_id or log lines carry a node name that no trace or metric shares and nothing joins. Both harness scripts disagreed with themselves: run-full-validation.sh wrote to node$i while setting validator-${i}, and benchmark.sh wrote to node$i while setting bench-node-${i}. Rename the directories to match the ids rather than the reverse, so no existing trace or metric label value moves and no harness expectation has to be re-checked. Only path references are renamed; the human-readable "node$i" in log and error messages is left as prose. The config template is not rendered by any script, so its DATA_DIR documentation gains a note about the same constraint instead. Also rename the deprecated otlphttp/filelog collector component names in the harness scripts and docs. --- docker/telemetry/docker-compose.workload.yaml | 4 +-- docker/telemetry/workload/README.md | 8 +++--- docker/telemetry/workload/benchmark.sh | 6 ++-- .../telemetry/workload/run-full-validation.sh | 28 +++++++++---------- .../telemetry/workload/validate_telemetry.py | 2 +- .../workload/xrpld-validator.cfg.template | 7 ++++- docs/telemetry-runbook.md | 22 +++++++-------- 7 files changed, 41 insertions(+), 36 deletions(-) diff --git a/docker/telemetry/docker-compose.workload.yaml b/docker/telemetry/docker-compose.workload.yaml index 6ab809d705..5e81570f95 100644 --- a/docker/telemetry/docker-compose.workload.yaml +++ b/docker/telemetry/docker-compose.workload.yaml @@ -11,7 +11,7 @@ # run-full-validation.sh starts NUM_NODES (default 5) xrpld instances on # 127.0.0.1, each with a cfg it generates inline, peered to each other via # [ips_fixed]. They reach the collector through the published ports below and -# write their logs into the bind-mounted workdir for the filelog receiver. +# write their logs into the bind-mounted workdir for the file_log receiver. # # Usage: # # Start the telemetry backend on its own: @@ -47,7 +47,7 @@ services: - "13133:13133" # Health check volumes: - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml:ro - # Mount the validation workdir so the filelog receiver can tail node + # Mount the validation workdir so the file_log receiver can tail node # logs. run-full-validation.sh sets XRPLD_LOG_DIR to its workdir; the # default matches that workdir so a bare `docker compose up` also works. - ${XRPLD_LOG_DIR:-/tmp/xrpld-validation}:/var/log/xrpld:ro diff --git a/docker/telemetry/workload/README.md b/docker/telemetry/workload/README.md index 07c3bacb0a..185baed713 100644 --- a/docker/telemetry/workload/README.md +++ b/docker/telemetry/workload/README.md @@ -32,7 +32,7 @@ run-full-validation.sh (shell orchestrator) | |-- docker-compose.workload.yaml | |-- otel-collector (otlp receiver: traces + beast::insight metrics; - | | filelog receiver: node debug.log -> Loki) + | | file_log receiver: node debug.log -> Loki) | |-- tempo (trace backend + TraceQL search API) | |-- prometheus (metrics scraping) | |-- loki (log aggregation for log-trace correlation) @@ -458,7 +458,7 @@ its own `check_log_correlation()`, but no workflow runs that script. Correlation depends on four independent legs, and a failed check on its own names none of them: the node must write a `debug.log` line carrying trace ids, the -collector container must see that file, its `filelog` receiver must parse and +collector container must see that file, its `file_log` receiver must parse and export the line, and Loki must return it for the validator's LogQL. `run-full-validation.sh` prints a per-leg diagnostic after the suite whenever the Loki checks are enabled — per-node correlated-line counts and severity mix, the @@ -467,7 +467,7 @@ internal log-record counters, and Loki's own entry counts for the selector with and without the line filter. Read that block first; it identifies the broken leg without reproducing anything. -Those two entry counts **must** be wrapped in `sum()`. The `filelog` receiver's +Those two entry counts **must** be wrapped in `sum()`. The `file_log` receiver's `regex_parser` leaves `message` and `timestamp` as log-record attributes, and Loki's OTLP path stores them as structured metadata that joins the label set of a metric query — so an unaggregated `count_over_time` returns one series per log @@ -510,7 +510,7 @@ docker/telemetry/workload/run-full-validation.sh --xrpld .build/xrpld ``` Re-run it after any change to log formatting, span activation, the collector's -`filelog` receiver, or the Loki exporter. +`file_log` receiver, or the Loki exporter. ### Pathfinding is not exercised diff --git a/docker/telemetry/workload/benchmark.sh b/docker/telemetry/workload/benchmark.sh index da22b350b1..7035a2bc43 100755 --- a/docker/telemetry/workload/benchmark.sh +++ b/docker/telemetry/workload/benchmark.sh @@ -189,7 +189,7 @@ start_cluster() { # Build per-node configs. for i in $(seq 1 "$NUM_NODES"); do - local node_dir="$WORKDIR/node$i" + local node_dir="$WORKDIR/bench-node-$i" mkdir -p "$node_dir/nudb" "$node_dir/db" || cannot_measure "Could not create node$i directories under $node_dir" @@ -361,7 +361,7 @@ stop_cluster() { log "Stopping cluster..." for i in $(seq 1 "$NUM_NODES"); do - local pidfile="$WORKDIR/node$i/xrpld.pid" + local pidfile="$WORKDIR/bench-node-$i/xrpld.pid" if [ -f "$pidfile" ]; then kill "$(cat "$pidfile")" 2>/dev/null || true fi @@ -422,7 +422,7 @@ ws_endpoints() { node_pids_csv() { local i out="" pid for i in $(seq 1 "$NUM_NODES"); do - pid=$(cat "$WORKDIR/node$i/xrpld.pid" 2>/dev/null) || continue + pid=$(cat "$WORKDIR/bench-node-$i/xrpld.pid" 2>/dev/null) || continue [ -n "$pid" ] && out="$out,$pid" done printf '%s' "${out#,}" diff --git a/docker/telemetry/workload/run-full-validation.sh b/docker/telemetry/workload/run-full-validation.sh index 44222deee2..59c7b1b372 100755 --- a/docker/telemetry/workload/run-full-validation.sh +++ b/docker/telemetry/workload/run-full-validation.sh @@ -266,7 +266,7 @@ mkdir -p "$WORKDIR" "$REPORT_DIR" || die "Could not create $WORKDIR and $REPORT_ # Step 1: Start observability stack # --------------------------------------------------------------------------- log "Step 1: Starting observability stack..." -# Point the collector's log mount at this run's workdir so the filelog +# Point the collector's log mount at this run's workdir so the file_log # receiver tails the per-node debug.log files generated below. XRPLD_LOG_DIR="$WORKDIR" docker compose -f "$COMPOSE_FILE" up -d || die "docker compose up failed for $COMPOSE_FILE — the observability stack did not start" @@ -311,7 +311,7 @@ bash "$SCRIPT_DIR/generate-validator-keys.sh" "$XRPLD" "$NUM_NODES" "$WORKDIR" | die "generate-validator-keys.sh failed — no validator keys for the $NUM_NODES-node cluster" for i in $(seq 1 "$NUM_NODES"); do - NODE_DIR="$WORKDIR/node$i" + NODE_DIR="$WORKDIR/validator-$i" mkdir -p "$NODE_DIR/nudb" "$NODE_DIR/db" || die "Could not create node$i directories under $NODE_DIR" RPC_PORT=$((RPC_PORT_BASE + i - 1)) @@ -478,15 +478,15 @@ node_running() { report_stopped_nodes() { local i pid status for i in $(seq 1 "$NUM_NODES"); do - pid=$(cat "$WORKDIR/node$i/xrpld.pid" 2>/dev/null || echo "") + pid=$(cat "$WORKDIR/validator-$i/xrpld.pid" 2>/dev/null || echo "") [ -n "$pid" ] || continue node_running "$pid" && continue status=0 wait "$pid" 2>/dev/null || status=$? warn "node$i (pid $pid) is not running — wait status $status" - if [ -s "$WORKDIR/node$i/stdout.log" ]; then + if [ -s "$WORKDIR/validator-$i/stdout.log" ]; then warn "node$i last output:" - tail -n 15 "$WORKDIR/node$i/stdout.log" | sed 's/^/ /' >&2 + tail -n 15 "$WORKDIR/validator-$i/stdout.log" | sed 's/^/ /' >&2 else warn "node$i wrote no stdout at all" fi @@ -606,7 +606,7 @@ fi # --------------------------------------------------------------------------- # Log-trace correlation has four legs and a failed check names none of them: # the node must write a debug.log line carrying trace ids, the collector -# container must see that file, its filelog receiver must parse and export the +# container must see that file, its file_log receiver must parse and export the # line, and Loki must return it for the validator's own LogQL. Each leg below # reports what it observed, so a reader with only the CI log can tell which one # broke instead of guessing. @@ -704,7 +704,7 @@ diag_node_logs() { local i log bytes total correlated sample echo " [leg 1/4 node] debug.log lines matching '$DIAG_TRACE_RE'" for i in $(seq 1 "$NUM_NODES"); do - log="$WORKDIR/node$i/debug.log" + log="$WORKDIR/validator-$i/debug.log" if [ ! -f "$log" ]; then echo " node$i: no debug.log at $log — the node never opened its log sink" continue @@ -786,10 +786,10 @@ diag_collector_mount() { sed 's/^/ /' || echo " (container-side listing failed)" } -# Leg 3 — collector: did the filelog receiver parse and export those lines? +# Leg 3 — collector: did the file_log receiver parse and export those lines? # # Two independent readings. The collector's own stderr names every file the -# receiver opened and carries any filelog parse or Loki export error. Its +# receiver opened and carries any file_log parse or Loki export error. Its # internal telemetry counts log records in and out: accepted>0 with sent=0 is # an export failure, accepted=0 while files are being watched is a parse # failure. @@ -801,7 +801,7 @@ diag_collector_mount() { # exists; when it reports nothing matching, the leg says so. diag_collector_pipeline() { local cid img watched problems metrics - echo " [leg 3/4 collector] filelog receiver state" + echo " [leg 3/4 collector] file_log receiver state" if ! command -v docker >/dev/null 2>&1; then echo " docker is not on PATH — leg skipped" return 0 @@ -822,12 +822,12 @@ diag_collector_pipeline() { # Second filter keys on the collector's own logs-pipeline markers so this # does not report warnings from the trace or metric pipelines. Nothing is # excluded beyond that: the collector's benign config-alias deprecation - # notices ("filelog" -> "file_log") do surface here, and suppressing lines + # notices ("file_log" -> "file_log") do surface here, and suppressing lines # because they are usually harmless is how a diagnostic hides the one that # was not. problems=$(diag_run docker logs "$cid" 2>&1 | grep -iE '(warn|error)' | - grep -iE 'filelog|fileconsumer|loki|signal": *"logs' | + grep -iE 'file_log|fileconsumer|loki|signal": *"logs' | tail -n 20 || true) if [ -n "$problems" ]; then echo " logs-pipeline warnings and errors (last 20):" @@ -869,7 +869,7 @@ diag_loki_stream() { [ -n "$selector" ] || selector="$DIAG_LOG_SELECTOR" [ -n "$correlation" ] || correlation="$DIAG_LOG_SELECTOR $DIAG_LOG_FILTER" # sum() is required, for the reason recorded at _log_loki_diagnostics in - # validate_telemetry.py: the filelog regex_parser leaves message/timestamp + # validate_telemetry.py: the file_log regex_parser leaves message/timestamp # as log-record attributes, Loki's OTLP path turns those into structured # metadata that joins a metric query's label set, so an unaggregated # count_over_time yields one series per log line and Loki rejects the query @@ -1076,7 +1076,7 @@ echo " xrpld nodes ($NUM_NODES) are running:" for i in $(seq 1 "$NUM_NODES"); do rpc=$((RPC_PORT_BASE + i - 1)) ws=$((WS_PORT_BASE + i - 1)) - pid=$(cat "$WORKDIR/node$i/xrpld.pid" 2>/dev/null || echo 'unknown') + pid=$(cat "$WORKDIR/validator-$i/xrpld.pid" 2>/dev/null || echo 'unknown') echo " Node $i: RPC=$rpc WS=$ws PID=$pid" done echo "" diff --git a/docker/telemetry/workload/validate_telemetry.py b/docker/telemetry/workload/validate_telemetry.py index 2b7b79eb23..988fe75f8f 100644 --- a/docker/telemetry/workload/validate_telemetry.py +++ b/docker/telemetry/workload/validate_telemetry.py @@ -1680,7 +1680,7 @@ async def _log_loki_diagnostics(session: aiohttp.ClientSession, loki_url: str) - "Loki diagnostic: service_name values: %s", ", ".join(found) or "(none)" ) - # sum() is load-bearing, not cosmetic. The filelog receiver's regex_parser + # sum() is load-bearing, not cosmetic. The file_log receiver's regex_parser # leaves message, timestamp, trace_id and span_id as log-record attributes, # and Loki's OTLP path stores those as structured metadata, which joins the # label set of a metric query. Because `message` and `timestamp` are unique diff --git a/docker/telemetry/workload/xrpld-validator.cfg.template b/docker/telemetry/workload/xrpld-validator.cfg.template index 623b781707..adeccea97a 100644 --- a/docker/telemetry/workload/xrpld-validator.cfg.template +++ b/docker/telemetry/workload/xrpld-validator.cfg.template @@ -14,7 +14,12 @@ # {{RPC_PORT}} — HTTP RPC port # {{WS_PORT}} — WebSocket port # {{PEER_PORT}} — Peer protocol port -# {{DATA_DIR}} — Node data directory +# {{DATA_DIR}} — Node data directory. Its last path segment must +# equal service_instance_id below: the collector's +# file_log receiver reads that segment off the log +# file path and stamps it as the Loki label +# service_instance_id, so a mismatch gives log lines +# a node name no trace or metric shares. # {{VALIDATION_SEED}} — Validator seed from key generation # {{VALIDATORS_FILE}} — Path to shared validators.txt # {{IPS_FIXED}} — Peer addresses (one per line) diff --git a/docs/telemetry-runbook.md b/docs/telemetry-runbook.md index 2d85f4e99f..1ddad3ec6c 100644 --- a/docs/telemetry-runbook.md +++ b/docs/telemetry-runbook.md @@ -111,7 +111,7 @@ Both set `[insight] server=otel` (native metrics → collector → Prometheus, w drives the dashboards) and `service_instance_id`, exposed by Prometheus as the `service_instance_id` label that the `$node` dashboard variable filters on. The mainnet config logs to `/var/log/xrpld/mainnet/debug.log` — the path -the collector's filelog receiver tails for log-trace correlation. +the collector's file_log receiver tails for log-trace correlation. Metrics begin flowing as soon as the node connects to peers (`server_state` ≥ `connected`); full ledger and consensus panels populate after sync @@ -210,12 +210,12 @@ To return to local-only export, bring the stack up with just the base The prepared config **dual-exports**: data goes to both the local stack and Grafana Cloud, so the on-box backends remain a fallback. For cloud-only, remove the local exporters (`debug`, `otlp/tempo`, `prometheus`, -`otlphttp/loki`) from the respective pipelines in +`otlp_http/loki`) from the respective pipelines in `otel-collector-config.grafanacloud.yaml`, leaving only -`otlphttp/grafanacloud`. +`otlp_http/grafanacloud`. > **Note**: shipping logs to Grafana Cloud requires keeping xrpld file -> logging on (at least `warning` level) so the collector's filelog receiver +> logging on (at least `warning` level) so the collector's file_log receiver > has a `debug.log` to tail. Traces and metrics are unaffected by log level. ### Importing dashboards to Grafana Cloud @@ -2843,7 +2843,7 @@ The sampled check is normally satisfied on a self-rooted consensus round — hea With all four satisfied, `info` is the minimum level at which the `log.trace_id_present` and `log.trace_id_cross_reference` checks pass by construction, and it is what the correlation-checking harnesses generate: the cfgs written by [run-full-validation.sh](../docker/telemetry/workload/run-full-validation.sh) and [integration-test.sh](../docker/telemetry/integration-test.sh) each set `enabled=1`, `trace_consensus=1` and `log_level info` together. `benchmark.sh` deliberately does not — it stays at `warning` to keep log I/O out of the overhead measurement, and it runs no correlation check. At `warning` and above that pair is suppressed and correlation becomes incidental — dependent on a `warn`-or-worse line happening to fire inside some active span. -> **CI exercises both checks.** `log.trace_id_present` and `log.trace_id_cross_reference` are gated on every CI run — see [CI workflow](#ci-workflow) for the invocation and the per-leg diagnostics printed alongside them. Run the same thing locally after any change to log formatting, span activation, the `filelog` receiver or the Loki exporter: +> **CI exercises both checks.** `log.trace_id_present` and `log.trace_id_cross_reference` are gated on every CI run — see [CI workflow](#ci-workflow) for the invocation and the per-leg diagnostics printed alongside them. Run the same thing locally after any change to log formatting, span activation, the `file_log` receiver or the Loki exporter: > > ```bash > docker/telemetry/workload/run-full-validation.sh --xrpld .build/xrpld @@ -2864,7 +2864,7 @@ log_level RPCHandler debug ### Log Ingestion Pipeline -Log files are ingested by the OTel Collector's `filelog` receiver, which tails `debug.log` files and parses them with a regex that extracts `timestamp`, `partition`, `severity`, `trace_id`, `span_id`, and `message` fields. Parsed entries are exported to Grafana Loki. +Log files are ingested by the OTel Collector's `file_log` receiver, which tails `debug.log` files and parses them with a regex that extracts `timestamp`, `partition`, `severity`, `trace_id`, `span_id`, and `message` fields. Parsed entries are exported to Grafana Loki. The receiver tails `/var/log/xrpld/*/debug.log` inside the collector container. docker-compose bind-mounts the host log root there; the source defaults to the repo-relative `docker/telemetry/data/logs`, which the telemetry configs write to (`data/logs//debug.log`) and which needs no root. To tail logs from elsewhere, set `XRPLD_LOG_DIR` before `docker compose up` (the integration test does this to point at its own workdir). The single trailing `*` matches one per-network or per-node subdirectory. @@ -2899,7 +2899,7 @@ after the selector and cannot be discovered by `label_values()`. # Logs from the last hour containing trace context. `partition`, `severity`, and # `trace_id` are already parsed into structured metadata by the collector's -# filelog receiver, so re-extracting them with regexp is unnecessary work. +# file_log receiver, so re-extracting them with regexp is unnecessary work. {service_name="xrpld"} | trace_id != "" # Count of traced vs untraced log lines @@ -3614,9 +3614,9 @@ not a sign the cache is working. ### No logs in Loki - Verify the log file mount in docker-compose.yml points to the correct xrpld log directory (default source `docker/telemetry/data/logs`, or the `XRPLD_LOG_DIR` override) and that xrpld actually writes `debug.log` there -- Check OTel Collector logs for filelog receiver errors: `docker compose logs otel-collector` +- Check OTel Collector logs for file_log receiver errors: `docker compose logs otel-collector` - Verify Loki is running: `curl http://localhost:3100/ready` -- Check the filelog receiver glob `/var/log/xrpld/*/debug.log` matches your log layout — the log file must sit one subdirectory below the mount root +- Check the file_log receiver glob `/var/log/xrpld/*/debug.log` matches your log layout — the log file must sit one subdirectory below the mount root ## Performance Tuning @@ -3920,12 +3920,12 @@ container as the main CI, so Conan and ccache hit the shared caches), and these checks are enabled: per-node counts of `debug.log` lines carrying the injected `trace_id`/`span_id` shape plus the severity mix, the container-side listing of `/var/log/xrpld` taken with the collector's own mounts and uid, the - `filelog` receiver's watched files, logs-pipeline warnings and internal + `file_log` receiver's watched files, logs-pipeline warnings and internal log-record counters, and Loki's entry counts for the stream selector with and without the line filter. The diagnostics are non-fatal by construction: each leg is isolated and a missing container or unreachable endpoint prints a note. Those two Loki entry counts are `sum(count_over_time(...))`, and the `sum()` is - load-bearing: the `filelog` receiver leaves `message` and `timestamp` as + load-bearing: the `file_log` receiver leaves `message` and `timestamp` as log-record attributes, Loki's OTLP path stores them as structured metadata, and structured metadata joins a metric query's label set — so an unaggregated `count_over_time` produces one series per log line and Loki answers `HTTP 400 From a59dfe33e322043fc4f101dfb33cc788a5fe5cb0 Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Wed, 9 Sep 2026 15:49:04 +0100 Subject: [PATCH 16/16] fix(telemetry): make the snapshot portable and repair the mTLS test set Three CI failures, one cause each. macOS could not compile the tracker: Apple's libc++ has no std::atomic for a shared_ptr, so the primary template's trivially-copyable assert fired. Publish through boost::atomic_shared_ptr instead, which every standard library the matrix covers can build. boost/smart_ptr is already used in this tree. The header's note no longer claims libstdc++ as the assumption. Four tests that predate the metrics_endpoint scheme guard set tls_client_cert and only put traces_endpoint on https, so the new guard threw before the check each one asserts. They now set both endpoints. Nine tests in the two files set a client cert; the other five stay correct because the pairing, use_tls and traces checks all run ahead of the metrics one. Eleven clang-tidy findings: redundant member initialisers, two aggregate initialisations that wanted designated form, two unbraced bodies, three unparenthesised multiplications, and a reserve before a loop that emplaces. Dropping std::make_shared also left unused in both files. --- .../libxrpl/telemetry/TelemetryConfig.cpp | 3 +++ .../telemetry/TraceExporterOptions.cpp | 9 +++++++++ .../libxrpl/telemetry/ValidationTracker.cpp | 9 ++++++--- src/xrpld/telemetry/ValidationTracker.h | 20 +++++++++++-------- .../telemetry/detail/ValidationTracker.cpp | 9 ++++++--- 5 files changed, 36 insertions(+), 14 deletions(-) diff --git a/src/tests/libxrpl/telemetry/TelemetryConfig.cpp b/src/tests/libxrpl/telemetry/TelemetryConfig.cpp index f8ecaa7de7..52e8ea37c6 100644 --- a/src/tests/libxrpl/telemetry/TelemetryConfig.cpp +++ b/src/tests/libxrpl/telemetry/TelemetryConfig.cpp @@ -503,6 +503,7 @@ TEST(TelemetryConfig, tls_missing_client_cert_file_throws) Section section = mtls::makeSection(true); section.set("use_tls", "1"); section.set(mtls::keyEndpoint, mtls::httpsEndpoint); + section.set(mtls::keyMetricsEndpoint, mtls::metricsHttpsEndpoint); section.set(mtls::keyClientCert, absentCert); section.set(mtls::keyClientKey, mtls::writeCertFile(dir.file("k.pem"))); @@ -521,6 +522,7 @@ TEST(TelemetryConfig, tls_missing_client_key_file_throws) Section section = mtls::makeSection(true); section.set("use_tls", "1"); section.set(mtls::keyEndpoint, mtls::httpsEndpoint); + section.set(mtls::keyMetricsEndpoint, mtls::metricsHttpsEndpoint); section.set(mtls::keyClientCert, mtls::writeCertFile(dir.file("c.pem"))); section.set(mtls::keyClientKey, absentKey); @@ -559,6 +561,7 @@ TEST(TelemetryConfig, tls_client_key_that_is_a_directory_throws) Section section = mtls::makeSection(true); section.set("use_tls", "1"); section.set(mtls::keyEndpoint, mtls::httpsEndpoint); + section.set(mtls::keyMetricsEndpoint, mtls::metricsHttpsEndpoint); section.set(mtls::keyClientCert, mtls::writeCertFile(dir.file("c.pem"))); section.set(mtls::keyClientKey, keyDir); diff --git a/src/tests/libxrpl/telemetry/TraceExporterOptions.cpp b/src/tests/libxrpl/telemetry/TraceExporterOptions.cpp index 8e7d1cb6ba..e1accd12f4 100644 --- a/src/tests/libxrpl/telemetry/TraceExporterOptions.cpp +++ b/src/tests/libxrpl/telemetry/TraceExporterOptions.cpp @@ -31,6 +31,14 @@ constexpr char const* clientKey = "/etc/xrpl/tls/node-client-private-key.pem"; constexpr char const* kHttpsEndpoint = "https://collector.example:4318/v1/traces"; +/** + * The metric endpoint the section-parsing case needs. + * + * A client certificate requires https on both endpoints, so a case that parses + * a whole section has to set this one as well or the parse throws. + */ +constexpr char const* kHttpsMetricsEndpoint = "https://collector.example:4318/v1/metrics"; + /** * Build a Setup with mutual TLS configured and nothing else set. * @@ -133,6 +141,7 @@ TEST(TraceExporterOptions, config_section_reaches_the_exporter_options) Section section; section.set("enabled", "1"); section.set("traces_endpoint", kHttpsEndpoint); + section.set("metrics_endpoint", kHttpsMetricsEndpoint); section.set("use_tls", "1"); section.set("tls_client_cert", cert); section.set("tls_client_key", key); diff --git a/src/tests/libxrpl/telemetry/ValidationTracker.cpp b/src/tests/libxrpl/telemetry/ValidationTracker.cpp index e9b9558e9b..57b90efe9d 100644 --- a/src/tests/libxrpl/telemetry/ValidationTracker.cpp +++ b/src/tests/libxrpl/telemetry/ValidationTracker.cpp @@ -355,7 +355,7 @@ TEST(ValidationTracker, an_event_exactly_one_day_old_has_left_the_day_window) t.recordNetworkValidation(makeHash(21), 21); settle(t); - advance(std::chrono::minutes(24 * 60 - 1) - Tracker::gracePeriod()); + advance(std::chrono::minutes((24 * 60) - 1) - Tracker::gracePeriod()); t.reconcile(); EXPECT_EQ(t.agreements24h(), 1u); @@ -396,7 +396,7 @@ TEST(ValidationTracker, steady_traffic_across_the_grid_boundary_keeps_recent_cou // that was not cleared shows up when that bucket is subtracted. auto t = makeTracker(); - constexpr std::uint64_t kMinutes = 2 * 7 * 24 * 60 + 5; + constexpr std::uint64_t kMinutes = (2 * 7 * 24 * 60) + 5; for (std::uint64_t i = 0; i < kMinutes; ++i) { t.recordOurValidation(makeHash(i), static_cast(i)); @@ -532,7 +532,7 @@ TEST(ValidationTracker, a_burst_larger_than_one_ring_is_counted_in_full_when_dra { for (std::size_t i = 0; i < perBatch; ++i) { - auto const n = b * perBatch + i + 1; + auto const n = (b * perBatch) + i + 1; t.recordOurValidation(makeHash(n), static_cast(n)); t.recordNetworkValidation(makeHash(n), static_cast(n)); } @@ -636,11 +636,14 @@ TEST(ValidationTracker, concurrent_reducer_entry_does_not_deadlock_or_double_cou advance(Tracker::gracePeriod() + std::chrono::seconds(1)); std::vector readers; + readers.reserve(8); for (int i = 0; i < 8; ++i) + { readers.emplace_back([&t] { for (int j = 0; j < 500; ++j) t.reconcile(); }); + } for (auto& r : readers) r.join(); diff --git a/src/xrpld/telemetry/ValidationTracker.h b/src/xrpld/telemetry/ValidationTracker.h index 30dad9061e..c5f3142bc0 100644 --- a/src/xrpld/telemetry/ValidationTracker.h +++ b/src/xrpld/telemetry/ValidationTracker.h @@ -9,13 +9,15 @@ #include #include +#include +#include + #include #include #include #include #include #include -#include namespace xrpl::telemetry { @@ -101,8 +103,10 @@ namespace xrpl::telemetry { * and the ledger master call them. reconcile() and the getters may be called * from any thread and any number of threads. * @note reconcile() and the getters share the published snapshot through an - * atomic shared_ptr, which libstdc++ guards with a short internal spin. No - * writer path touches it, so nothing a consensus thread calls can spin. + * atomic shared_ptr, which every implementation guards with a short internal + * spin. No writer path touches it, so nothing a consensus thread calls can + * spin. Boost's is used because Apple's libc++ has no std::atomic for a + * shared_ptr, so the std spelling does not compile there. * @note A writer whose ring is full discards the event and bumps * droppedEvents(). Counts are then low but never wrong. * @note Window edges are rounded to whole minutes, because counts are kept in @@ -369,7 +373,7 @@ private: */ LedgerIndex seq{0}; - TimePoint at{}; ///< When the writer recorded it. + TimePoint at; ///< When the writer recorded it. }; /** @@ -407,7 +411,7 @@ private: if (head - tail_.load(std::memory_order_acquire) >= kRingCapacity) return false; - slots_[head & (kRingCapacity - 1)] = Slot{hash, seq, at}; + slots_[head & (kRingCapacity - 1)] = Slot{.hash = hash, .seq = seq, .at = at}; head_.store(head + 1, std::memory_order_release); return true; } @@ -449,7 +453,7 @@ private: */ struct LedgerEvent { - TimePoint recordTime{}; ///< Time the event was first recorded. + TimePoint recordTime; ///< Time the event was first recorded. std::uint64_t minute{0}; ///< Minute bucket the event belongs to. bool weValidated{false}; ///< True if we sent a validation. bool networkValidated{false}; ///< True if network reached consensus. @@ -620,7 +624,7 @@ private: * Set while a thread is inside reconcile(). A second caller sees it set * and returns rather than waiting. */ - std::atomic_flag reducing_{}; + std::atomic_flag reducing_; /** * Pending ledger events indexed by ledger hash. Touched only inside @@ -692,7 +696,7 @@ private: * a reader that took the old one keeps it alive while it reads. Null until * the first reconcile(). */ - std::atomic> published_; + boost::atomic_shared_ptr published_; /** * Lifetime count of agreements. diff --git a/src/xrpld/telemetry/detail/ValidationTracker.cpp b/src/xrpld/telemetry/detail/ValidationTracker.cpp index 297a6e88b8..d355fb0c76 100644 --- a/src/xrpld/telemetry/detail/ValidationTracker.cpp +++ b/src/xrpld/telemetry/detail/ValidationTracker.cpp @@ -8,12 +8,13 @@ #include #include +#include + #include #include #include #include #include -#include namespace xrpl::telemetry { @@ -122,8 +123,10 @@ ValidationTracker::decidePending(TimePoint now) // Nothing can be repaired past the window, so the entry is dead weight. auto const cutoff = now - kLateRepairWindow; for (auto it = pending_.begin(); it != pending_.end();) + { it = (it->second.decided && it->second.recordTime < cutoff) ? pending_.erase(it) : std::next(it); + } } void @@ -248,7 +251,7 @@ void ValidationTracker::publish() { published_.store( - std::make_shared(Snapshot{c1h_, c24h_, c7d_}), std::memory_order_release); + boost::make_shared(Snapshot{.w1h = c1h_, .w24h = c24h_, .w7d = c7d_})); } ValidationTracker::Snapshot @@ -256,7 +259,7 @@ ValidationTracker::read() const { // Holding the shared_ptr keeps this snapshot alive for as long as the // caller needs it, so the reducer can never write the values being read. - auto const s = published_.load(std::memory_order_acquire); + auto const s = published_.load(); return s ? *s : Snapshot{}; }