mirror of
https://github.com/XRPLF/rippled.git
synced 2026-09-26 23:19:07 +00:00
merge: bring the review fixes forward from otel-phase8-log-correlation
This commit is contained in:
Pratik Mankawde
@@ -318,7 +318,7 @@ Establish-phase gap fill and cross-node correlation attributes (Phase 4a):
|
||||
| --------------------- | ------ | --------------------------------------------------------- |
|
||||
| `consensus_round_id` | int64 | Consensus round number |
|
||||
| `consensus_ledger_id` | string | `previousLedger.id()` — shared across nodes |
|
||||
| `trace_strategy` | string | `"deterministic"` or `"attribute"` |
|
||||
| `trace_strategy` | string | `"deterministic"` or `"random"` |
|
||||
| `converge_percent` | int64 | Convergence % (0-100+) |
|
||||
| `establish_count` | int64 | Number of establish iterations |
|
||||
| `disputes_count` | int64 | Active disputed transactions |
|
||||
@@ -644,7 +644,8 @@ The first 16 bytes are used as trace_id. See [Phase 4a implementation status](./
|
||||
and `createDeterministicContext()` in `RCLConsensus.cpp` for the implementation.
|
||||
|
||||
Switchable via `consensus_trace_strategy` config:
|
||||
`"deterministic"` (default) or `"attribute"` (random trace_id, correlation via attribute queries).
|
||||
`"deterministic"` (default) or `"random"` (random trace_id, correlation via attribute queries).
|
||||
`"random"` is experimental and not used: it would break cross-node trace correlation.
|
||||
|
||||
#### Why Not Random IDs with Propagation Only?
|
||||
|
||||
|
||||
@@ -66,33 +66,26 @@ The authoritative `[telemetry]` example lives in `cfg/xrpld-example.cfg`. Teleme
|
||||
|
||||
### 5.1.2 Configuration Options Summary
|
||||
|
||||
| Option | Type | Default | Description |
|
||||
| -------------------------- | ------ | ---------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `enabled` | 0 or 1 | `0` | Enable/disable telemetry |
|
||||
| `traces_endpoint` | string | `http://localhost:4318/v1/traces` | OTLP/HTTP collector endpoint for **traces** |
|
||||
| `metrics_endpoint` | string | `http://localhost:4318/v1/metrics` | OTLP/HTTP collector endpoint for the native metrics pipeline (`MetricsRegistry`). Read in `Application.cpp:1670` |
|
||||
| `use_tls` | 0 or 1 | `0` | Enable TLS for exporter connection |
|
||||
| `tls_ca_cert` | string | `""` | Path to CA certificate file |
|
||||
| `tls_client_cert` | string | `""` | Client cert (PEM) for mTLS; empty = one-way; if `enabled=1`, needs key + `use_tls=1` or startup fails |
|
||||
| `tls_client_key` | string | `""` | Private key (PEM) for `tls_client_cert`; if set with `enabled=1`, needs the cert + `use_tls=1` or fails |
|
||||
| `batch_size` | uint | `512` | Spans per export batch |
|
||||
| `batch_delay_ms` | uint | `5000` | Max delay before sending batch (ms) |
|
||||
| `max_queue_size` | uint | `2048` | Maximum queued spans |
|
||||
| `trace_transactions` | 0 or 1 | `1` | Enable transaction tracing |
|
||||
| `trace_consensus` | 0 or 1 | `1` | Enable consensus tracing |
|
||||
| `trace_rpc` | 0 or 1 | `1` | Enable RPC tracing |
|
||||
| `trace_peer` | 0 or 1 | `1` | Enable peer message tracing (high volume) |
|
||||
| `trace_ledger` | 0 or 1 | `1` | Enable ledger tracing |
|
||||
| `consensus_trace_strategy` | string | `"deterministic"` | Consensus trace ID strategy: `"deterministic"` (trace_id = prevLedgerHash[0:16]) or `"attribute"` (random). Parsed at `TelemetryConfig.cpp:155-156`, consumed at `RCLConsensus.cpp:1291,1296`. **Not validated** — see the note below |
|
||||
| `service_name` | string | `"xrpld"` | Service name (`service.name`) for traces and metrics |
|
||||
| `service_instance_id` | string | node public key (base58) | Instance identifier (`service.instance.id`). Traces, span metrics and native `XRPL_METRIC_*` metrics all fall back to the node key; **`beast::insight` metrics do not** — see the note in §5.1.1 |
|
||||
|
||||
**`consensus_trace_strategy` is not validated.** `TelemetryConfig.cpp:155-156`
|
||||
copies the raw string into `Setup::consensusTraceStrategy` without checking it
|
||||
against an allowed set, and the only comparison in the code is
|
||||
`strategy == "attribute"` (`RCLConsensus.cpp:1296`). Any unrecognised value —
|
||||
including a typo — silently takes the deterministic branch with no log warning.
|
||||
The two accepted values are documented at `include/xrpl/telemetry/Telemetry.h:287-292`.
|
||||
| Option | Type | Default | Description |
|
||||
| -------------------------- | ------ | ---------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
|
||||
| `enabled` | 0 or 1 | `0` | Enable/disable telemetry |
|
||||
| `traces_endpoint` | string | `http://localhost:4318/v1/traces` | OTLP/HTTP collector endpoint for **traces** |
|
||||
| `metrics_endpoint` | string | `http://localhost:4318/v1/metrics` | OTLP/HTTP collector endpoint for the native metrics pipeline (`MetricsRegistry`). Read in `Application.cpp:1670` |
|
||||
| `use_tls` | 0 or 1 | `0` | Enable TLS for exporter connection |
|
||||
| `tls_ca_cert` | string | `""` | Path to CA certificate file |
|
||||
| `tls_client_cert` | string | `""` | Client cert (PEM) for mTLS; empty = one-way; if `enabled=1`, needs key + `use_tls=1` or startup fails |
|
||||
| `tls_client_key` | string | `""` | Private key (PEM) for `tls_client_cert`; if set with `enabled=1`, needs the cert + `use_tls=1` or fails |
|
||||
| `batch_size` | uint | `512` | Spans per export batch |
|
||||
| `batch_delay_ms` | uint | `5000` | Max delay before sending batch (ms) |
|
||||
| `max_queue_size` | uint | `2048` | Maximum queued spans |
|
||||
| `trace_transactions` | 0 or 1 | `1` | Enable transaction tracing |
|
||||
| `trace_consensus` | 0 or 1 | `1` | Enable consensus tracing |
|
||||
| `trace_rpc` | 0 or 1 | `1` | Enable RPC tracing |
|
||||
| `trace_peer` | 0 or 1 | `1` | Enable peer message tracing (high volume) |
|
||||
| `trace_ledger` | 0 or 1 | `1` | Enable ledger tracing |
|
||||
| `consensus_trace_strategy` | string | `"deterministic"` | Consensus trace ID strategy: `"deterministic"` (trace_id = prevLedgerHash[0:16]) or `"random"` (experimental, not used). Rejected at startup if it is neither spelling |
|
||||
| `service_name` | string | `"xrpld"` | Service name (`service.name`) for traces and metrics |
|
||||
| `service_instance_id` | string | node public key (base58) | Instance identifier (`service.instance.id`). Traces, span metrics and native `XRPL_METRIC_*` metrics all fall back to the node key; **`beast::insight` metrics do not** — see the note in §5.1.1 |
|
||||
|
||||
**Not a config key — deterministic transaction trace IDs are unconditional.**
|
||||
Earlier drafts of this document listed a `tx_trace_strategy` option
|
||||
|
||||
@@ -253,7 +253,8 @@ Phase 4a (establish-phase gap fill & cross-node correlation) adds:
|
||||
|
||||
- **Deterministic trace ID** derived from `previousLedger.id()` so all validators
|
||||
in the same round share the same `trace_id` (switchable via
|
||||
`consensus_trace_strategy` config: `"deterministic"` or `"attribute"`).
|
||||
`consensus_trace_strategy` config: `"deterministic"`, or `"random"` which is
|
||||
experimental and not used).
|
||||
See [Configuration Reference](./05-configuration-reference.md) for full
|
||||
configuration options.
|
||||
- **Round lifecycle spans**: `consensus.round` with round-to-round span links.
|
||||
|
||||
@@ -1741,6 +1741,13 @@ validators.txt
|
||||
# beast::insight metrics ([insight] server=otel) do not follow this
|
||||
# setting; they use metrics_endpoint below.
|
||||
#
|
||||
# The scheme of this URL is what decides whether the connection is
|
||||
# encrypted, and it is matched exactly: only a lower-case https:// URL
|
||||
# gives TLS. Because of that, setting tls_client_cert requires this URL
|
||||
# to start with https:// — including leaving it at the default above,
|
||||
# which makes xrpld fail to start rather than export without the client
|
||||
# identity it was configured with.
|
||||
#
|
||||
# --- TLS settings for the OTLP exporter connection ---
|
||||
#
|
||||
# use_tls=0
|
||||
@@ -1774,11 +1781,12 @@ validators.txt
|
||||
# To enable mTLS, both tls_client_cert and tls_client_key must be
|
||||
# specified. If only one is provided, xrpld will fail to start. Providing
|
||||
# them while use_tls=0 also fails to start, rather than being ignored.
|
||||
# With use_tls=1 each path is opened at startup, so one that does not
|
||||
# exist or cannot be read fails to start too, rather than failing later
|
||||
# as an opaque TLS handshake error. All three checks apply only when
|
||||
# enabled=1; with telemetry disabled these settings are read but never
|
||||
# validated.
|
||||
# traces_endpoint must be an https:// URL, because that scheme is what
|
||||
# makes the exporter present the certificate at all. With use_tls=1 each
|
||||
# path is opened at startup, so one that does not exist or cannot be read
|
||||
# fails to start too, rather than failing later as an opaque TLS
|
||||
# handshake error. All four checks apply only when enabled=1; with
|
||||
# telemetry disabled these settings are read but never validated.
|
||||
#
|
||||
# tls_client_key=
|
||||
#
|
||||
@@ -1823,6 +1831,20 @@ validators.txt
|
||||
# Enable tracing for ledger close and accept operations — ledger
|
||||
# building, state hashing, and write-back to the node store. Default: 1.
|
||||
#
|
||||
# consensus_trace_strategy=deterministic
|
||||
#
|
||||
# How the consensus round span picks its trace id. Two values are
|
||||
# accepted, and anything else makes xrpld fail to start.
|
||||
#
|
||||
# deterministic (the default, and the value to use): the trace id comes
|
||||
# from the previous ledger hash, so every validator of a round reports
|
||||
# into one trace and the round can be read end to end across nodes.
|
||||
#
|
||||
# random: experimental only, and not used. Each node invents its own
|
||||
# trace id, so a single round arrives as one separate trace per node.
|
||||
# Those traces can only be lined up by hand through the
|
||||
# consensus_ledger_id span attribute.
|
||||
#
|
||||
# --- Batch processor tuning ---
|
||||
#
|
||||
# batch_size=512
|
||||
|
||||
@@ -124,27 +124,29 @@ curl -s http://localhost:5015 -d '{"method":"server_info"}' |
|
||||
|
||||
## Configuration Reference
|
||||
|
||||
| Option | Default | Description |
|
||||
| -------------------------- | --------------------------------- | ------------------------------------------------------------ |
|
||||
| `enabled` | `0` | Master switch for telemetry |
|
||||
| `traces_endpoint` | `http://localhost:4318/v1/traces` | OTLP/HTTP endpoint |
|
||||
| `service_name` | `xrpld` | OpenTelemetry service name resource attribute |
|
||||
| `service_instance_id` | node public key | OpenTelemetry service instance ID resource attribute |
|
||||
| `trace_rpc` | `1` | Enable RPC request tracing |
|
||||
| `trace_transactions` | `1` | Enable transaction tracing |
|
||||
| `trace_consensus` | `1` | Enable consensus tracing |
|
||||
| `trace_peer` | `1` | Enable peer message tracing (high volume) |
|
||||
| `trace_ledger` | `1` | Enable ledger tracing |
|
||||
| `consensus_trace_strategy` | `deterministic` | Consensus trace ID strategy (`deterministic` or `attribute`) |
|
||||
| `batch_size` | `512` | Max spans per batch export |
|
||||
| `batch_delay_ms` | `5000` | Delay between batch exports |
|
||||
| `max_queue_size` | `2048` | Max spans queued before dropping |
|
||||
| `use_tls` | `0` | Use TLS for exporter connection |
|
||||
| `tls_ca_cert` | (empty) | Path to CA certificate bundle |
|
||||
| `tls_client_cert` | (empty) | Client cert (PEM) for mTLS; empty = one-way. See note |
|
||||
| `tls_client_key` | (empty) | Private key (PEM) for `tls_client_cert`. See note |
|
||||
| Option | Default | Description |
|
||||
| -------------------------- | --------------------------------- | ----------------------------------------------------------------------------------------------------- |
|
||||
| `enabled` | `0` | Master switch for telemetry |
|
||||
| `traces_endpoint` | `http://localhost:4318/v1/traces` | Full OTLP/HTTP URL for spans, used verbatim |
|
||||
| `service_name` | `xrpld` | OpenTelemetry service name resource attribute |
|
||||
| `service_instance_id` | node public key | OpenTelemetry service instance ID resource attribute |
|
||||
| `trace_rpc` | `1` | Enable RPC request tracing |
|
||||
| `trace_transactions` | `1` | Enable transaction tracing |
|
||||
| `trace_consensus` | `1` | Enable consensus tracing |
|
||||
| `trace_peer` | `1` | Enable peer message tracing (high volume) |
|
||||
| `trace_ledger` | `1` | Enable ledger tracing |
|
||||
| `consensus_trace_strategy` | `deterministic` | Consensus trace ID strategy. `deterministic` is the value to use; `random` is experimental — see note |
|
||||
| `batch_size` | `512` | Max spans per batch export |
|
||||
| `batch_delay_ms` | `5000` | Delay between batch exports |
|
||||
| `max_queue_size` | `2048` | Max spans queued before dropping |
|
||||
| `use_tls` | `0` | Use TLS for exporter connection |
|
||||
| `tls_ca_cert` | (empty) | Path to CA certificate bundle |
|
||||
| `tls_client_cert` | (empty) | Client cert (PEM) for mTLS; empty = one-way. See note |
|
||||
| `tls_client_key` | (empty) | Private key (PEM) for `tls_client_cert`. See note |
|
||||
|
||||
> **mTLS (mutual TLS) note**: `tls_client_cert` and `tls_client_key` are optional — leaving both empty gives one-way (server-only) TLS. **If either one is set**, `enabled=1` requires both of them **and** `use_tls=1`, or the node exits at startup; see the Troubleshooting entry for `Unable to start ...: [telemetry] ...`. When `enabled=0` they are read but never validated.
|
||||
> **mTLS (mutual TLS) note**: `tls_client_cert` and `tls_client_key` are optional — leaving both empty gives one-way (server-only) TLS. **If either one is set**, `enabled=1` requires both of them, `use_tls=1`, **and** a `traces_endpoint` starting with `https://` — the exporter decides encryption from the URL scheme, so the certificate is only ever presented on an `https://` endpoint. The default `traces_endpoint` is plain HTTP, so mTLS means setting that key too. Breaking any of these makes the node exit at startup; see the Troubleshooting entry for `Unable to start ...: [telemetry] ...`. When `enabled=0` they are read but never validated.
|
||||
|
||||
> **`consensus_trace_strategy` note**: only `deterministic` and `random` are accepted, and anything else makes the node exit at startup. Use `deterministic`: it seeds the round's trace ID from the previous ledger hash, so every validator of a round reports into one trace. `random` is experimental and not used — each node would invent its own trace ID, so a single round would arrive as one separate trace per node, joinable only by hand through `consensus_ledger_id`.
|
||||
|
||||
> **Traces and metrics also carry `xrpl.node.id`.** xrpld sets it as a resource
|
||||
> attribute alongside `service.instance.id`; the value is the node public key
|
||||
@@ -342,17 +344,17 @@ read it from the parent rather than filtering `tx.apply` on it.
|
||||
|
||||
#### Consensus Span Events
|
||||
|
||||
| Parent Span | Event Name | Event Attributes | Description |
|
||||
| ---------------------------- | ------------------ | ----------------------------------------------------------- | -------------------------------------------------------- |
|
||||
| `consensus.update_positions` | `dispute.resolve` | `tx_id`, `dispute_our_vote`, `dispute_yays`, `dispute_nays` | Emitted per dispute when votes are tallied |
|
||||
| `consensus.accept.apply` | `tx.included` | `tx_id` | Emitted per transaction included in the accepted ledger |
|
||||
| `consensus.round` | `phase.open` | -- | Round entered the open phase (also re-fired on recovery) |
|
||||
| `consensus.round` | `phase.recovery` | -- | Round started with `StartRoundReason::Recovered` |
|
||||
| `consensus.round` | `phase.establish` | -- | Round entered the establish phase on close |
|
||||
| `consensus.round` | `phase.accepted` | -- | Round reached the accepted phase |
|
||||
| `consensus.round` | `outcome.yes` | -- | Round settled with consensus reached |
|
||||
| `consensus.round` | `outcome.moved_on` | -- | Round abandoned; the network moved on without us |
|
||||
| `consensus.round` | `outcome.expired` | -- | Round expired without settling |
|
||||
| Parent Span | Event Name | Event Attributes | Description |
|
||||
| ---------------------------- | ------------------ | ----------------------------------------------------------- | ------------------------------------------------------------------------------------------ |
|
||||
| `consensus.update_positions` | `dispute.resolve` | `tx_id`, `dispute_our_vote`, `dispute_yays`, `dispute_nays` | Emitted per dispute when votes are tallied |
|
||||
| `consensus.accept.apply` | `tx.included` | `tx_id` | Emitted per transaction of the agreed consensus set, before the ledger is built — see note |
|
||||
| `consensus.round` | `phase.open` | -- | Round entered the open phase (also re-fired on recovery) |
|
||||
| `consensus.round` | `phase.recovery` | -- | Round started with `StartRoundReason::Recovered` |
|
||||
| `consensus.round` | `phase.establish` | -- | Round entered the establish phase on close |
|
||||
| `consensus.round` | `phase.accepted` | -- | Round reached the accepted phase |
|
||||
| `consensus.round` | `outcome.yes` | -- | Round settled with consensus reached |
|
||||
| `consensus.round` | `outcome.moved_on` | -- | Round abandoned; the network moved on without us |
|
||||
| `consensus.round` | `outcome.expired` | -- | Round expired without settling |
|
||||
|
||||
The nine events above are the complete set. The seven on `consensus.round`
|
||||
carry **no event attributes** — they are timestamps marking phase entry and the
|
||||
@@ -366,9 +368,16 @@ attribute, which is why `phase.recovery` is the one phase event that leaves
|
||||
from `result_->state` at
|
||||
[Consensus.h:1517-1525](../include/xrpl/consensus/Consensus.h#L1517).
|
||||
|
||||
> **`tx.included` note**: the event is recorded while the canonical transaction set is being assembled, which happens before `buildLCL()` applies anything. So a transaction that fails to apply, or is left over to retry in a later ledger, still has a `tx.included` event. Treat the events as the round's **input** set, not as proof a transaction reached the accepted ledger; `tx_count` on the same span counts the same set. A transaction whose bytes cannot be parsed gets no event, and nothing in the accepted ledger is missing one, so the events are always a superset of the ledger's contents. To confirm a transaction actually applied, read `ter_result` and `applied` on its `tx.transactor` span.
|
||||
|
||||
#### Close Time Queries (Tempo TraceQL)
|
||||
|
||||
Span attributes are filtered with `span.<attr>` inside `{}`. Combine conditions with `&&`.
|
||||
> **TraceQL syntax**: an attribute filter belongs inside the braces, as
|
||||
> `{name="x" && span.attr = value}`. The `{name="x"} | attr = value` form used
|
||||
> by several examples in this document is rejected by current Tempo with a parse
|
||||
> error, so convert an example to the braced form before running it. Numeric
|
||||
> attributes such as `consensus_round_id` and `retries_remaining` must be
|
||||
> compared unquoted.
|
||||
|
||||
```
|
||||
# Find rounds where validators disagreed on close time
|
||||
@@ -383,11 +392,14 @@ Span attributes are filtered with `span.<attr>` inside `{}`. Combine conditions
|
||||
# Find specific ledger's consensus details
|
||||
{name="consensus.accept.apply" && span.ledger_seq = 92345678}
|
||||
|
||||
# Find all spans in a consensus round (deterministic trace strategy)
|
||||
{name="consensus.round" && span.consensus_round_id = "<round_id>"}
|
||||
# Find a consensus round by its id. consensus_round_id is an int64 — the
|
||||
# previous ledger sequence plus one — so it must not be quoted, and it is set
|
||||
# only on consensus.round.
|
||||
{name="consensus.round" && span.consensus_round_id = 92345678}
|
||||
|
||||
# Find dispute resolutions
|
||||
{name="consensus.update_positions"} >> {event:name="dispute.resolve"}
|
||||
# Find dispute resolutions. The event is recorded on the update_positions
|
||||
# span itself, so it is a condition on that span, not on a descendant.
|
||||
{name="consensus.update_positions" && event:name="dispute.resolve"}
|
||||
```
|
||||
|
||||
### Ledger Spans
|
||||
@@ -1338,8 +1350,11 @@ sum by (stage) (rate(span_calls_total{span_name=~"tx.preflight|tx.preclaim|tx.tr
|
||||
# Find transactions being retried
|
||||
{name="txq.accept_tx" && span.txq_status = "retried"}
|
||||
|
||||
# Find transactions that exhausted retries
|
||||
{name="txq.accept_tx" && span.txq_status = "retried" && span.retries_remaining = 0}
|
||||
# Find transactions dropped because they had no retries left.
|
||||
# retries_remaining is recorded before the attempt, and the "retried" branch
|
||||
# is only reached while retries are left, so exhaustion always shows up as
|
||||
# "failed" with a zero count.
|
||||
{name="txq.accept_tx" && span.txq_status = "failed" && span.retries_remaining <= 0}
|
||||
|
||||
# Which transaction types get queued most often?
|
||||
{name="txq.enqueue" && span.tx_type = "Payment"}
|
||||
@@ -1546,8 +1561,12 @@ all its normal attributes, it just lacks a cross-node parent link.
|
||||
# Trace a transaction across the network by its hash
|
||||
{name =~ "tx.*" && span.tx_hash = "<hash>"}
|
||||
|
||||
# Find all spans in a cross-node consensus trace
|
||||
{resource.service.name="xrpld" && span.consensus_round_id = "<round_id>"}
|
||||
# Find a cross-node consensus trace by round id, then open the returned trace
|
||||
# to see every node's spans. Under the deterministic strategy all validators of
|
||||
# a round share one trace id, so one trace holds all of them. The value is an
|
||||
# integer, so it must not be quoted, and it only matches the consensus.round
|
||||
# span that carries it.
|
||||
{name="consensus.round" && span.consensus_round_id = 92345678}
|
||||
|
||||
# Compare latency between sender and receiver for validations
|
||||
{name = "consensus.validation.send" || name = "consensus.validation.receive"}
|
||||
|
||||
@@ -317,7 +317,9 @@ namespace event {
|
||||
*/
|
||||
inline constexpr auto disputeResolve = join(makeStr("dispute"), makeStr("resolve"));
|
||||
/**
|
||||
* "tx.included"
|
||||
* "tx.included" — one per transaction of the agreed consensus set, recorded
|
||||
* before the ledger is built. A transaction that then fails to apply still
|
||||
* has an event, so this is a superset of the accepted ledger's contents.
|
||||
*/
|
||||
inline constexpr auto txIncluded = join(makeStr("tx"), makeStr("included"));
|
||||
|
||||
|
||||
@@ -103,6 +103,7 @@
|
||||
|
||||
#ifdef XRPL_ENABLE_TELEMETRY
|
||||
#include <opentelemetry/context/context.h>
|
||||
#include <opentelemetry/exporters/otlp/otlp_http_exporter_options.h>
|
||||
#include <opentelemetry/metrics/meter.h>
|
||||
#include <opentelemetry/nostd/shared_ptr.h>
|
||||
#include <opentelemetry/trace/span.h>
|
||||
@@ -150,6 +151,69 @@ inline constexpr auto kDefaultMetricExportInterval = std::chrono::milliseconds{1
|
||||
*/
|
||||
inline constexpr auto kDefaultMetricExportTimeout = std::chrono::milliseconds{500};
|
||||
|
||||
/**
|
||||
* How a consensus round span picks its trace id.
|
||||
*
|
||||
* consensus_trace_strategy (xrpld.cfg)
|
||||
* |
|
||||
* v
|
||||
* makeTelemetrySetup() ──> Setup::consensusTraceStrategy
|
||||
* |
|
||||
* v
|
||||
* RCLConsensus::Adaptor::startRoundTracing()
|
||||
* |
|
||||
* +-- Deterministic ──> SpanGuard::hashSpan(prev ledger hash)
|
||||
* +-- Random ──> SpanGuard::span() / linkedSpan()
|
||||
*
|
||||
* Deterministic is the strategy in use. Every validator of a round hashes the
|
||||
* same previous ledger id, so all of them land in one trace.
|
||||
*
|
||||
* Random is experimental and not used. Each node would invent its own trace
|
||||
* id, so one round would arrive as one trace per node, joinable only by the
|
||||
* `consensus_ledger_id` attribute.
|
||||
*
|
||||
* @code
|
||||
* // Branch on the strategy rather than on a string.
|
||||
* if (telemetry.getConsensusTraceStrategy() == ConsensusTraceStrategy::Random)
|
||||
* span = SpanGuard::span(TraceCategory::Consensus, seg::consensus, op::round);
|
||||
* else
|
||||
* span = SpanGuard::hashSpan(TraceCategory::Consensus, name, id.data(), id.kBytes);
|
||||
*
|
||||
* // Edge case: the value also goes on a span attribute, so it needs its
|
||||
* // config spelling back.
|
||||
* span.setAttribute(attr::traceStrategy, strategyName(ConsensusTraceStrategy::Random));
|
||||
* @endcode
|
||||
*
|
||||
* @note Adding an enumerator means adding a spelling to strategyName() below
|
||||
* and to the parser in TelemetryConfig.cpp. Both switch without a default, so
|
||||
* the compiler catches a missed one.
|
||||
*/
|
||||
enum class ConsensusTraceStrategy : std::uint8_t { Deterministic, Random };
|
||||
|
||||
/**
|
||||
* Config spelling of a consensus trace strategy.
|
||||
*
|
||||
* This is the same text `consensus_trace_strategy` accepts, and it is what
|
||||
* goes on the `trace_strategy` span attribute, so the two cannot drift.
|
||||
*
|
||||
* @param strategy Strategy to name.
|
||||
* @return "deterministic" or "random", pointing at a string literal.
|
||||
*/
|
||||
[[nodiscard]] constexpr char const*
|
||||
strategyName(ConsensusTraceStrategy strategy)
|
||||
{
|
||||
switch (strategy)
|
||||
{
|
||||
case ConsensusTraceStrategy::Deterministic:
|
||||
return "deterministic";
|
||||
case ConsensusTraceStrategy::Random:
|
||||
return "random";
|
||||
}
|
||||
// The switch covers every enumerator. This return only satisfies the
|
||||
// compiler, which cannot rule out a value outside the enumeration.
|
||||
return "deterministic";
|
||||
}
|
||||
|
||||
class Telemetry
|
||||
{
|
||||
/**
|
||||
@@ -338,12 +402,12 @@ public:
|
||||
bool traceLedger = true;
|
||||
|
||||
/**
|
||||
* Strategy for cross-node consensus trace correlation.
|
||||
* "deterministic" — derive trace_id from ledger hash so all
|
||||
* validators in the same round share the same trace_id.
|
||||
* "attribute" — random trace_id, correlate via ledger_id attribute.
|
||||
* How a consensus round span picks its trace id.
|
||||
*
|
||||
* Read from `consensus_trace_strategy`. Deterministic is the strategy
|
||||
* in use; Random is experimental. See ConsensusTraceStrategy.
|
||||
*/
|
||||
std::string consensusTraceStrategy = "deterministic";
|
||||
ConsensusTraceStrategy consensusTraceStrategy = ConsensusTraceStrategy::Deterministic;
|
||||
};
|
||||
|
||||
virtual ~Telemetry() = default;
|
||||
@@ -433,9 +497,9 @@ public:
|
||||
shouldTraceLedger() const = 0;
|
||||
|
||||
/**
|
||||
* @return The configured consensus trace correlation strategy.
|
||||
* @return How a consensus round span picks its trace id.
|
||||
*/
|
||||
[[nodiscard]] virtual std::string const&
|
||||
[[nodiscard]] virtual ConsensusTraceStrategy
|
||||
getConsensusTraceStrategy() const = 0;
|
||||
|
||||
#ifdef XRPL_ENABLE_TELEMETRY
|
||||
@@ -527,10 +591,12 @@ makeTelemetry(Telemetry::Setup const& setup, beast::Journal journal);
|
||||
* @return A populated Setup struct with defaults for missing values.
|
||||
* @throws std::runtime_error If `enabled` is set and the mutual TLS (mTLS)
|
||||
* settings contradict each other: only one of `tls_client_cert`/`tls_client_key`
|
||||
* is given, or a client certificate is given while `use_tls` is 0. Also if
|
||||
* is given, a client certificate is given while `use_tls` is 0, or a client
|
||||
* certificate is given while `traces_endpoint` is not an `https://` URL — which
|
||||
* includes leaving `traces_endpoint` at its plain-HTTP default. Also if
|
||||
* `enabled` and `use_tls` are both set and a non-empty `tls_ca_cert`,
|
||||
* `tls_client_cert` or `tls_client_key` cannot be read; an empty path is skipped,
|
||||
* so an empty `tls_ca_cert` still means "use the system CA store". All three
|
||||
* so an empty `tls_ca_cert` still means "use the system CA store". All four
|
||||
* checks are skipped when `enabled` is 0. Also if `metric_export_interval_ms`
|
||||
* or `metric_export_timeout_ms` is unreadable, is not positive, or the timeout
|
||||
* is not below the interval. Those three run whether telemetry is on or off.
|
||||
@@ -558,4 +624,46 @@ makeTelemetrySetup(
|
||||
[[nodiscard]] std::string
|
||||
networkTypeFromId(std::uint32_t networkId);
|
||||
|
||||
#ifdef XRPL_ENABLE_TELEMETRY
|
||||
/**
|
||||
* Build the OTLP/HTTP trace exporter options that Setup asks for.
|
||||
*
|
||||
* Telemetry::Setup ──> makeTraceExporterOptions() ──> OtlpHttpExporterOptions
|
||||
* |
|
||||
* v
|
||||
* OtlpHttpExporterFactory::Create
|
||||
*
|
||||
* Named and declared here rather than left inline in start() so the mapping
|
||||
* from config to exporter options can be asserted directly. A swapped
|
||||
* certificate and key, or a CA path written to the wrong field, is invisible
|
||||
* from outside a running exporter.
|
||||
*
|
||||
* TLS fields are set only when `use_tls` is on. Whether the transport is
|
||||
* actually encrypted is decided by the scheme of the URL, not by this function;
|
||||
* makeTelemetrySetup() is what rejects a client certificate on a plain-HTTP
|
||||
* endpoint.
|
||||
*
|
||||
* @code
|
||||
* // Primary use: one-way TLS with a custom CA bundle.
|
||||
* Telemetry::Setup setup;
|
||||
* setup.useTls = true;
|
||||
* setup.tlsCertPath = "/etc/ssl/ca.pem";
|
||||
* auto const opts = makeTraceExporterOptions(setup); // ssl_ca_cert_path set
|
||||
*
|
||||
* // Edge case: use_tls off leaves every ssl_ field empty, even when paths
|
||||
* // are configured.
|
||||
* setup.useTls = false;
|
||||
* auto const plain = makeTraceExporterOptions(setup); // ssl_ca_cert_path empty
|
||||
* @endcode
|
||||
*
|
||||
* @param setup Parsed [telemetry] configuration.
|
||||
* @return Options carrying `traces_endpoint` as the URL, plus the TLS paths
|
||||
* when `use_tls` is on. Every other field keeps its SDK default.
|
||||
* @note Pure: reads `setup` and touches no global state, so it is safe to call
|
||||
* from any thread.
|
||||
*/
|
||||
[[nodiscard]] opentelemetry::exporter::otlp::OtlpHttpExporterOptions
|
||||
makeTraceExporterOptions(Telemetry::Setup const& setup);
|
||||
#endif
|
||||
|
||||
} // namespace xrpl::telemetry
|
||||
|
||||
@@ -109,7 +109,7 @@ public:
|
||||
return false;
|
||||
}
|
||||
|
||||
[[nodiscard]] std::string const&
|
||||
[[nodiscard]] ConsensusTraceStrategy
|
||||
getConsensusTraceStrategy() const override
|
||||
{
|
||||
return setup_.consensusTraceStrategy;
|
||||
|
||||
@@ -244,7 +244,7 @@ public:
|
||||
return false;
|
||||
}
|
||||
|
||||
[[nodiscard]] std::string const&
|
||||
[[nodiscard]] ConsensusTraceStrategy
|
||||
getConsensusTraceStrategy() const override
|
||||
{
|
||||
return setup_.consensusTraceStrategy;
|
||||
@@ -385,19 +385,8 @@ public:
|
||||
<< " metrics_endpoint=" << setup_.metricsEndpoint
|
||||
<< " sampling=" << setup_.samplingRatio;
|
||||
|
||||
// Configure OTLP HTTP exporter
|
||||
otlp_http::OtlpHttpExporterOptions exporterOpts;
|
||||
exporterOpts.url = setup_.tracesEndpoint;
|
||||
if (setup_.useTls)
|
||||
{
|
||||
exporterOpts.ssl_ca_cert_path = setup_.tlsCertPath;
|
||||
// Present a client cert for mutual TLS. When both paths are
|
||||
// empty the connection falls back to one-way (server) TLS.
|
||||
exporterOpts.ssl_client_cert_path = setup_.tlsClientCertPath;
|
||||
exporterOpts.ssl_client_key_path = setup_.tlsClientKeyPath;
|
||||
}
|
||||
|
||||
auto exporter = otlp_http::OtlpHttpExporterFactory::Create(exporterOpts);
|
||||
auto exporter =
|
||||
otlp_http::OtlpHttpExporterFactory::Create(makeTraceExporterOptions(setup_));
|
||||
|
||||
// Configure batch processor
|
||||
trace_sdk::BatchSpanProcessorOptions processorOpts;
|
||||
@@ -686,7 +675,7 @@ public:
|
||||
return setup_.traceLedger;
|
||||
}
|
||||
|
||||
[[nodiscard]] std::string const&
|
||||
[[nodiscard]] ConsensusTraceStrategy
|
||||
getConsensusTraceStrategy() const override
|
||||
{
|
||||
return setup_.consensusTraceStrategy;
|
||||
@@ -738,6 +727,22 @@ public:
|
||||
|
||||
} // namespace
|
||||
|
||||
opentelemetry::exporter::otlp::OtlpHttpExporterOptions
|
||||
makeTraceExporterOptions(Telemetry::Setup const& setup)
|
||||
{
|
||||
otlp_http::OtlpHttpExporterOptions opts;
|
||||
opts.url = setup.tracesEndpoint;
|
||||
if (setup.useTls)
|
||||
{
|
||||
opts.ssl_ca_cert_path = setup.tlsCertPath;
|
||||
// Present a client cert for mutual TLS. When both paths are
|
||||
// empty the connection falls back to one-way (server) TLS.
|
||||
opts.ssl_client_cert_path = setup.tlsClientCertPath;
|
||||
opts.ssl_client_key_path = setup.tlsClientKeyPath;
|
||||
}
|
||||
return opts;
|
||||
}
|
||||
|
||||
std::unique_ptr<Telemetry>
|
||||
makeTelemetry(Telemetry::Setup const& setup, beast::Journal journal)
|
||||
{
|
||||
|
||||
@@ -19,6 +19,7 @@
|
||||
#include <optional>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
#include <system_error>
|
||||
#include <type_traits>
|
||||
|
||||
@@ -54,6 +55,7 @@ constexpr char const* traceConsensus = "trace_consensus";
|
||||
constexpr char const* traceRpc = "trace_rpc";
|
||||
constexpr char const* tracePeer = "trace_peer";
|
||||
constexpr char const* traceLedger = "trace_ledger";
|
||||
constexpr char const* consensusTraceStrategy = "consensus_trace_strategy";
|
||||
} // namespace key
|
||||
|
||||
/**
|
||||
@@ -226,6 +228,61 @@ requirePositive(std::chrono::milliseconds value, char const* configKey)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Throw unless an endpoint URL is one the client certificate can be used on.
|
||||
*
|
||||
* The OTLP/HTTP exporter turns TLS on from the URL scheme alone, and matches
|
||||
* "https:" exactly and case-sensitively. So a client certificate only reaches
|
||||
* the collector on an https endpoint, and this check is what holds that
|
||||
* invariant: with a client certificate configured, the endpoint is an https URL.
|
||||
* "https://" is required in full, which is stricter than the exporter's own
|
||||
* test, so anything this accepts the exporter also treats as TLS.
|
||||
*
|
||||
* @param endpoint Endpoint URL from the config, or the built-in default.
|
||||
* @param configKey Config key the URL came from, named in the message.
|
||||
* @throws std::runtime_error If the URL does not begin with "https://".
|
||||
*/
|
||||
void
|
||||
requireHttpsEndpoint(std::string const& endpoint, char const* configKey)
|
||||
{
|
||||
constexpr std::string_view kHttpsPrefix{"https://"};
|
||||
|
||||
if (std::string_view{endpoint}.starts_with(kHttpsPrefix))
|
||||
return;
|
||||
|
||||
Throw<std::runtime_error>(
|
||||
std::string("Invalid value '") + configKey + "' in " + kSectionLabel +
|
||||
": must start with '" + std::string{kHttpsPrefix} + "' when " + key::tlsClientCert +
|
||||
" is set, but is '" + endpoint + "'.");
|
||||
}
|
||||
|
||||
/**
|
||||
* Map a `consensus_trace_strategy` value onto its enumerator.
|
||||
*
|
||||
* Only the two documented spellings are accepted. A typo would otherwise pick
|
||||
* the default silently, and the operator would never learn the setting had no
|
||||
* effect. Matching is exact and case-sensitive, like every other value in this
|
||||
* section.
|
||||
*
|
||||
* @param value Raw config value; empty means the key was absent.
|
||||
* @return The matching strategy, or Deterministic when the key was absent.
|
||||
* @throws std::runtime_error If the value is neither documented spelling.
|
||||
*/
|
||||
[[nodiscard]] ConsensusTraceStrategy
|
||||
readConsensusTraceStrategy(std::string const& value)
|
||||
{
|
||||
if (value.empty() || value == strategyName(ConsensusTraceStrategy::Deterministic))
|
||||
return ConsensusTraceStrategy::Deterministic;
|
||||
|
||||
if (value == strategyName(ConsensusTraceStrategy::Random))
|
||||
return ConsensusTraceStrategy::Random;
|
||||
|
||||
Throw<std::runtime_error>(
|
||||
std::string("Invalid value '") + key::consensusTraceStrategy + "' in " + kSectionLabel +
|
||||
": must be '" + strategyName(ConsensusTraceStrategy::Deterministic) + "' or '" +
|
||||
strategyName(ConsensusTraceStrategy::Random) + "'.");
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
/**
|
||||
@@ -308,6 +365,15 @@ makeTelemetrySetup(
|
||||
"(set use_tls=1 to enable mutual TLS, or remove the cert paths).");
|
||||
}
|
||||
|
||||
// Still inside the enabled branch, and checked before the files are
|
||||
// opened so a scheme problem is not hidden behind a path problem. The
|
||||
// exporter reads TLS off the endpoint scheme, so a client certificate is
|
||||
// only presented on an https endpoint. tls_ca_cert is left out of this
|
||||
// check: it only names a trust store, while a client certificate is this
|
||||
// node's own identity and has to reach the collector to mean anything.
|
||||
if (!setup.tlsClientCertPath.empty())
|
||||
requireHttpsEndpoint(setup.tracesEndpoint, key::tracesEndpoint);
|
||||
|
||||
// Still inside the enabled branch. The exporter opens these files only
|
||||
// when TLS is on, so check them only then: a bad path behind use_tls=0
|
||||
// stops nothing. Checking here turns what would otherwise surface much
|
||||
@@ -376,7 +442,7 @@ makeTelemetrySetup(
|
||||
setup.traceLedger = section.valueOr<int>(key::traceLedger, 1) != 0;
|
||||
|
||||
setup.consensusTraceStrategy =
|
||||
section.valueOr<std::string>("consensus_trace_strategy", "deterministic");
|
||||
readConsensusTraceStrategy(section.valueOr<std::string>(key::consensusTraceStrategy, ""));
|
||||
|
||||
return setup;
|
||||
}
|
||||
|
||||
@@ -171,14 +171,13 @@ public:
|
||||
}
|
||||
|
||||
/**
|
||||
* @return A fixed strategy label; the scope tests do not exercise
|
||||
* deterministic trace-id correlation, so any stable value works.
|
||||
* @return A fixed strategy; the scope tests do not exercise trace-id
|
||||
* correlation, so either value works.
|
||||
*/
|
||||
[[nodiscard]] std::string const&
|
||||
[[nodiscard]] ConsensusTraceStrategy
|
||||
getConsensusTraceStrategy() const override
|
||||
{
|
||||
static std::string const kStrategy{"none"};
|
||||
return kStrategy;
|
||||
return ConsensusTraceStrategy::Deterministic;
|
||||
}
|
||||
|
||||
opentelemetry::nostd::shared_ptr<opentelemetry::trace::Tracer>
|
||||
|
||||
@@ -56,6 +56,25 @@ constexpr char const* pairingError = "must be set together";
|
||||
constexpr char const* useTlsError = "require use_tls=1";
|
||||
constexpr char const* readError = "cannot be read";
|
||||
|
||||
/**
|
||||
* Endpoint values and the message fragment of the scheme guard.
|
||||
*
|
||||
* keyEndpoint is the config key, spelled once for the same reason as the two
|
||||
* client-certificate keys above. httpEndpoint and httpsEndpoint differ only in
|
||||
* scheme, so a case that swaps them changes nothing else. defaultEndpoint is
|
||||
* the parser's own default, restated here so the omitted-key case can assert
|
||||
* that the default is what got rejected; if the default ever changes, the case
|
||||
* that names it fails rather than quietly testing a different URL.
|
||||
*
|
||||
* schemeError occurs in no other message in this file, so matching it proves
|
||||
* the scheme guard fired and not the pairing, use_tls or readability guard.
|
||||
*/
|
||||
constexpr char const* keyEndpoint = "traces_endpoint";
|
||||
constexpr char const* httpEndpoint = "http://collector:4318/v1/traces";
|
||||
constexpr char const* httpsEndpoint = "https://collector:4318/v1/traces";
|
||||
constexpr char const* defaultEndpoint = "http://localhost:4318/v1/traces";
|
||||
constexpr char const* schemeError = "must start with 'https://'";
|
||||
|
||||
/**
|
||||
* Build a [telemetry] section carrying only the `enabled` key.
|
||||
*
|
||||
@@ -122,6 +141,7 @@ namespace key {
|
||||
constexpr char const* batchSize = "batch_size";
|
||||
constexpr char const* batchDelayMs = "batch_delay_ms";
|
||||
constexpr char const* maxQueueSize = "max_queue_size";
|
||||
constexpr char const* consensusTraceStrategy = "consensus_trace_strategy";
|
||||
} // namespace key
|
||||
|
||||
/**
|
||||
@@ -260,6 +280,7 @@ TEST(TelemetryConfig, setup_defaults)
|
||||
EXPECT_TRUE(s.traceRpc);
|
||||
EXPECT_TRUE(s.tracePeer);
|
||||
EXPECT_TRUE(s.traceLedger);
|
||||
EXPECT_EQ(s.consensusTraceStrategy, telemetry::ConsensusTraceStrategy::Deterministic);
|
||||
}
|
||||
|
||||
TEST(TelemetryConfig, parse_empty_section)
|
||||
@@ -341,6 +362,7 @@ TEST(TelemetryConfig, mtls_cert_and_key_both_set)
|
||||
auto const key = mtls::writeCertFile(dir.file("client.key"));
|
||||
Section section = mtls::makeSection(true);
|
||||
section.set("use_tls", "1");
|
||||
section.set(mtls::keyEndpoint, mtls::httpsEndpoint);
|
||||
section.set(mtls::keyClientCert, cert);
|
||||
section.set(mtls::keyClientKey, key);
|
||||
|
||||
@@ -463,6 +485,7 @@ TEST(TelemetryConfig, tls_missing_client_cert_file_throws)
|
||||
auto const absentCert = dir.file("absent.pem");
|
||||
Section section = mtls::makeSection(true);
|
||||
section.set("use_tls", "1");
|
||||
section.set(mtls::keyEndpoint, mtls::httpsEndpoint);
|
||||
section.set(mtls::keyClientCert, absentCert);
|
||||
section.set(mtls::keyClientKey, mtls::writeCertFile(dir.file("k.pem")));
|
||||
|
||||
@@ -480,6 +503,7 @@ TEST(TelemetryConfig, tls_missing_client_key_file_throws)
|
||||
auto const absentKey = dir.file("absent.key");
|
||||
Section section = mtls::makeSection(true);
|
||||
section.set("use_tls", "1");
|
||||
section.set(mtls::keyEndpoint, mtls::httpsEndpoint);
|
||||
section.set(mtls::keyClientCert, mtls::writeCertFile(dir.file("c.pem")));
|
||||
section.set(mtls::keyClientKey, absentKey);
|
||||
|
||||
@@ -515,6 +539,7 @@ TEST(TelemetryConfig, tls_readable_files_are_accepted)
|
||||
auto const key = mtls::writeCertFile(dir.file("k.pem"));
|
||||
Section section = mtls::makeSection(true);
|
||||
section.set("use_tls", "1");
|
||||
section.set(mtls::keyEndpoint, mtls::httpsEndpoint);
|
||||
section.set("tls_ca_cert", ca);
|
||||
section.set(mtls::keyClientCert, cert);
|
||||
section.set(mtls::keyClientKey, key);
|
||||
@@ -568,6 +593,123 @@ TEST(TelemetryConfig, tls_ca_cert_not_checked_when_use_tls_off)
|
||||
EXPECT_EQ(setup.tlsCertPath, absentCa);
|
||||
}
|
||||
|
||||
TEST(TelemetryConfig, mtls_client_cert_on_a_plain_http_endpoint_throws)
|
||||
{
|
||||
// Full mTLS on an http:// endpoint. Both paths are set and readable and
|
||||
// use_tls=1, so the pairing, use_tls and readability guards are all
|
||||
// satisfied and the scheme guard is the only reachable throw. The message
|
||||
// must name the endpoint key and the rejected URL.
|
||||
TempDir const dir;
|
||||
Section section = mtls::makeSection(true);
|
||||
section.set("use_tls", "1");
|
||||
section.set(mtls::keyEndpoint, mtls::httpEndpoint);
|
||||
section.set(mtls::keyClientCert, mtls::writeCertFile(dir.file("c.pem")));
|
||||
section.set(mtls::keyClientKey, mtls::writeCertFile(dir.file("k.pem")));
|
||||
|
||||
EXPECT_THAT(
|
||||
[§ion] { mtls::parseSection(section); },
|
||||
ThrowsMessage<std::runtime_error>(AllOf(
|
||||
HasSubstr(mtls::schemeError),
|
||||
HasSubstr(mtls::keyEndpoint),
|
||||
HasSubstr(mtls::httpEndpoint))));
|
||||
}
|
||||
|
||||
TEST(TelemetryConfig, mtls_client_cert_with_the_default_endpoint_throws)
|
||||
{
|
||||
// The endpoint key is omitted, so the parser's own default applies — and
|
||||
// that default is plain HTTP. This is the case an operator reaches by
|
||||
// configuring mTLS and nothing else, so it must be rejected exactly like
|
||||
// an explicit http:// URL, naming the default it rejected.
|
||||
TempDir const dir;
|
||||
Section section = mtls::makeSection(true);
|
||||
section.set("use_tls", "1");
|
||||
section.set(mtls::keyClientCert, mtls::writeCertFile(dir.file("c.pem")));
|
||||
section.set(mtls::keyClientKey, mtls::writeCertFile(dir.file("k.pem")));
|
||||
|
||||
EXPECT_THAT(
|
||||
[§ion] { mtls::parseSection(section); },
|
||||
ThrowsMessage<std::runtime_error>(AllOf(
|
||||
HasSubstr(mtls::schemeError),
|
||||
HasSubstr(mtls::keyEndpoint),
|
||||
HasSubstr(mtls::defaultEndpoint))));
|
||||
}
|
||||
|
||||
TEST(TelemetryConfig, mtls_client_cert_on_an_https_endpoint_is_accepted)
|
||||
{
|
||||
// The same configuration as the two cases above with only the scheme
|
||||
// changed, so nothing but the scheme can explain the different outcome.
|
||||
TempDir const dir;
|
||||
auto const cert = mtls::writeCertFile(dir.file("c.pem"));
|
||||
auto const key = mtls::writeCertFile(dir.file("k.pem"));
|
||||
Section section = mtls::makeSection(true);
|
||||
section.set("use_tls", "1");
|
||||
section.set(mtls::keyEndpoint, mtls::httpsEndpoint);
|
||||
section.set(mtls::keyClientCert, cert);
|
||||
section.set(mtls::keyClientKey, key);
|
||||
|
||||
telemetry::Telemetry::Setup setup;
|
||||
ASSERT_NO_THROW(setup = mtls::parseSection(section));
|
||||
EXPECT_EQ(setup.tracesEndpoint, mtls::httpsEndpoint);
|
||||
EXPECT_EQ(setup.tlsClientCertPath, cert);
|
||||
EXPECT_EQ(setup.tlsClientKeyPath, key);
|
||||
}
|
||||
|
||||
TEST(TelemetryConfig, mtls_scheme_check_is_case_sensitive_like_the_exporter)
|
||||
{
|
||||
// The exporter compares the scheme byte for byte, so "HTTPS://" leaves it
|
||||
// exporting in the clear. Accepting the upper-case spelling here would let
|
||||
// a configuration pass validation and still drop the client identity.
|
||||
TempDir const dir;
|
||||
Section section = mtls::makeSection(true);
|
||||
section.set("use_tls", "1");
|
||||
section.set(mtls::keyEndpoint, "HTTPS://collector:4318/v1/traces");
|
||||
section.set(mtls::keyClientCert, mtls::writeCertFile(dir.file("c.pem")));
|
||||
section.set(mtls::keyClientKey, mtls::writeCertFile(dir.file("k.pem")));
|
||||
|
||||
EXPECT_THAT(
|
||||
[§ion] { mtls::parseSection(section); },
|
||||
ThrowsMessage<std::runtime_error>(HasSubstr(mtls::schemeError)));
|
||||
}
|
||||
|
||||
TEST(TelemetryConfig, one_way_tls_on_a_plain_http_endpoint_is_accepted)
|
||||
{
|
||||
// The control for the guard's scope: same http:// endpoint and use_tls=1,
|
||||
// but no client certificate. Only a client identity can be silently
|
||||
// dropped, so this configuration is left alone. Widen the guard to every
|
||||
// use_tls=1 node and this case starts failing.
|
||||
TempDir const dir;
|
||||
auto const ca = mtls::writeCertFile(dir.file("ca.pem"));
|
||||
Section section = mtls::makeSection(true);
|
||||
section.set("use_tls", "1");
|
||||
section.set(mtls::keyEndpoint, mtls::httpEndpoint);
|
||||
section.set("tls_ca_cert", ca);
|
||||
|
||||
telemetry::Telemetry::Setup setup;
|
||||
ASSERT_NO_THROW(setup = mtls::parseSection(section));
|
||||
EXPECT_EQ(setup.tracesEndpoint, mtls::httpEndpoint);
|
||||
EXPECT_EQ(setup.tlsCertPath, ca);
|
||||
EXPECT_TRUE(setup.tlsClientCertPath.empty());
|
||||
}
|
||||
|
||||
TEST(TelemetryConfig, mtls_scheme_not_checked_when_telemetry_disabled)
|
||||
{
|
||||
// Telemetry off, so a leftover mTLS block on a plain endpoint must not stop
|
||||
// the node from booting. use_tls stays 1 and the paths are absent files, so
|
||||
// the `enabled` gate is the only thing suppressing every guard.
|
||||
TempDir const dir;
|
||||
Section section = mtls::makeSection(false);
|
||||
section.set("use_tls", "1");
|
||||
section.set(mtls::keyEndpoint, mtls::httpEndpoint);
|
||||
section.set(mtls::keyClientCert, mtls::clientCert);
|
||||
section.set(mtls::keyClientKey, mtls::clientKey);
|
||||
|
||||
telemetry::Telemetry::Setup setup;
|
||||
ASSERT_NO_THROW(setup = mtls::parseSection(section));
|
||||
EXPECT_FALSE(setup.enabled);
|
||||
EXPECT_EQ(setup.tracesEndpoint, mtls::httpEndpoint);
|
||||
EXPECT_EQ(setup.tlsClientCertPath, mtls::clientCert);
|
||||
}
|
||||
|
||||
TEST(TelemetryConfig, batch_settings_accept_the_lower_bound_exactly)
|
||||
{
|
||||
auto const setup =
|
||||
@@ -859,6 +1001,71 @@ TEST(TelemetryConfig, metric_export_small_interval_against_default_timeout_throw
|
||||
HasSubstr(cadence::keyInterval))));
|
||||
}
|
||||
|
||||
TEST(TelemetryConfig, consensus_trace_strategy_names_match_the_config_spellings)
|
||||
{
|
||||
// strategyName() feeds both the parser and the trace_strategy span
|
||||
// attribute, so these two strings are the whole public vocabulary.
|
||||
EXPECT_STREQ(
|
||||
telemetry::strategyName(telemetry::ConsensusTraceStrategy::Deterministic), "deterministic");
|
||||
EXPECT_STREQ(telemetry::strategyName(telemetry::ConsensusTraceStrategy::Random), "random");
|
||||
}
|
||||
|
||||
TEST(TelemetryConfig, consensus_trace_strategy_defaults_to_deterministic)
|
||||
{
|
||||
// The key is absent, so the default applies. Deterministic is the only
|
||||
// strategy in use, and a default of Random would break cross-node
|
||||
// correlation on every node that omits the key.
|
||||
EXPECT_EQ(
|
||||
parseBatch({}).consensusTraceStrategy, telemetry::ConsensusTraceStrategy::Deterministic);
|
||||
}
|
||||
|
||||
TEST(TelemetryConfig, consensus_trace_strategy_accepts_deterministic)
|
||||
{
|
||||
EXPECT_EQ(
|
||||
parseBatch({{key::consensusTraceStrategy, "deterministic"}}).consensusTraceStrategy,
|
||||
telemetry::ConsensusTraceStrategy::Deterministic);
|
||||
}
|
||||
|
||||
TEST(TelemetryConfig, consensus_trace_strategy_accepts_random)
|
||||
{
|
||||
// Random is experimental and unused, but it is a documented spelling, so
|
||||
// the parser must still map it to its own enumerator rather than reject it
|
||||
// or fold it into the default.
|
||||
EXPECT_EQ(
|
||||
parseBatch({{key::consensusTraceStrategy, "random"}}).consensusTraceStrategy,
|
||||
telemetry::ConsensusTraceStrategy::Random);
|
||||
}
|
||||
|
||||
TEST(TelemetryConfig, consensus_trace_strategy_empty_value_is_the_default)
|
||||
{
|
||||
// `consensus_trace_strategy=` with nothing after it. An empty value means
|
||||
// the operator wrote the key and no value, which is the default, not a typo.
|
||||
EXPECT_EQ(
|
||||
parseBatch({{key::consensusTraceStrategy, ""}}).consensusTraceStrategy,
|
||||
telemetry::ConsensusTraceStrategy::Deterministic);
|
||||
}
|
||||
|
||||
TEST(TelemetryConfig, consensus_trace_strategy_rejects_an_undocumented_value)
|
||||
{
|
||||
// "attribute" is not a spelling this parser accepts. Rejecting rather than
|
||||
// defaulting is the point: a silent fallback would leave the operator
|
||||
// believing a setting took effect.
|
||||
EXPECT_EQ(
|
||||
batchRejection({{key::consensusTraceStrategy, "attribute"}}),
|
||||
"Invalid value 'consensus_trace_strategy' in [telemetry]: must be 'deterministic' or "
|
||||
"'random'.");
|
||||
}
|
||||
|
||||
TEST(TelemetryConfig, consensus_trace_strategy_matching_is_case_sensitive)
|
||||
{
|
||||
// Every other value in this section is matched exactly, so "Random" is a
|
||||
// typo and must be reported as one.
|
||||
EXPECT_EQ(
|
||||
batchRejection({{key::consensusTraceStrategy, "Random"}}),
|
||||
"Invalid value 'consensus_trace_strategy' in [telemetry]: must be 'deterministic' or "
|
||||
"'random'.");
|
||||
}
|
||||
|
||||
TEST(TelemetryConfig, null_telemetry_factory)
|
||||
{
|
||||
telemetry::Telemetry::Setup setup;
|
||||
|
||||
150
src/tests/libxrpl/telemetry/TraceExporterOptions.cpp
Normal file
150
src/tests/libxrpl/telemetry/TraceExporterOptions.cpp
Normal file
@@ -0,0 +1,150 @@
|
||||
// The whole file is telemetry-only: makeTraceExporterOptions() and the OTel
|
||||
// exporter options type it returns are both declared behind
|
||||
// XRPL_ENABLE_TELEMETRY, so without it there is nothing here to test.
|
||||
#ifdef XRPL_ENABLE_TELEMETRY
|
||||
|
||||
#include <xrpl/basics/FileUtilities.h>
|
||||
#include <xrpl/config/BasicConfig.h>
|
||||
#include <xrpl/telemetry/Telemetry.h>
|
||||
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
#include <fstream>
|
||||
#include <string>
|
||||
|
||||
using namespace xrpl;
|
||||
|
||||
namespace {
|
||||
|
||||
/**
|
||||
* Distinct placeholder paths for the three TLS files.
|
||||
*
|
||||
* They are deliberately different from one another in more than a suffix, so a
|
||||
* certificate written into the key field, or a CA bundle written into either,
|
||||
* shows up as an inequality naming both paths rather than as a near-miss.
|
||||
*/
|
||||
namespace tlsPath {
|
||||
constexpr char const* ca = "/etc/xrpl/tls/collector-ca-bundle.pem";
|
||||
constexpr char const* clientCert = "/etc/xrpl/tls/node-client-certificate.pem";
|
||||
constexpr char const* clientKey = "/etc/xrpl/tls/node-client-private-key.pem";
|
||||
} // namespace tlsPath
|
||||
|
||||
constexpr char const* kHttpsEndpoint = "https://collector.example:4318/v1/traces";
|
||||
|
||||
/**
|
||||
* Build a Setup with mutual TLS configured and nothing else set.
|
||||
*
|
||||
* The struct is filled directly rather than parsed, so these cases isolate the
|
||||
* Setup-to-exporter mapping. The end-to-end case at the bottom of this file
|
||||
* covers the config-file side.
|
||||
*
|
||||
* @param useTls Value for Setup::useTls; the only thing the cases vary.
|
||||
* @return The populated Setup.
|
||||
*/
|
||||
telemetry::Telemetry::Setup
|
||||
makeMtlsSetup(bool useTls)
|
||||
{
|
||||
telemetry::Telemetry::Setup setup;
|
||||
setup.enabled = true;
|
||||
setup.tracesEndpoint = kHttpsEndpoint;
|
||||
setup.useTls = useTls;
|
||||
setup.tlsCertPath = tlsPath::ca;
|
||||
setup.tlsClientCertPath = tlsPath::clientCert;
|
||||
setup.tlsClientKeyPath = tlsPath::clientKey;
|
||||
return setup;
|
||||
}
|
||||
|
||||
/**
|
||||
* Write a placeholder certificate file at the given path.
|
||||
*
|
||||
* makeTelemetrySetup() only needs the file to exist and be readable; nothing
|
||||
* checks that the contents parse as PEM.
|
||||
*
|
||||
* @param path Where to write the file, typically from TempDir::file().
|
||||
* @return The same path, ready to pass to Section::set().
|
||||
*/
|
||||
std::string
|
||||
writeCertFile(std::string const& path)
|
||||
{
|
||||
std::ofstream out{path};
|
||||
out << "placeholder\n";
|
||||
out.close();
|
||||
EXPECT_TRUE(out.good()) << "could not create " << path;
|
||||
return path;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
TEST(TraceExporterOptions, mtls_paths_reach_the_matching_exporter_fields)
|
||||
{
|
||||
// The assertion the exporter boundary was missing: each configured path
|
||||
// lands in its own field. The three paths differ, so swapping the client
|
||||
// certificate and key, or writing the CA bundle into a client field, fails
|
||||
// here instead of failing as a TLS handshake error on a live collector.
|
||||
auto const opts = telemetry::makeTraceExporterOptions(makeMtlsSetup(true));
|
||||
|
||||
EXPECT_EQ(opts.url, kHttpsEndpoint);
|
||||
EXPECT_EQ(opts.ssl_ca_cert_path, tlsPath::ca);
|
||||
EXPECT_EQ(opts.ssl_client_cert_path, tlsPath::clientCert);
|
||||
EXPECT_EQ(opts.ssl_client_key_path, tlsPath::clientKey);
|
||||
}
|
||||
|
||||
TEST(TraceExporterOptions, one_way_tls_leaves_the_client_fields_empty)
|
||||
{
|
||||
// The one-way TLS control: a CA bundle and no client identity. The client
|
||||
// fields must stay empty, so an unset client certificate cannot pick up a
|
||||
// path from somewhere else in Setup.
|
||||
auto setup = makeMtlsSetup(true);
|
||||
setup.tlsClientCertPath.clear();
|
||||
setup.tlsClientKeyPath.clear();
|
||||
|
||||
auto const opts = telemetry::makeTraceExporterOptions(setup);
|
||||
|
||||
EXPECT_EQ(opts.url, kHttpsEndpoint);
|
||||
EXPECT_EQ(opts.ssl_ca_cert_path, tlsPath::ca);
|
||||
EXPECT_EQ(opts.ssl_client_cert_path, "");
|
||||
EXPECT_EQ(opts.ssl_client_key_path, "");
|
||||
}
|
||||
|
||||
TEST(TraceExporterOptions, use_tls_off_passes_no_tls_paths_at_all)
|
||||
{
|
||||
// Every path is configured and use_tls is off, so all three fields must
|
||||
// stay empty while the URL still goes through. This is the only case that
|
||||
// distinguishes "gated on use_tls" from "always copied".
|
||||
auto const opts = telemetry::makeTraceExporterOptions(makeMtlsSetup(false));
|
||||
|
||||
EXPECT_EQ(opts.url, kHttpsEndpoint);
|
||||
EXPECT_EQ(opts.ssl_ca_cert_path, "");
|
||||
EXPECT_EQ(opts.ssl_client_cert_path, "");
|
||||
EXPECT_EQ(opts.ssl_client_key_path, "");
|
||||
}
|
||||
|
||||
TEST(TraceExporterOptions, config_section_reaches_the_exporter_options)
|
||||
{
|
||||
// The whole path in one case: a [telemetry] section with an https endpoint
|
||||
// and two different real files, parsed by makeTelemetrySetup() and then
|
||||
// mapped. Nothing between the config file and the exporter is stubbed, so a
|
||||
// break anywhere along it lands here.
|
||||
TempDir const dir;
|
||||
auto const cert = writeCertFile(dir.file("node-client-certificate.pem"));
|
||||
auto const key = writeCertFile(dir.file("node-client-private-key.pem"));
|
||||
ASSERT_NE(cert, key);
|
||||
|
||||
Section section;
|
||||
section.set("enabled", "1");
|
||||
section.set("traces_endpoint", kHttpsEndpoint);
|
||||
section.set("use_tls", "1");
|
||||
section.set("tls_client_cert", cert);
|
||||
section.set("tls_client_key", key);
|
||||
|
||||
auto const setup = telemetry::makeTelemetrySetup(section, "nHUtest123", "2.0.0", 0);
|
||||
auto const opts = telemetry::makeTraceExporterOptions(setup);
|
||||
|
||||
EXPECT_EQ(opts.url, kHttpsEndpoint);
|
||||
EXPECT_EQ(opts.ssl_client_cert_path, cert);
|
||||
EXPECT_EQ(opts.ssl_client_key_path, key);
|
||||
// No tls_ca_cert in the section, so the exporter keeps its own trust store.
|
||||
EXPECT_EQ(opts.ssl_ca_cert_path, "");
|
||||
}
|
||||
|
||||
#endif // XRPL_ENABLE_TELEMETRY
|
||||
@@ -697,6 +697,11 @@ RCLConsensus::Adaptor::doAccept(
|
||||
|
||||
JLOG(j_.debug()) << "Building canonical tx set: " << retriableTxs.key();
|
||||
|
||||
// One tx.included event per transaction of the agreed consensus set, which
|
||||
// is not yet the accepted ledger: buildLCL() below applies these and some
|
||||
// may fail, so the events are a superset of what the ledger ends up with. A
|
||||
// transaction whose bytes cannot be parsed gets no event at all.
|
||||
//
|
||||
// txCount and the per-transaction event feed the span and nothing else, so
|
||||
// both are guarded on the span being active. Unguarded, every accepted
|
||||
// ledger builds one 64-character hash string per transaction that no one
|
||||
@@ -1361,19 +1366,19 @@ RCLConsensus::Adaptor::startRoundTracing(RCLCxLedger const& prevLgr)
|
||||
if (roundSpan_)
|
||||
roundSpan_.reset();
|
||||
|
||||
auto const& strategy = app_.getTelemetry().getConsensusTraceStrategy();
|
||||
auto const strategy = app_.getTelemetry().getConsensusTraceStrategy();
|
||||
|
||||
telemetry::SpanContext const* const link =
|
||||
prevRoundSpanContext_.isValid() ? &prevRoundSpanContext_ : nullptr;
|
||||
|
||||
if (strategy == "attribute")
|
||||
if (strategy == telemetry::ConsensusTraceStrategy::Random)
|
||||
{
|
||||
// Non-deterministic strategy: each node gets a random trace_id,
|
||||
// correlated via the consensus_ledger_id attribute rather than a
|
||||
// shared trace_id. Still attach a follows-from link to the prior
|
||||
// round so consecutive rounds stay navigable. linkedSpan is not
|
||||
// TraceCategory-aware, so gate it explicitly to match the gating
|
||||
// of the hashSpan/span factories used below.
|
||||
// Experimental strategy, not used on a live network: each node gets a
|
||||
// random trace_id, so one round arrives as one trace per node, joinable
|
||||
// only by the consensus_ledger_id attribute. Still attach a follows-from
|
||||
// link to the prior round so consecutive rounds stay navigable.
|
||||
// linkedSpan is not TraceCategory-aware, so gate it explicitly to match
|
||||
// the gating of the hashSpan/span factories used below.
|
||||
if (link != nullptr && app_.getTelemetry().shouldTraceConsensus())
|
||||
{
|
||||
roundSpan_.emplace(telemetry::SpanGuard::linkedSpan(cs::round, *link));
|
||||
@@ -1387,7 +1392,7 @@ RCLConsensus::Adaptor::startRoundTracing(RCLCxLedger const& prevLgr)
|
||||
}
|
||||
else
|
||||
{
|
||||
// "deterministic" (the default): derive the trace_id from the previous
|
||||
// Deterministic (the default): derive the trace_id from the previous
|
||||
// ledger hash so all validators tracing the same round share one trace.
|
||||
roundSpan_.emplace(
|
||||
telemetry::SpanGuard::hashSpan(
|
||||
@@ -1405,7 +1410,7 @@ RCLConsensus::Adaptor::startRoundTracing(RCLCxLedger const& prevLgr)
|
||||
|
||||
roundSpan_->setAttribute(cs::attr::ledgerId, to_string(prevLgr.id()).c_str());
|
||||
roundSpan_->setAttribute(cs::attr::ledgerSeq, static_cast<int64_t>(prevLgr.seq()) + 1);
|
||||
roundSpan_->setAttribute(cs::attr::traceStrategy, strategy.c_str());
|
||||
roundSpan_->setAttribute(cs::attr::traceStrategy, telemetry::strategyName(strategy));
|
||||
roundSpan_->setAttribute(cs::attr::roundId, static_cast<int64_t>(prevLgr.seq()) + 1);
|
||||
roundSpan_->setAttribute(cs::attr::previousLedgerSeq, static_cast<int64_t>(prevLgr.seq()));
|
||||
roundSpan_->setAttribute(cs::attr::previousProposers, static_cast<int64_t>(prevProposers_));
|
||||
|
||||
@@ -92,8 +92,8 @@ class RCLConsensus
|
||||
* Span for the current consensus round.
|
||||
*
|
||||
* Created in preStartRound(), ended (via reset()) when the next
|
||||
* round begins. When consensusTraceStrategy is "deterministic",
|
||||
* the trace_id is derived from previousLedger.id() so that all
|
||||
* round begins. Under ConsensusTraceStrategy::Deterministic the
|
||||
* trace_id is derived from previousLedger.id() so that all
|
||||
* validators in the same round share the same trace_id.
|
||||
*
|
||||
* Thread-free: a SpanGuard owns no thread-local Scope, so it can be
|
||||
|
||||
@@ -121,7 +121,10 @@ inline constexpr auto expiredCount = makeStr("expired_count");
|
||||
*/
|
||||
inline constexpr auto terCode = makeStr("ter_code");
|
||||
/**
|
||||
* "retries_remaining" — retries left before discard.
|
||||
* "retries_remaining" — retries left as this attempt started, recorded before
|
||||
* the transaction is applied and before any decrement. A span with
|
||||
* txq_status="retried" therefore always shows a non-zero count; exhaustion
|
||||
* shows up as txq_status="failed" with zero.
|
||||
*/
|
||||
inline constexpr auto retriesRemaining = makeStr("retries_remaining");
|
||||
/**
|
||||
|
||||
Reference in New Issue
Block a user