diff --git a/.github/scripts/levelization/results/loops.txt b/.github/scripts/levelization/results/loops.txt index 852a99f202..a1b2bcd92e 100644 --- a/.github/scripts/levelization/results/loops.txt +++ b/.github/scripts/levelization/results/loops.txt @@ -13,9 +13,6 @@ Loop: xrpld.app xrpld.rpc Loop: xrpld.app xrpld.shamap xrpld.shamap > xrpld.app -Loop: xrpld.app xrpld.telemetry - xrpld.telemetry ~= xrpld.app - Loop: xrpld.overlay xrpld.rpc xrpld.rpc ~= xrpld.overlay diff --git a/.github/scripts/levelization/results/ordering.txt b/.github/scripts/levelization/results/ordering.txt index adb494a757..2f12ece6af 100644 --- a/.github/scripts/levelization/results/ordering.txt +++ b/.github/scripts/levelization/results/ordering.txt @@ -47,6 +47,7 @@ libxrpl.shamap > xrpl.protocol libxrpl.shamap > xrpl.shamap libxrpl.telemetry > xrpl.basics libxrpl.telemetry > xrpl.config +libxrpl.telemetry > xrpl.protocol libxrpl.telemetry > xrpl.telemetry libxrpl.tx > xrpl.basics libxrpl.tx > xrpl.conditions @@ -263,6 +264,7 @@ xrpld.app > xrpl.config xrpld.app > xrpl.core xrpld.app > xrpld.consensus xrpld.app > xrpld.core +xrpld.app > xrpld.telemetry xrpld.app > xrpl.json xrpld.app > xrpl.ledger xrpld.app > xrpl.net diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c2aa11ba28..b787d06ad7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -321,7 +321,7 @@ other layer must match them. A CI check enforces this end to end. 3. Collision qualifier: `_` — only when a bare name would collide with a DIFFERENT concept in the shared spanmetrics label space, or with the OTel-reserved `status` key (e.g. `rpc_status`, `grpc_status`, - `consensus_state`, `consensus_round`). This disambiguates distinct concepts + `consensus_phase`, `consensus_round`). This disambiguates distinct concepts that share a word; it is NOT used to tag the same concept with the workflow that emitted it — that is rule 2 (one shared name). 4. Resource attribute: dotted `xrpl..` — reserved ONLY for diff --git a/OpenTelemetryPlan/02-design-decisions.md b/OpenTelemetryPlan/02-design-decisions.md index 3bee549eac..9bfee122c6 100644 --- a/OpenTelemetryPlan/02-design-decisions.md +++ b/OpenTelemetryPlan/02-design-decisions.md @@ -185,7 +185,7 @@ agree with the code. A CI check enforces this end to end. 3. **Collision qualifier** → `_`, only when a bare name would collide with a DIFFERENT concept in the shared spanmetrics label space or with the OTel-reserved `status` key (e.g. `rpc_status`, `grpc_status`, - `consensus_state`, `consensus_round`, `consensus_mode`). This disambiguates + `consensus_phase`, `consensus_round`, `consensus_mode`). This disambiguates distinct concepts that share a word; it is NOT used to tag the same concept with its emitting workflow — that is rule 2 (one shared name). 4. **Resource attribute** → dotted `xrpl..`, reserved ONLY @@ -245,6 +245,8 @@ keys (the dotted form is reserved for resource scope per §2.3.3). | `tx_fee` | int64 | Fee in drops | | `tx_result` | string | `"tesSUCCESS"`, `"tecPATH_DRY"`, etc. | | `ledger_index` | int64 | Ledger containing transaction | +| `relay_count` | int64 | Peers the transaction was relayed to | +| `suppressed` | bool | `true` when HashRouter dropped a dup | #### Consensus Attributes @@ -278,12 +280,14 @@ Establish-phase gap fill and cross-node correlation attributes (Phase 4a): #### RPC Attributes -| Key | Type | Description | -| ---------- | ------ | ----------------------------------------------------- | -| `command` | string | Command name (per-span unique on `rpc.command`) | -| `version` | int64 | API version | -| `rpc_role` | string | `"admin"` or `"user"` (qualified — `role` is generic) | -| `params` | string | Sanitized parameters (optional) | +| Key | Type | Description | +| ------------- | ------- | ----------------------------------------------------------------------------- | +| `command` | string | Command name (per-span unique on `rpc.command`) | +| `version` | int64 | API version | +| `rpc_role` | string | `"admin"` or `"user"` (qualified — `role` is generic) | +| `params` | string | Sanitized parameters (optional) | +| `rpc_status` | string | Response status: `success` \| `error` (qualified — `status` is OTel-reserved) | +| `duration_ms` | float64 | Request duration in milliseconds | #### Peer & Message Attributes @@ -395,32 +399,48 @@ The following data is explicitly **excluded** from telemetry collection: #### Privacy Protection Mechanisms -| Mechanism | Description | -| ----------------------------- | ------------------------------------------------------------------------- | -| **Account Hashing** | `tx_account` is hashed at collector level before storage | -| **Configurable Redaction** | Sensitive fields can be excluded via `[telemetry]` config section | -| **Sampling** | Only 10% of traces recorded by default, reducing data exposure | -| **Local Control** | Node operators have full control over what gets exported | -| **No Raw Payloads** | Transaction content is never recorded, only metadata (hash, type, result) | -| **Collector-Level Filtering** | Additional redaction/hashing can be configured at OTel Collector | +| Mechanism | Description | +| ----------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| **Account Hashing** | Account addresses are hashed both SDK-side (`pathfind_source_account`, `pathfind_dest_account` — always hashed before emission) and again at the collector level, so raw addresses never reach storage | +| **Configurable Redaction** | Sensitive fields can be excluded via `[telemetry]` config section | +| **Collector Tail Sampling** | xrpld head sampling is fixed at 1.0 (every span emitted); the collector retains ~10% of non-error traces, reducing stored data exposure | +| **Sampling** | Only 10% of traces recorded by default, reducing data exposure | +| **Local Control** | Node operators have full control over what gets exported | +| **No Raw Payloads** | Transaction content is never recorded, only metadata (hash, type, result) | +| **Collector-Level Filtering** | Additional redaction/hashing can be configured at OTel Collector | + +#### Account Address Hashing + +Account addresses are **always** hashed before they reach the telemetry +backend — there is no opt-out flag and therefore no insecure-by-default +failure mode. Protection is applied in two independent layers: + +1. **SDK-side** (this node): the path-finding RPC handlers call + `redactAccount()` (`xrpl::telemetry`, `Redaction.h`) before setting the + `pathfind_source_account` / `pathfind_dest_account` span attributes. The + helper emits the first 16 characters of `sha512Half(address)` as + lowercase hex — deterministic (spans for one account still correlate) + but non-reversible. +2. **Collector-side** (defense-in-depth): an `attributes/hash` processor in + the OpenTelemetry Collector re-hashes those same attributes, so any node + that emitted a raw value is still redacted before storage. #### Collector-Level Data Protection The OpenTelemetry Collector can be configured (via an `attributes` processor) to hash or redact sensitive attributes before export — for example, hashing -`tx_account`, deleting `peer_address` to drop IP addresses, and deleting -`params` to redact request parameters. +`pathfind_source_account` / `pathfind_dest_account`, deleting `peer_address` +to drop IP addresses, and deleting `params` to redact request parameters. #### Configuration Options for Privacy In `xrpld.cfg`, operators control data collection granularity through the `[telemetry]` section. Besides `enabled`, per-component toggles (`trace_transactions`, `trace_consensus`, `trace_rpc`, `trace_peer` — the last -often disabled due to high volume) select which spans are emitted, and -redaction flags (`redact_account` to hash account addresses, `redact_peer_address` -to remove peer IP addresses) control SDK-level redaction before export. - -> **Note**: The `redact_account` configuration in `xrpld.cfg` controls SDK-level redaction before export, while collector-level filtering (see [Collector-Level Data Protection](#collector-level-data-protection) above) provides an additional defense-in-depth layer. Both can operate independently. +often disabled due to high volume) select which spans are emitted. Account +address hashing is not configurable: addresses are hashed unconditionally by +the SDK helper described above, with collector-level hashing as a second +layer. > **Key Principle**: Telemetry collects **operational metadata** (timing, counts, hashes) — never **sensitive content** (keys, balances, amounts, raw payloads). @@ -646,7 +666,7 @@ flowchart TB - **xrpld Process (dark gray)**: The single xrpld node running all three observability frameworks side by side. Each framework operates independently with no interference. - **PerfLog to perf.log**: PerfLog writes JSON-formatted event logs to a local file. Grafana can ingest these via Loki or a file-based datasource. - **Beast Insight to StatsD Server**: Insight sends aggregated metrics (counters, gauges) over UDP to a StatsD server. Grafana reads from StatsD-compatible backends like Graphite or Prometheus (via StatsD exporter). -- **OpenTelemetry to OTLP Collector**: OTel exports spans over OTLP/gRPC to a Collector, which then forwards to a trace backend (Tempo). +- **OpenTelemetry to OTLP Collector**: OTel exports spans over OTLP/HTTP to a Collector, which then forwards to a trace backend (Tempo). (OTLP/gRPC is future work — §2.2.2.) - **Grafana (red, unified UI)**: All three data streams converge in Grafana, enabling operators to correlate logs, metrics, and traces in a single dashboard. **Phase 7 target state**: Beast Insight routes to `OTelCollector` (new `Collector` implementation) which exports via OTLP/HTTP to the same collector endpoint as traces. StatsD UDP path becomes a deprecated fallback (`[insight] server=statsd`). See [06-implementation-phases.md §6.8](./06-implementation-phases.md) and [Phase7_taskList.md](./Phase7_taskList.md) for details. diff --git a/OpenTelemetryPlan/03-implementation-strategy.md b/OpenTelemetryPlan/03-implementation-strategy.md index eecd144617..922d3d4999 100644 --- a/OpenTelemetryPlan/03-implementation-strategy.md +++ b/OpenTelemetryPlan/03-implementation-strategy.md @@ -180,14 +180,14 @@ pie showData | TracerProvider singleton | ~64 KB | At startup | | BatchSpanProcessor (circular buffer) | ~16 KB | At startup | | BatchSpanProcessor (worker thread) | ~8 MB | At startup | -| OTLP exporter (gRPC channel init) | ~256 KB | At startup | +| OTLP/HTTP exporter (client init) | ~64 KB | At startup | | Propagator registry | ~8 KB | At startup | -| **Total static** | **~8.3 MB** | | +| **Total static** | **~8.1 MB** | | > **Why higher than earlier estimate**: The BatchSpanProcessor's circular buffer itself is only ~16 KB > (2049 x 8-byte `AtomicUniquePtr` entries), but it spawns a dedicated worker thread whose default -> stack size on Linux is ~8 MB. The OTLP gRPC exporter allocates memory for channel stubs and TLS -> initialization. The worker thread stack dominates the static footprint. +> stack size on Linux is ~8 MB. The OTLP/HTTP exporter allocates a small client and TLS +> initialization buffer. The worker thread stack dominates the static footprint. ### 3.5.2 Dynamic Memory diff --git a/OpenTelemetryPlan/05-configuration-reference.md b/OpenTelemetryPlan/05-configuration-reference.md index a48e410b49..04b51d45a6 100644 --- a/OpenTelemetryPlan/05-configuration-reference.md +++ b/OpenTelemetryPlan/05-configuration-reference.md @@ -21,8 +21,8 @@ The authoritative `[telemetry]` example lives in `cfg/xrpld-example.cfg`. Teleme | `endpoint` | string | `http://localhost:4318/v1/traces` | OTLP/HTTP collector endpoint | | `use_tls` | bool | `false` | Enable TLS for exporter connection | | `tls_ca_cert` | string | `""` | Path to CA certificate file | -| `tls_client_cert` | string | `""` | Path to node's client certificate (PEM) for mutual TLS; empty = one-way TLS | -| `tls_client_key` | string | `""` | Path to private key (PEM) for `tls_client_cert`; required when it is set | +| `tls_client_cert` | string | `""` | Path to node's client certificate (PEM) for mutual TLS; requires `use_tls=1`; empty = one-way TLS | +| `tls_client_key` | string | `""` | Path to private key (PEM) for `tls_client_cert`; requires `use_tls=1`; required when the cert is set | | `batch_size` | uint | `512` | Spans per export batch | | `batch_delay_ms` | uint | `5000` | Max delay before sending batch (ms) | | `max_queue_size` | uint | `2048` | Maximum queued spans | @@ -168,10 +168,10 @@ flowchart TB **Reading the diagram:** -- **Configuration Sources**: `xrpld.cfg` provides runtime settings (endpoint, sampling) while the CMake flag controls whether telemetry is compiled in at all. +- **Configuration Sources**: `xrpld.cfg` provides runtime settings (endpoint, per-component trace toggles) while the CMake flag controls whether telemetry is compiled in at all. Head sampling is fixed at 1.0 and is not a config option; volume reduction happens via tail sampling in the collector. - **Initialization**: `setupTelemetry()` parses config values, then `makeTelemetry()` constructs the provider, processor, and exporter objects. - **Runtime Components**: The `TracerProvider` creates spans, the `BatchProcessor` buffers them, and the `OTLP Exporter` serializes and sends them over the wire. -- **OTLP arrow to Collector**: Trace data leaves the xrpld process via OTLP (gRPC or HTTP) and enters the external Collector pipeline. +- **OTLP arrow to Collector**: Trace data leaves the xrpld process via OTLP/HTTP and enters the external Collector pipeline. (OTLP/gRPC is future work — see design decisions §2.2.2.) - **Collector Pipeline**: `Receivers` ingest OTLP data, `Processors` apply sampling/filtering/enrichment, and `Exporters` forward traces to storage backends (Tempo, etc.). --- diff --git a/OpenTelemetryPlan/06-implementation-phases.md b/OpenTelemetryPlan/06-implementation-phases.md index 33246d618a..2a766b182f 100644 --- a/OpenTelemetryPlan/06-implementation-phases.md +++ b/OpenTelemetryPlan/06-implementation-phases.md @@ -278,9 +278,11 @@ See [Phase4_taskList.md](./Phase4_taskList.md) for full task details. **Objective**: Wire `TraceContextPropagator` for P2P messages (proposals, validations) to enable true distributed tracing between nodes. -**Status**: Design documented, NOT implemented. Protobuf fields (field 1001) -and `TraceContextPropagator` free functions exist. Wiring deferred until Phase 4a is -validated in a multi-node environment. +**Status**: Partially implemented. Send-side injection (proposals and +validations) and receive-side extraction (`consensus.{proposal,validation}. +receive` spans parented on the sender's context) are wired in Phase 4a. +Remaining Phase 4b work: relay spans in `share(RCLCxPeerPos)` and multi-node +validation of the propagation path. **Prerequisites**: Phase 4a complete and validated. diff --git a/OpenTelemetryPlan/07-observability-backends.md b/OpenTelemetryPlan/07-observability-backends.md index cb17d2817a..4ebb6028fd 100644 --- a/OpenTelemetryPlan/07-observability-backends.md +++ b/OpenTelemetryPlan/07-observability-backends.md @@ -237,7 +237,7 @@ A Tempo-backed dashboard (uid `xrpld-consensus-health`) with four panels, all dr - **Proposers per Round** (stat): average of the `span.proposers` attribute on `consensus.round` spans. - **Recent Slow Rounds (>5s)** (table): `consensus.round` spans filtered to `duration > 5s`. -The underlying TraceQL queries are listed in section 7.7.3 and used throughout this doc. +Each panel's TraceQL query is described inline in its bullet above. ### 7.6.2 Node Overview Dashboard @@ -261,9 +261,9 @@ Grafana provisions three TraceQL-based alert rules (group `xrpld-tracing-alerts` - **RPC Error Rate Spike** (critical, `for: 2m`): fires when the error rate across `rpc.command.*` spans exceeds 5%. Error _rate_ is a ratio, so it must divide the error-span rate by the total-span rate — a single TraceQL `rate()` returns spans/second, not a percentage, and would fire on traffic volume alone. This uses span metrics emitted by the collector's `spanmetrics` connector (Prometheus datasource), not a TraceQL query: ``` - sum(rate(calls_total{service_name="xrpld", span_name=~"rpc.command.*", status_code="STATUS_CODE_ERROR"}[5m])) + sum(rate(traces_spanmetrics_calls_total{service_name="xrpld", span_name=~"rpc.command.*", status_code="STATUS_CODE_ERROR"}[5m])) / - sum(rate(calls_total{service_name="xrpld", span_name=~"rpc.command.*"}[5m])) + sum(rate(traces_spanmetrics_calls_total{service_name="xrpld", span_name=~"rpc.command.*"}[5m])) > 0.05 ``` diff --git a/OpenTelemetryPlan/08-appendix.md b/OpenTelemetryPlan/08-appendix.md index 0ca7b0ca09..9c1b656494 100644 --- a/OpenTelemetryPlan/08-appendix.md +++ b/OpenTelemetryPlan/08-appendix.md @@ -186,15 +186,14 @@ flowchart TB ### Task Lists -| Document | Description | -| -------------------------------------------------------------------------- | --------------------------------------------------- | -| [Phase2_taskList.md](./Phase2_taskList.md) | RPC layer trace instrumentation | -| [Phase3_taskList.md](./Phase3_taskList.md) | Peer overlay & consensus tracing | -| [Phase4_taskList.md](./Phase4_taskList.md) | Transaction lifecycle tracing | -| [Phase5_taskList.md](./Phase5_taskList.md) | Ledger processing & advanced tracing | -| [Phase5_IntegrationTest_taskList.md](./Phase5_IntegrationTest_taskList.md) | Observability stack integration tests | -| [Phase7_taskList.md](./Phase7_taskList.md) | Native OTel metrics migration | -| [presentation.md](./presentation.md) | Presentation slides for OpenTelemetry plan overview | +| Document | Description | +| -------------------------------------------------------------------------- | ------------------------------------- | +| [Phase2_taskList.md](./Phase2_taskList.md) | RPC layer trace instrumentation | +| [Phase3_taskList.md](./Phase3_taskList.md) | Peer overlay & consensus tracing | +| [Phase4_taskList.md](./Phase4_taskList.md) | Transaction lifecycle tracing | +| [Phase5_taskList.md](./Phase5_taskList.md) | Ledger processing & advanced tracing | +| [Phase5_IntegrationTest_taskList.md](./Phase5_IntegrationTest_taskList.md) | Observability stack integration tests | +| [Phase7_taskList.md](./Phase7_taskList.md) | Native OTel metrics migration | --- diff --git a/OpenTelemetryPlan/OpenTelemetryPlan.md b/OpenTelemetryPlan/OpenTelemetryPlan.md index ce9f352cc2..96d05aaeef 100644 --- a/OpenTelemetryPlan/OpenTelemetryPlan.md +++ b/OpenTelemetryPlan/OpenTelemetryPlan.md @@ -1,4 +1,4 @@ -# [OpenTelemetry](00-tracing-fundamentals.md) Distributed Tracing Implementation Plan for xrpld (xrpld) +# [OpenTelemetry](00-tracing-fundamentals.md) Distributed Tracing Implementation Plan for xrpld ## Executive Summary @@ -134,7 +134,7 @@ Key trace points span across transaction submission via RPC, peer-to-peer messag > **OTLP** = OpenTelemetry Protocol | **CNCF** = Cloud Native Computing Foundation -The OpenTelemetry C++ SDK is selected for its CNCF backing, active development, and native performance characteristics. Traces are exported via OTLP/gRPC (primary) or OTLP/HTTP (fallback) to an OpenTelemetry Collector, which provides flexible routing and sampling. +The OpenTelemetry C++ SDK is selected for its CNCF backing, active development, and native performance characteristics. Traces are exported via OTLP/HTTP to an OpenTelemetry Collector, which provides flexible routing and sampling. OTLP/gRPC is planned future work (see design decisions §2.2.2). Span naming follows a hierarchical `.` convention (e.g., `rpc.submit`, `tx.relay`, `consensus.round`). Context propagation uses W3C Trace Context headers for HTTP and embedded Protocol Buffer fields for P2P messages. The implementation coexists with existing PerfLog and Insight observability systems through correlation IDs. @@ -158,7 +158,7 @@ Performance optimization strategies include head sampling fixed at 100% (intenti > **OTLP** = OpenTelemetry Protocol | **APM** = Application Performance Monitoring -Configuration is handled through the `[telemetry]` section in `xrpld.cfg` with options for enabling/disabling, exporter selection, endpoint configuration, sampling ratios, and component-level filtering. CMake integration includes a `XRPL_ENABLE_TELEMETRY` option for compile-time control. +Configuration is handled through the `[telemetry]` section in `xrpld.cfg` with options for enabling/disabling, exporter selection, endpoint configuration, and component-level filtering. Head sampling is fixed at 1.0 (not operator-configurable); volume reduction is done by tail sampling in the collector. CMake integration includes a `XRPL_ENABLE_TELEMETRY` option for compile-time control. OpenTelemetry Collector configurations are provided for development and production (with tail-based sampling, Tempo, and Elastic APM). Docker Compose examples enable quick local development environment setup. diff --git a/OpenTelemetryPlan/Phase2_taskList.md b/OpenTelemetryPlan/Phase2_taskList.md index 1f572ecfa9..89d9ddd6e0 100644 --- a/OpenTelemetryPlan/Phase2_taskList.md +++ b/OpenTelemetryPlan/Phase2_taskList.md @@ -169,11 +169,12 @@ Node health (`amendment_blocked`, `server_state`) is not part of the telemetry s | `rpc.process` | `batch_size` | int64 | `params.size()` (only when batch) | | `rpc.ws_message` | `command` | string | `jv[command]` or `jv[method]` | | `rpc.command.*` | `load_type` | string | `context.loadType.label()` | -| `pathfind.compute` | `pathfind_dest_amount` | string | `saDstAmount_.getFullText()` | | `pathfind.compute` | `pathfind_dest_currency` | string | `to_string(saDstAmount_.asset())` | | `pathfind.discover` | `pathfind_num_source_assets` | int64 | `sourceAssets.size()` | -**New attr keys**: `RpcSpanNames.h` (`isBatch`, `batchSize`, `loadType`), `PathFindSpanNames.h` (`destAmount`, `destCurrency`, `numSourceAssets`). +_Note: `pathfind_dest_amount` was removed — the destination amount is a financial value excluded by the privacy policy (design §2.4.4)._ + +**New attr keys**: `RpcSpanNames.h` (`isBatch`, `batchSize`, `loadType`), `PathFindSpanNames.h` (`destCurrency`, `numSourceAssets`). **Modified files**: diff --git a/OpenTelemetryPlan/Phase3_taskList.md b/OpenTelemetryPlan/Phase3_taskList.md index ab7dd89022..82dbb4ffb3 100644 --- a/OpenTelemetryPlan/Phase3_taskList.md +++ b/OpenTelemetryPlan/Phase3_taskList.md @@ -166,7 +166,8 @@ ## Task 3.6: Context Propagation in Transaction Relay -**Status**: COMPLETE +**Status**: COMPLETE (transaction relay). Consensus proposal/validation +propagation is deferred to Phase 4 — see "Planned (Phase 4)" below. **Objective**: Ensure trace context flows correctly when transactions are relayed between peers, creating linked spans across nodes. @@ -178,17 +179,6 @@ wired in PeerImp) extracts the parent span_id and creates the tx.receive span as a child of the sender's tx.process span. -- **Proposal send/receive**: `RCLConsensus::Adaptor::propose()` injects the - current thread's active span context into the `TMProposeSet` protobuf via - `telemetry::injectToProtobuf()`. PeerImp creates a - `consensus.proposal.receive` span that extracts the sender's trace context - as parent (via `ConsensusReceiveTracing.h`). - -- **Validation send/receive**: `RCLConsensus::Adaptor::validate()` injects - the current thread's active span context into the `TMValidation` protobuf. - PeerImp creates a `consensus.validation.receive` span that extracts the - sender's trace context as parent. - - **Edge cases**: Missing trace context (older peers) degrades gracefully to standalone spans. Invalid/corrupted context is treated as absent. Trace flags are propagated and respected. @@ -200,20 +190,36 @@ - `PropagationHelpers.h` — `injectSpanContext(SpanGuard&, proto)` bridge between SpanGuard and protobuf TraceContext. - `TraceContextPropagator.h` — `injectToProtobuf(ctx, proto)` for - same-thread injection via OTel RuntimeContext (used in propose/validate). -- `ConsensusReceiveTracing.h` — `proposalReceiveSpan()` and - `validationReceiveSpan()` helper functions that create receive spans with - optional parent context extraction from incoming protobuf messages. + same-thread injection via OTel RuntimeContext. **Key modified files**: - `src/xrpld/app/misc/NetworkOPs.cpp` — tx relay injection -- `src/xrpld/app/consensus/RCLConsensus.cpp` — proposal/validation send injection -- `src/xrpld/overlay/detail/PeerImp.cpp` — proposal/validation receive spans - `include/xrpl/telemetry/SpanGuard.h` — `TraceBytes` struct, `getTraceBytes()` - `src/libxrpl/telemetry/SpanGuard.cpp` — `getTraceBytes()` implementation - `src/xrpld/telemetry/PropagationHelpers.h` — inject helpers (new file) -- `src/xrpld/telemetry/ConsensusReceiveTracing.h` — receive span helpers (new file) + +**Planned (Phase 4 — not in this PR)**: + +The consensus proposal/validation propagation below is Phase 4 scope and is +not implemented on this branch. It is listed here only to record the intended +design. + +- **Proposal send/receive**: `RCLConsensus::Adaptor::propose()` injects the + current thread's active span context into the `TMProposeSet` protobuf via + `telemetry::injectToProtobuf()`. PeerImp creates a + `consensus.proposal.receive` span that extracts the sender's trace context + as parent (via `ConsensusReceiveTracing.h`). + +- **Validation send/receive**: `RCLConsensus::Adaptor::validate()` injects + the current thread's active span context into the `TMValidation` protobuf. + PeerImp creates a `consensus.validation.receive` span that extracts the + sender's trace context as parent. + +- Planned files: `src/xrpld/app/consensus/RCLConsensus.cpp` (send injection), + `src/xrpld/overlay/detail/PeerImp.cpp` (receive spans), + `src/xrpld/telemetry/ConsensusReceiveTracing.h` (receive span helpers, + new file). **Reference**: @@ -396,7 +402,7 @@ This gives the best of both worlds: guaranteed cross-node correlation via determ - `src/xrpld/overlay/detail/PeerImp.cpp` — restructured span creation - `src/xrpld/app/misc/NetworkOPs.cpp` — deterministic context for tx.process -- `src/xrpld/app/misc/TxSpanNames.h` — new `traceStrategy` attribute constant +- `src/xrpld/telemetry/TxSpanNames.h` — new `traceStrategy` attribute constant - New or shared utility for `createDeterministicTxContext()` (location TBD: could be a shared header like `include/xrpl/telemetry/DeterministicContext.h`, or file-local if only used in two places) @@ -431,7 +437,7 @@ This gives the best of both worlds: guaranteed cross-node correlation via determ - `txSpan(category, group, name, txHash, parentCtx)` — child span (deterministic trace_id combined with protobuf-extracted parent span_id for relay ordering). -- **`TxTracing.h` helper functions** (`src/xrpld/overlay/detail/TxTracing.h`): +- **`TxTracing.h` helper functions** (`src/xrpld/telemetry/TxTracing.h`): File-local helpers that wrap `SpanGuard::txSpan()` for the two main PeerImp call sites: - `txReceiveSpan(txHash, parentCtx)` — creates `tx.receive` span with @@ -495,7 +501,7 @@ This gives the best of both worlds: guaranteed cross-node correlation via determ **Modified files**: -- `src/xrpld/app/misc/TxSpanNames.h` +- `src/xrpld/telemetry/TxSpanNames.h` - `src/xrpld/app/misc/detail/TxQSpanNames.h` - `src/xrpld/app/misc/NetworkOPs.cpp` - `src/xrpld/overlay/detail/PeerImp.cpp` diff --git a/OpenTelemetryPlan/Phase4_taskList.md b/OpenTelemetryPlan/Phase4_taskList.md index 9973331651..028a7a7e85 100644 --- a/OpenTelemetryPlan/Phase4_taskList.md +++ b/OpenTelemetryPlan/Phase4_taskList.md @@ -95,10 +95,16 @@ - Creates `consensus.proposal.receive` span - Sets `trusted` attribute (bool) -**Not implemented** (deferred to Phase 4b — cross-node propagation): +**Done here** (cross-node propagation, send + receive): -- `consensus.proposal.relay` span in `share(RCLCxPeerPos)` — requires trace context injection -- Trace context injection/extraction for `TMProposeSet::trace_context` +- Trace context injection for `TMProposeSet::trace_context` in `propose()` +- Receive-side extraction in `PeerImp::onMessage(TMProposeSet)` via + `telemetry::proposalReceiveSpan()` (parents the receive span on the + sender's context when a valid `trace_context` is present) + +**Not implemented** (deferred to Phase 4b): + +- `consensus.proposal.relay` span in `share(RCLCxPeerPos)` **Key modified files**: diff --git a/OpenTelemetryPlan/Phase5_IntegrationTest_taskList.md b/OpenTelemetryPlan/Phase5_IntegrationTest_taskList.md index 6f9df2f969..2bf33a390c 100644 --- a/OpenTelemetryPlan/Phase5_IntegrationTest_taskList.md +++ b/OpenTelemetryPlan/Phase5_IntegrationTest_taskList.md @@ -107,13 +107,13 @@ Tempo/Prometheus. - `consensus.validation.send` (Adaptor::validate) - Verify attributes: - `xrpl.consensus.mode` on `consensus.ledger_close` - - `xrpl.consensus.proposers` on `consensus.accept` + - `proposers` on `consensus.accept` - `xrpl.consensus.ledger.seq` on `consensus.validation.send` **Verification**: - [ ] Tempo shows `consensus.ledger_close` traces with `xrpl.consensus.mode` -- [ ] Tempo shows `consensus.accept` traces with `xrpl.consensus.proposers` +- [ ] Tempo shows `consensus.accept` traces with `proposers` - [ ] Tempo shows `consensus.proposal.send` traces - [ ] Tempo shows `consensus.validation.send` traces @@ -147,7 +147,7 @@ Tempo/Prometheus. - Prerequisites section - Single-node standalone test (quick verification) - 6-node consensus test (full verification) - - Expected span catalog (all 12 span names with attributes) + - Expected span catalog (all 11 span names with attributes) - Verification queries (Tempo API, Prometheus API) - Troubleshooting guide @@ -156,7 +156,7 @@ Tempo/Prometheus. **Verification**: - [ ] Document covers both single-node and multi-node testing -- [ ] All 12 span names documented with source file and attributes +- [ ] All 11 span names documented with source file and attributes - [ ] Troubleshooting section covers common failure modes --- diff --git a/OpenTelemetryPlan/presentation.md b/OpenTelemetryPlan/presentation.md deleted file mode 100644 index bc83f82fc2..0000000000 --- a/OpenTelemetryPlan/presentation.md +++ /dev/null @@ -1,682 +0,0 @@ -# OpenTelemetry Distributed Tracing for xrpld - ---- - -## Slide 1: Introduction - -> **CNCF** = Cloud Native Computing Foundation - -### What is OpenTelemetry? - -OpenTelemetry is an open-source, CNCF-backed observability framework for distributed tracing, metrics, and logs. - -### Why OpenTelemetry for xrpld? - -- **End-to-End Transaction Visibility**: Track transactions from submission → consensus → ledger inclusion -- **Cross-Node Correlation**: Follow requests across multiple independent nodes using a unique `trace_id` -- **Consensus Round Analysis**: Understand timing and behavior across validators -- **Incident Debugging**: Correlate events across distributed nodes during issues - -```mermaid -flowchart LR - A["Node A
tx.receive
trace_id: abc123"] --> B["Node B
tx.relay
trace_id: abc123"] --> C["Node C
tx.validate
trace_id: abc123"] --> D["Node D
ledger.apply
trace_id: abc123"] - - style A fill:#1565c0,stroke:#0d47a1,color:#fff - style B fill:#2e7d32,stroke:#1b5e20,color:#fff - style C fill:#2e7d32,stroke:#1b5e20,color:#fff - style D fill:#e65100,stroke:#bf360c,color:#fff -``` - -**Reading the diagram:** - -- **Node A (blue, leftmost)**: The originating node that first receives the transaction and assigns a new `trace_id: abc123`; this ID becomes the correlation key for the entire distributed trace. -- **Node B and Node C (green, middle)**: Relay and validation nodes — each creates its own span but carries the same `trace_id`, so their work is linked to the original submission without any central coordinator. -- **Node D (orange, rightmost)**: The final node that applies the transaction to the ledger; the trace now spans the full lifecycle from submission to ledger inclusion. -- **Left-to-right flow**: The horizontal progression shows the real-world message path — a transaction hops from node to node, and the shared `trace_id` stitches all hops into a single queryable trace. - -> **Trace ID: abc123** — All nodes share the same trace, enabling cross-node correlation. - ---- - -## Slide 2: OpenTelemetry vs Open Source Alternatives - -> **CNCF** = Cloud Native Computing Foundation - -| Feature | OpenTelemetry | Jaeger | Zipkin | SkyWalking | Pinpoint | Prometheus | -| ------------------- | ---------------- | ---------------- | ------------------ | ---------- | ---------- | ---------- | -| **Tracing** | YES | YES | YES | YES | YES | NO | -| **Metrics** | YES | NO | NO | YES | YES | YES | -| **Logs** | YES | NO | NO | YES | NO | NO | -| **C++ SDK** | YES Official | YES (Deprecated) | YES (Unmaintained) | NO | NO | YES | -| **Vendor Neutral** | YES Primary goal | NO | NO | NO | NO | NO | -| **Instrumentation** | Manual + Auto | Manual | Manual | Auto-first | Auto-first | Manual | -| **Backend** | Any (exporters) | Self | Self | Self | Self | Self | -| **CNCF Status** | Incubating | Graduated | NO | Incubating | NO | Graduated | - -> **Why OpenTelemetry?** It's the only actively maintained, full-featured C++ option with vendor neutrality — allowing export to Tempo, Prometheus, Grafana, or any commercial backend without changing instrumentation. - ---- - -## Slide 3: Adoption Scope — Traces Only (Current Plan) - -OpenTelemetry supports three signal types: **Traces**, **Metrics**, and **Logs**. xrpld already captures metrics (StatsD via Beast Insight) and logs (Journal/PerfLog). The question is: how much of OTel do we adopt? - -> **Scenario A**: Add distributed tracing. Keep StatsD for metrics and Journal for logs. - -```mermaid -flowchart LR - subgraph xrpld["xrpld Process"] - direction TB - OTel["OTel SDK
(Traces)"] - Insight["Beast Insight
(StatsD Metrics)"] - Journal["Journal + PerfLog
(Logging)"] - end - - OTel -->|"OTLP"| Collector["OTel Collector"] - Insight -->|"UDP"| StatsD["StatsD Server"] - Journal -->|"File I/O"| LogFile["perf.log / debug.log"] - - Collector --> Tempo["Tempo"] - StatsD --> Graphite["Graphite / Grafana"] - LogFile --> Loki["Loki (optional)"] - - style xrpld fill:#424242,stroke:#212121,color:#fff - style OTel fill:#2e7d32,stroke:#1b5e20,color:#fff - style Insight fill:#1565c0,stroke:#0d47a1,color:#fff - style Journal fill:#e65100,stroke:#bf360c,color:#fff - style Collector fill:#2e7d32,stroke:#1b5e20,color:#fff -``` - -| Aspect | Details | -| ------------------------------ | --------------------------------------------------------------------------------------------------------------- | -| **What changes for operators** | Deploy OTel Collector + trace backend. Existing StatsD and log pipelines stay as-is. | -| **Codebase impact** | New `Telemetry` module (~1500 LOC). Beast Insight and Journal untouched. | -| **New capabilities** | Cross-node trace correlation, span-based debugging, request lifecycle visibility. | -| **What we still can't do** | Correlate metrics with specific traces natively. StatsD metrics remain fire-and-forget with no trace exemplars. | -| **Maintenance burden** | Three separate observability systems to maintain (OTel + StatsD + Journal). | -| **Risk** | Lowest — additive change, no existing systems disturbed. | - ---- - -## Slide 4: Future Adoption — Metrics & Logs via OTel - -### Scenario B: + OTel Metrics (Replace StatsD) - -> Migrate StatsD to OTel Metrics API, exposing Prometheus-compatible metrics. Remove Beast Insight. - -```mermaid -flowchart LR - subgraph xrpld["xrpld Process"] - direction TB - OTel["OTel SDK
(Traces + Metrics)"] - Journal["Journal + PerfLog
(Logging)"] - end - - OTel -->|"OTLP"| Collector["OTel Collector"] - Journal -->|"File I/O"| LogFile["perf.log / debug.log"] - - Collector --> Tempo["Tempo
(Traces)"] - Collector --> Prom["Prometheus
(Metrics)"] - LogFile --> Loki["Loki (optional)"] - - style xrpld fill:#424242,stroke:#212121,color:#fff - style OTel fill:#2e7d32,stroke:#1b5e20,color:#fff - style Journal fill:#e65100,stroke:#bf360c,color:#fff - style Collector fill:#2e7d32,stroke:#1b5e20,color:#fff -``` - -- **Better metrics?** Yes — Prometheus gives native histograms (p50/p95/p99), multi-dimensional labels, and exemplars linking metric spikes to traces. -- **Codebase**: Remove `Beast::Insight` + `StatsDCollector` (~2000 LOC). Single SDK for traces and metrics. -- **Operator effort**: Rewrite dashboards from StatsD/Graphite queries to PromQL. Run both in parallel during transition. -- **Risk**: Medium — operators must migrate monitoring infrastructure. - -### Scenario C: + OTel Logs (Full Stack) - -> Also replace Journal logging with OTel Logs API. Single SDK for everything. - -```mermaid -flowchart LR - subgraph xrpld["xrpld Process"] - OTel["OTel SDK
(Traces + Metrics + Logs)"] - end - - OTel -->|"OTLP"| Collector["OTel Collector"] - - Collector --> Tempo["Tempo
(Traces)"] - Collector --> Prom["Prometheus
(Metrics)"] - Collector --> Loki["Loki / Elastic
(Logs)"] - - style xrpld fill:#424242,stroke:#212121,color:#fff - style OTel fill:#2e7d32,stroke:#1b5e20,color:#fff - style Collector fill:#2e7d32,stroke:#1b5e20,color:#fff -``` - -- **Structured logging**: OTel Logs API outputs structured records with `trace_id`, `span_id`, severity, and attributes by design. -- **Full correlation**: Every log line carries `trace_id`. Click trace → see logs. Click metric spike → see trace → see logs. -- **Codebase**: Remove Beast Insight (~2000 LOC) + simplify Journal/PerfLog (~3000 LOC). One dependency instead of three. -- **Risk**: Highest — `beast::Journal` is deeply embedded in every component. Large refactor. OTel C++ Logs API is newer (stable since v1.11, less battle-tested). - -### Recommendation - -```mermaid -flowchart LR - A["Phase 1
Traces Only
(Current Plan)"] --> B["Phase 2
+ Metrics
(Replace StatsD)"] --> C["Phase 3
+ Logs
(Full OTel)"] - - style A fill:#2e7d32,stroke:#1b5e20,color:#fff - style B fill:#1565c0,stroke:#0d47a1,color:#fff - style C fill:#e65100,stroke:#bf360c,color:#fff -``` - -| Phase | Signal | Strategy | Risk | -| -------------------- | --------- | -------------------------------------------------------------- | ------ | -| **Phase 1** (now) | Traces | Add OTel traces. Keep StatsD and Journal. Prove value. | Low | -| **Phase 2** (future) | + Metrics | Migrate StatsD → Prometheus via OTel. Remove Beast Insight. | Medium | -| **Phase 3** (future) | + Logs | Adopt OTel Logs API. Align with structured logging initiative. | High | - -> **Key Takeaway**: Start with traces (unique value, lowest risk), then incrementally adopt metrics and logs as the OTel infrastructure proves itself. - ---- - -## Slide 5: Comparison with xrpld's Existing Solutions - -### Current Observability Stack - -| Aspect | PerfLog (JSON) | StatsD (Metrics) | OpenTelemetry (NEW) | -| --------------------- | --------------------- | --------------------- | --------------------------- | -| **Type** | Logging | Metrics | Distributed Tracing | -| **Scope** | Single node | Single node | **Cross-node** | -| **Data** | JSON log entries | Counters, gauges | Spans with context | -| **Correlation** | By timestamp | By metric name | By `trace_id` | -| **Overhead** | Low (file I/O) | Low (UDP) | Low-Medium (configurable) | -| **Question Answered** | "What happened here?" | "How many? How fast?" | **"What was the journey?"** | - -### Use Case Matrix - -| Scenario | PerfLog | StatsD | OpenTelemetry | -| -------------------------------- | ------- | ------ | ------------- | -| "How many TXs per second?" | ❌ | ✅ | ❌ | -| "Why was this specific TX slow?" | ⚠️ | ❌ | ✅ | -| "Which node delayed consensus?" | ❌ | ❌ | ✅ | -| "Show TX journey across 5 nodes" | ❌ | ❌ | ✅ | - -> **Key Insight**: In the **traces-only** approach (Phase 1), OpenTelemetry **complements** existing systems. In future phases, OTel metrics and logs could **replace** StatsD and Journal respectively — see Slides 3-4 for the full adoption roadmap. - ---- - -## Slide 6: Architecture - -> **OTLP** = OpenTelemetry Protocol | **WS** = WebSocket - -### High-Level Integration Architecture - -```mermaid -flowchart TB - subgraph xrpld["xrpld Node"] - subgraph services["Core Services"] - direction LR - RPC["RPC Server
(HTTP/WS)"] ~~~ Overlay["Overlay
(P2P Network)"] ~~~ Consensus["Consensus
(RCLConsensus)"] - end - - Telemetry["Telemetry Module
(OpenTelemetry SDK)"] - - services --> Telemetry - end - - Telemetry -->|OTLP/HTTP| Collector["OTel Collector"] - - Collector --> Tempo["Grafana Tempo"] - Collector --> Elastic["Elastic APM"] - - style xrpld fill:#424242,stroke:#212121,color:#fff - style services fill:#1565c0,stroke:#0d47a1,color:#fff - style Telemetry fill:#2e7d32,stroke:#1b5e20,color:#fff - style Collector fill:#e65100,stroke:#bf360c,color:#fff -``` - -**Reading the diagram:** - -- **Core Services (blue, top)**: RPC Server, Overlay, and Consensus are the three primary components that generate trace data — they represent the entry points for client requests, peer messages, and consensus rounds respectively. -- **Telemetry Module (green, middle)**: The OpenTelemetry SDK sits below the core services and receives span data from all three; it acts as a single collection point within the xrpld process. -- **OTel Collector (orange, center)**: An external process that receives spans over OTLP/HTTP from the Telemetry Module; it decouples xrpld from backend choices and handles batching, sampling, and routing. -- **Backends (bottom row)**: Tempo and Elastic APM are interchangeable — the Collector fans out to any combination, so operators can switch backends without modifying xrpld code. -- **Top-to-bottom flow**: Data flows from instrumented code down through the SDK, out over the network to the Collector, and finally into storage/visualization backends. - -### Context Propagation - -```mermaid -sequenceDiagram - participant Client - participant NodeA as Node A - participant NodeB as Node B - - Client->>NodeA: Submit TX (no context) - Note over NodeA: Creates trace_id: abc123
span: tx.receive - NodeA->>NodeB: Relay TX
(traceparent: abc123) - Note over NodeB: Links to trace_id: abc123
span: tx.relay -``` - -- **HTTP/RPC**: W3C Trace Context headers (`traceparent`) -- **P2P Messages**: Protocol Buffer extension fields - ---- - -## Slide 7: Implementation Plan - -### 5-Phase Rollout (9 Weeks) - -> **Note**: Dates shown are relative to project start, not calendar dates. - -```mermaid -gantt - title Implementation Timeline - dateFormat YYYY-MM-DD - axisFormat Week %W - - section Phase 1 - Core Infrastructure :p1, 2024-01-01, 2w - - section Phase 2 - RPC Tracing :p2, after p1, 2w - - section Phase 3 - Transaction Tracing :p3, after p2, 2w - - section Phase 4 - Consensus Tracing :p4, after p3, 2w - - section Phase 5 - Documentation :p5, after p4, 1w -``` - -### Phase Details - -| Phase | Focus | Key Deliverables | Effort | -| ----- | ------------------- | -------------------------------------------- | ------- | -| 1 | Core Infrastructure | SDK integration, Telemetry interface, Config | 10 days | -| 2 | RPC Tracing | HTTP context extraction, Handler spans | 10 days | -| 3 | Transaction Tracing | Protobuf context, P2P relay propagation | 10 days | -| 4 | Consensus Tracing | Round spans, Proposal/validation tracing | 10 days | -| 5 | Documentation | Runbook, Dashboards, Training | 7 days | - -**Total Effort**: ~47 developer-days (2 developers) - -> **Future Phases** (not in current scope): After traces are stable, OTel metrics can replace StatsD (~3 weeks), and OTel logs can replace Journal (~4 weeks, aligned with structured logging initiative). See Slides 3-4 for the full adoption roadmap. - ---- - -## Slide 8: Performance Overhead - -> **OTLP** = OpenTelemetry Protocol - -### Estimated System Impact - -| Metric | Overhead | Notes | -| ----------------- | ---------- | ------------------------------------------------ | -| **CPU** | 1-3% | Span creation and attribute setting | -| **Memory** | ~10 MB | SDK statics + batch buffer + worker thread stack | -| **Network** | 10-50 KB/s | Compressed OTLP export to collector | -| **Latency (p99)** | <2% | With proper sampling configuration | - -#### How We Arrived at These Numbers - -**Assumptions (XRPL mainnet baseline)**: - -| Parameter | Value | Source | -| ------------------------- | ---------------------- | --------------------------------------------------------------------------------------------------- | -| Transaction throughput | ~25 TPS (peaks to ~50) | Mainnet average | -| Default peers per node | 21 | `peerfinder/detail/Tuning.h` (`defaultMaxPeers`) | -| Consensus round frequency | ~1 round / 3-4 seconds | `ConsensusParms.h` (`ledgerMIN_CONSENSUS=1950ms`) | -| Proposers per round | ~20-35 | Mainnet UNL size | -| P2P message rate | ~160 msgs/sec | See message breakdown below | -| Avg TX processing time | ~200 μs | Profiled baseline | -| Single span creation cost | 500-1000 ns | OTel C++ SDK benchmarks (see [3.5.4](./03-implementation-strategy.md#354-performance-data-sources)) | - -**P2P message breakdown** (per node, mainnet): - -| Message Type | Rate | Derivation | -| ------------- | ------------ | --------------------------------------------------------------------- | -| TMTransaction | ~100/sec | ~25 TPS × ~4 relay hops per TX, deduplicated by HashRouter | -| TMValidation | ~50/sec | ~35 validators × ~1 validation/3s round ≈ ~12/sec, plus relay fan-out | -| TMProposeSet | ~10/sec | ~35 proposers / 3s round ≈ ~12/round, clustered in establish phase | -| **Total** | **~160/sec** | **Only traced message types counted** | - -**CPU (1-3%) — Calculation**: - -Per-transaction tracing cost breakdown: - -| Operation | Cost | Notes | -| ----------------------------------------------- | ----------- | ------------------------------------------ | -| `tx.receive` span (create + end + 4 attributes) | ~1400 ns | ~1000ns create + ~200ns end + 4×50ns attrs | -| `tx.validate` span | ~1200 ns | ~1000ns create + ~200ns for 2 attributes | -| `tx.relay` span | ~1200 ns | ~1000ns create + ~200ns for 2 attributes | -| Context injection into P2P message | ~200 ns | Serialize trace_id + span_id into protobuf | -| **Total per TX** | **~4.0 μs** | | - -> **CPU overhead**: 4.0 μs / 200 μs baseline = **~2.0% per transaction**. Under high load with consensus + RPC spans overlapping, reaches ~3%. Consensus itself adds only ~36 μs per 3-second round (~0.001%), so the TX path dominates. On production server hardware (3+ GHz Xeon), span creation drops to ~500-600 ns, bringing per-TX cost to ~2.6 μs (~1.3%). See [Section 3.5.4](./03-implementation-strategy.md#354-performance-data-sources) for benchmark sources. - -**Memory (~10 MB) — Calculation**: - -| Component | Size | Notes | -| --------------------------------------------- | ------------------ | ------------------------------------- | -| TracerProvider + Exporter (gRPC channel init) | ~320 KB | Allocated once at startup | -| BatchSpanProcessor (circular buffer) | ~16 KB | 2049 × 8-byte AtomicUniquePtr entries | -| BatchSpanProcessor (worker thread stack) | ~8 MB | Default Linux thread stack size | -| Active spans (in-flight, max ~1000) | ~500-800 KB | ~500-800 bytes/span × 1000 concurrent | -| Export queue (batch buffer, max 2048 spans) | ~1 MB | ~500 bytes/span × 2048 queue depth | -| Thread-local context storage (~100 threads) | ~6.4 KB | ~64 bytes/thread | -| **Total** | **~10 MB ceiling** | | - -> Memory plateaus once the export queue fills — the `max_queue_size=2048` config bounds growth. -> The worker thread stack (~8 MB) dominates the static footprint but is virtual memory; actual RSS -> depends on stack usage (typically much less). Active spans are larger than originally estimated -> (~500-800 bytes) because the OTel SDK `Span` object includes a mutex (~40 bytes), `SpanData` -> recordable (~250 bytes base), and `std::map`-based attribute storage (~200-500 bytes for 3-5 -> string attributes). See [Section 3.5.4](./03-implementation-strategy.md#354-performance-data-sources) for source references. - -> **Measured (perf-iac, telemetry on vs off, 9 nodes under payment load)**: the ~10 MB -> above is a theoretical SDK-footprint ceiling, dominated by virtual (not resident) thread-stack -> memory. In practice, per-node RSS showed **no measurable increase over the telemetry-off -> baseline** (~15 GiB mean / ~18–19 GiB peak on both sides), with no OOM, swap, or leak over the -> run. Treat memory overhead as negligible; the ceiling is a provisioning safety margin, not an -> expected increase. - -**Network (10-50 KB/s) — Calculation**: - -Two sources of network overhead: - -**(A) OTLP span export to Collector:** - -| Sampling Rate | Effective Spans/sec | Avg Span Size (compressed) | Bandwidth | -| -------------------------- | ------------------- | -------------------------- | ------------ | -| 100% (dev only) | ~500 | ~500 bytes | ~250 KB/s | -| **10% (recommended prod)** | **~50** | **~500 bytes** | **~25 KB/s** | -| 1% (minimal) | ~5 | ~500 bytes | ~2.5 KB/s | - -> The ~500 spans/sec at 100% comes from: ~100 TX spans + ~160 P2P context spans + ~23 consensus spans/round + ~50 RPC spans = ~500/sec. OTLP protobuf with gzip compression yields ~500 bytes/span average. - -**(B) P2P trace context overhead** (added to existing messages, always-on regardless of sampling): - -| Message Type | Rate | Context Size | Bandwidth | -| ------------- | -------- | ------------ | ------------- | -| TMTransaction | ~100/sec | 29 bytes | ~2.9 KB/s | -| TMValidation | ~50/sec | 29 bytes | ~1.5 KB/s | -| TMProposeSet | ~10/sec | 29 bytes | ~0.3 KB/s | -| **Total P2P** | | | **~4.7 KB/s** | - -> **Combined**: 25 KB/s (OTLP export at 10%) + 5 KB/s (P2P context) ≈ **~30 KB/s typical**. The 10-50 KB/s range covers 10-20% sampling under normal to peak mainnet load. - -**Latency (<2%) — Calculation**: - -| Path | Tracing Cost | Baseline | Overhead | -| ------------------------------ | ------------ | -------- | -------- | -| Fast RPC (e.g., `server_info`) | 2.75 μs | ~1 ms | 0.275% | -| Slow RPC (e.g., `path_find`) | 2.75 μs | ~100 ms | 0.003% | -| Transaction processing | 4.0 μs | ~200 μs | 2.0% | -| Consensus round | 36 μs | ~3 sec | 0.001% | - -> At p99, even the worst case (TX processing at 2.0%) is within the 1-3% range. RPC and consensus overhead are negligible. On production hardware, TX overhead drops to ~1.3%. - -### Per-Message Overhead (Context Propagation) - -Each P2P message carries trace context with the following overhead: - -| Field | Size | Description | -| ------------- | ------------- | ----------------------------------------- | -| `trace_id` | 16 bytes | Unique identifier for the entire trace | -| `span_id` | 8 bytes | Current span (becomes parent on receiver) | -| `trace_flags` | 1 byte | Sampling decision flags | -| `trace_state` | 0-4 bytes | Optional vendor-specific data | -| **Total** | **~29 bytes** | **Added per traced P2P message** | - -```mermaid -flowchart LR - subgraph msg["P2P Message with Trace Context"] - A["Original Message
(variable size)"] --> B["+ TraceContext
(~29 bytes)"] - end - - subgraph breakdown["Context Breakdown"] - C["trace_id
16 bytes"] - D["span_id
8 bytes"] - E["flags
1 byte"] - F["state
0-4 bytes"] - end - - B --> breakdown - - style A fill:#424242,stroke:#212121,color:#fff - style B fill:#2e7d32,stroke:#1b5e20,color:#fff - style C fill:#1565c0,stroke:#0d47a1,color:#fff - style D fill:#1565c0,stroke:#0d47a1,color:#fff - style E fill:#e65100,stroke:#bf360c,color:#fff - style F fill:#4a148c,stroke:#2e0d57,color:#fff -``` - -**Reading the diagram:** - -- **Original Message (gray, left)**: The existing P2P message payload of variable size — this is unchanged; trace context is appended, never modifying the original data. -- **+ TraceContext (green, right of message)**: The additional 29-byte context block attached to each traced message; the arrow from the original message shows it is a pure addition. -- **Context Breakdown (right subgraph)**: The four fields — `trace_id` (16 bytes), `span_id` (8 bytes), `flags` (1 byte), and `state` (0-4 bytes) — show exactly what is added and their individual sizes. -- **Color coding**: Blue fields (`trace_id`, `span_id`) are the core identifiers required for trace correlation; orange (`flags`) controls sampling decisions; purple (`state`) is optional vendor data typically omitted. - -> **Note**: 29 bytes represents ~1-6% overhead depending on message size (500B simple TX to 5KB proposal), which is acceptable for the observability benefits provided. - -### Mitigation Strategies - -```mermaid -flowchart LR - A["Head Sampling
fixed 1.0 (record all)"] --> B["Tail Sampling
Keep errors/slow"] --> C["Batch Export
Reduce I/O"] --> D["Conditional Compile
XRPL_ENABLE_TELEMETRY"] - - style A fill:#1565c0,stroke:#0d47a1,color:#fff - style B fill:#2e7d32,stroke:#1b5e20,color:#fff - style C fill:#e65100,stroke:#bf360c,color:#fff - style D fill:#4a148c,stroke:#2e0d57,color:#fff -``` - -> For a detailed explanation of head vs. tail sampling, see Slide 9. - -### Kill Switches (Rollback Options) - -1. **Config Disable**: Set `enabled=0` in config → instant disable, no restart needed for sampling -2. **Rebuild**: Compile with `XRPL_ENABLE_TELEMETRY=OFF` → zero overhead (no-op) -3. **Full Revert**: Clean separation allows easy commit reversion - ---- - -## Slide 9: Sampling Strategies — Head vs. Tail - -> Sampling controls **which traces are recorded and exported**. Without sampling, every operation generates a trace — at 500+ spans/sec, this overwhelms storage and network. Sampling lets you keep the signal, discard the noise. - -### Head Sampling (Decision at Start) - -The sampling decision is made **when a trace begins**, before any work is done. A random number is generated; if it falls within the configured ratio, the entire trace is recorded. Otherwise, the trace is silently dropped. - -```mermaid -flowchart LR - A["New Request
Arrives"] --> B{"Random < 10%?"} - B -->|"Yes (1 in 10)"| C["Record Entire Trace
(all spans)"] - B -->|"No (9 in 10)"| D["Drop Entire Trace
(zero overhead)"] - - style C fill:#2e7d32,stroke:#1b5e20,color:#fff - style D fill:#c62828,stroke:#8c2809,color:#fff - style B fill:#1565c0,stroke:#0d47a1,color:#fff -``` - -| Aspect | Details | -| ----------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **Where it runs** | Inside xrpld (SDK-level). In xrpld the ratio is fixed at 1.0 and not read from config (tail sampling in the collector needs every span) — the example below shows the general head-sampling mechanism. | -| **When the decision happens** | At trace creation time — before the first span is even populated. | -| **How it works** | `sampling_ratio=0.1` means each trace has a 10% probability of being recorded. Dropped traces incur near-zero overhead (no spans created, no attributes set, no export). | -| **Propagation** | Once a trace is sampled, the `trace_flags` field (1 byte in the context header) tells downstream nodes to also sample it. Unsampled traces propagate `trace_flags=0`, so downstream nodes skip them too. | -| **Pros** | Lowest overhead. Simple to configure. Predictable resource usage. | -| **Cons** | **Blind** — it doesn't know if the trace will be interesting. A rare error or slow consensus round has only a 10% chance of being captured. | -| **Best for** | High-volume, steady-state traffic where most traces look similar (e.g., routine RPC requests). | - -**xrpld configuration**: - -```ini -[telemetry] -# xrpld fixes head sampling at 1.0 (record every trace). This value is -# NOT read from config — the collector performs tail sampling instead, -# which needs all spans to arrive. See Slide 9 (tail sampling) and §7.4.2. -sampling_ratio=1.0 -``` - -### Tail Sampling (Decision at End) - -The sampling decision is made **after the trace completes**, based on its actual content — was it slow? Did it error? Was it a consensus round? This requires buffering complete traces before deciding. - -```mermaid -flowchart TB - A["All Traces
Buffered (100%)"] --> B["OTel Collector
Evaluates Rules"] - - B --> C{"Error?"} - C -->|Yes| K["KEEP"] - - C -->|No| D{"Slow?
(>5s consensus,
>1s RPC)"} - D -->|Yes| K - - D -->|No| E{"Random < 10%?"} - E -->|Yes| K - E -->|No| F["DROP"] - - style K fill:#2e7d32,stroke:#1b5e20,color:#fff - style F fill:#c62828,stroke:#8c2809,color:#fff - style B fill:#1565c0,stroke:#0d47a1,color:#fff - style C fill:#e65100,stroke:#bf360c,color:#fff - style D fill:#e65100,stroke:#bf360c,color:#fff - style E fill:#4a148c,stroke:#2e0d57,color:#fff -``` - -| Aspect | Details | -| ----------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **Where it runs** | In the **OTel Collector** (external process), not inside xrpld. xrpld exports 100% of traces; the Collector decides what to keep. | -| **When the decision happens** | After the Collector has received all spans for a trace (waits `decision_wait=10s` for stragglers). | -| **How it works** | Policy rules evaluate the completed trace: keep all errors, keep slow operations above a threshold, keep all consensus rounds, then probabilistically sample the rest at 10%. | -| **Pros** | **Never misses important traces**. Errors, slow requests, and consensus anomalies are always captured regardless of probability. | -| **Cons** | Higher resource usage — xrpld must export 100% of spans to the Collector, which buffers them in memory before deciding. The Collector needs more RAM (configured via `num_traces` and `decision_wait`). | -| **Best for** | Production troubleshooting where you can't afford to miss errors or anomalies. | - -**Collector configuration** (tail sampling rules for xrpld): - -```yaml -processors: - tail_sampling: - decision_wait: 10s # Wait for all spans in a trace - num_traces: 100000 # Buffer up to 100K concurrent traces - policies: - - name: errors # Always keep error traces - type: status_code - status_code: { status_codes: [ERROR] } - - - name: slow-consensus # Keep consensus rounds >5s - type: latency - latency: { threshold_ms: 5000 } - - - name: slow-rpc # Keep slow RPC requests >1s - type: latency - latency: { threshold_ms: 1000 } - - - name: probabilistic # Sample 10% of everything else - type: probabilistic - probabilistic: { sampling_percentage: 10 } -``` - -### Head vs. Tail — Side-by-Side - -| | Head Sampling | Tail Sampling | -| ----------------------------- | ----------------------------------------- | ------------------------------------------------ | -| **Decision point** | Trace start (inside xrpld) | Trace end (in OTel Collector) | -| **Knows trace content?** | No (random coin flip) | Yes (evaluates completed trace) | -| **Overhead on xrpld** | Lowest (dropped traces = no-op) | Higher (must export 100% to Collector) | -| **Collector resource usage** | Low (receives only sampled traces) | Higher (buffers all traces before deciding) | -| **Captures all errors?** | No (only if trace was randomly selected) | **Yes** (error policy catches them) | -| **Captures slow operations?** | No (random) | **Yes** (latency policy catches them) | -| **Configuration** | Fixed at `1.0` in xrpld (not config-read) | `otel-collector.yaml`: `tail_sampling` processor | -| **Best for** | High-throughput steady-state | Troubleshooting & anomaly detection | - -### Recommended Strategy for xrpld - -Use **both** in a layered approach: - -```mermaid -flowchart LR - subgraph xrpld["xrpld (Head Sampling)"] - HS["sampling_ratio=1.0
(export everything)"] - end - - subgraph collector["OTel Collector (Tail Sampling)"] - TS["Keep: errors + slow + 10% random
Drop: routine traces"] - end - - subgraph storage["Backend Storage"] - ST["Only interesting traces
stored long-term"] - end - - xrpld -->|"100% of spans"| collector -->|"~15-20% kept"| storage - - style xrpld fill:#424242,stroke:#212121,color:#fff - style collector fill:#1565c0,stroke:#0d47a1,color:#fff - style storage fill:#2e7d32,stroke:#1b5e20,color:#fff -``` - -> **Why this works**: xrpld exports everything (no blind drops), the Collector applies intelligent filtering (keep errors/slow/anomalies, sample the rest), and only ~15-20% of traces reach storage. xrpld's head sampling is fixed at 1.0 and not configurable, because tail sampling can only see traces that reach the Collector — any head drop would blind the error/slow policies. To reduce volume, tune the Collector's tail-sampling rules rather than adding head sampling. - ---- - -## Slide 10: Data Collection & Privacy - -### What Data is Collected - -| Category | Attributes Collected | Purpose | -| --------------- | -------------------------------------------------------------------------------------------------------------------- | --------------------------- | -| **Transaction** | `tx_hash`, `tx_type`, `tx_result`, `tx_fee`, `ledger_index` | Trace transaction lifecycle | -| **Consensus** | `consensus_round`, `consensus_phase`, `consensus_mode`, `proposers` (count of proposing validators), `round_time_ms` | Analyze consensus timing | -| **RPC** | `command`, `version`, `rpc_status`, `duration_ms` | Monitor RPC performance | -| **Peer** | `peer_id`(public key), `peer_latency_ms`, `message_type`, `message_size_bytes` | Network topology analysis | -| **Ledger** | `ledger_hash`, `ledger_index`, `close_time`, `ledger_tx_count` | Ledger progression tracking | -| **Job** | `job_type`, `job_queue_ms`, `job_worker` | JobQueue performance | - -### What is NOT Collected (Privacy Guarantees) - -```mermaid -flowchart LR - subgraph notCollected["❌ NOT Collected"] - direction LR - A["Private Keys"] ~~~ B["Account Balances"] ~~~ C["Transaction Amounts"] - end - - subgraph alsoNot["❌ Also Excluded"] - direction LR - D["IP Addresses
(configurable)"] ~~~ E["Personal Data"] ~~~ F["Raw TX Payloads"] - end - - style A fill:#c62828,stroke:#8c2809,color:#fff - style B fill:#c62828,stroke:#8c2809,color:#fff - style C fill:#c62828,stroke:#8c2809,color:#fff - style D fill:#c62828,stroke:#8c2809,color:#fff - style E fill:#c62828,stroke:#8c2809,color:#fff - style F fill:#c62828,stroke:#8c2809,color:#fff -``` - -**Reading the diagram:** - -- **NOT Collected (top row, red)**: Private Keys, Account Balances, and Transaction Amounts are explicitly excluded — these are financial/security-sensitive fields that telemetry never touches. -- **Also Excluded (bottom row, red)**: IP Addresses (configurable per deployment), Personal Data, and Raw TX Payloads are also excluded — these protect operator and user privacy. -- **All-red styling**: Every box is styled in red to visually reinforce that these are hard exclusions, not optional — the telemetry system has no code path to collect any of these fields. -- **Two-row layout**: The split between "NOT Collected" and "Also Excluded" distinguishes between financial data (top) and operational/personal data (bottom), making the privacy boundaries clear to auditors. - -### Privacy Protection Mechanisms - -| Mechanism | Description | -| -------------------------- | --------------------------------------------------------- | -| **Account Hashing** | `tx_account` is hashed at collector level before storage | -| **Configurable Redaction** | Sensitive fields can be excluded via config | -| **Sampling** | Only 10% of traces recorded by default (reduces exposure) | -| **Local Control** | Node operators control what gets exported | -| **No Raw Payloads** | Transaction content is never recorded, only metadata | - -> **Key Principle**: Telemetry collects **operational metadata** (timing, counts, hashes) — never **sensitive content** (keys, balances, amounts). - ---- - -_End of Presentation_ diff --git a/OpenTelemetryPlan/secure-OTel.md b/OpenTelemetryPlan/secure-OTel.md index c570a06ecf..3d22de5452 100644 --- a/OpenTelemetryPlan/secure-OTel.md +++ b/OpenTelemetryPlan/secure-OTel.md @@ -20,7 +20,7 @@ The most effective way to prevent spoofing from external sources is to treat you server { listen 80; - location { + location / { # Clear out untrusted incoming trace headers proxy_set_header traceparent ""; proxy_set_header tracestate ""; @@ -92,10 +92,11 @@ receivers: grpc: endpoint: 0.0.0.0:4317 tls: + # Setting client_ca_file makes the collector require and verify a + # client cert, rejecting connections without a trusted one. client_ca_file: /certs/client_ca.pem # CA that signs trusted client certs cert_file: /certs/collector.pem key_file: /certs/collector.key - auth_type: require_and_verify_client_cert # Rejects unauthorized clients ``` ### **Approach C: Application Layer Authentication (Basic Auth Extension)** @@ -175,7 +176,7 @@ xrpld has **two distinct attack surfaces**, not one. The original guide conflate │ │ OTLP/gRPC │ │ + mTLS │ ▼ - └───────────────────────────────────────── [require_and_verify_client_cert] + └───────────────────────────────────────── [client_ca_file: verify client cert] OTel Collector (in private subnet, NetPol) ``` @@ -196,15 +197,15 @@ The guide's NGINX header stripping and OTTL stale-span filtering target HTTP gat Evaluated for the across-network deployment shape: -| Approach | Across-network fit | Cost | Verdict | -| ------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------- | ---------------------------------- | -| **A. NetworkPolicy / firewall** | Necessary baseline (don't expose `4317`/`4318` to the internet), but insufficient on its own when traffic genuinely crosses networks — you cannot NetworkPolicy the public internet. | Cheap. | **Defense-in-depth, not primary.** | -| **B. mTLS** | Strongest fit. Every xrpld node holds a client cert; collector verifies with `require_and_verify_client_cert`. Encrypts in transit (raw OTLP over the internet leaks transaction patterns and validator identity). Compromised node = revoke one cert, no shared secret to rotate everywhere. | Cert issuance + rotation pipeline. | **Primary.** | -| **C. Basic Auth** | Worst shape for this topology. Single shared password across all xrpld nodes — one leaked node config compromises the whole fleet. Doesn't encrypt; you'd need TLS underneath anyway, at which point you're 80% of the way to mTLS. | Cheap to set up, expensive to operate (rotation across N operators). | **Skip.** | +| Approach | Across-network fit | Cost | Verdict | +| ------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------- | ---------------------------------- | +| **A. NetworkPolicy / firewall** | Necessary baseline (don't expose `4317`/`4318` to the internet), but insufficient on its own when traffic genuinely crosses networks — you cannot NetworkPolicy the public internet. | Cheap. | **Defense-in-depth, not primary.** | +| **B. mTLS** | Strongest fit. Every xrpld node holds a client cert; the collector verifies it via `client_ca_file` in the receiver's `tls` block. Encrypts in transit (raw OTLP over the internet leaks transaction patterns and validator identity). Compromised node = revoke one cert, no shared secret to rotate everywhere. | Cert issuance + rotation pipeline. | **Primary.** | +| **C. Basic Auth** | Worst shape for this topology. Single shared password across all xrpld nodes — one leaked node config compromises the whole fleet. Doesn't encrypt; you'd need TLS underneath anyway, at which point you're 80% of the way to mTLS. | Cheap to set up, expensive to operate (rotation across N operators). | **Skip.** | ## Decision -**Primary defense:** mTLS (Approach B) on the collector's OTLP receivers, with `auth_type: require_and_verify_client_cert`. +**Primary defense:** mTLS (Approach B) on the collector's OTLP receivers. The collector requires and verifies each client certificate when `client_ca_file` is set in the receiver's `tls` block (there is no `auth_type` field — setting `client_ca_file` is what enforces client-cert verification). **Defense-in-depth:** NetworkPolicy / firewall rules (Approach A) so `4317`/`4318` are never reachable from outside the expected operator subnets even if mTLS were misconfigured. diff --git a/cmake/XrplCore.cmake b/cmake/XrplCore.cmake index f59c67510c..fa54ca7255 100644 --- a/cmake/XrplCore.cmake +++ b/cmake/XrplCore.cmake @@ -210,10 +210,13 @@ target_link_libraries( # When telemetry=ON, links the Conan-provided umbrella target # opentelemetry-cpp::opentelemetry-cpp (individual component targets like # ::api, ::sdk are not available in the Conan package). +# +# Links xrpl.libxrpl.protocol PRIVATELY for sha512Half (digest.h) add_module(xrpl telemetry) target_link_libraries( xrpl.libxrpl.telemetry PUBLIC xrpl.libxrpl.basics xrpl.libxrpl.beast xrpl.libxrpl.config + PRIVATE xrpl.libxrpl.protocol ) if(telemetry) target_link_libraries( diff --git a/docker/telemetry/docker-compose.yml b/docker/telemetry/docker-compose.yml index b2db1be183..95b793f2bc 100644 --- a/docker/telemetry/docker-compose.yml +++ b/docker/telemetry/docker-compose.yml @@ -54,7 +54,8 @@ services: - xrpld-telemetry prometheus: - image: prom/prometheus:latest + # Pinned to the v2.53 LTS line for reproducible, config-stable runs. + image: prom/prometheus:v2.53.3 ports: - "9090:9090" volumes: diff --git a/docker/telemetry/grafana/provisioning/datasources/tempo.yaml b/docker/telemetry/grafana/provisioning/datasources/tempo.yaml index 356fe7d504..56c0d9df65 100644 --- a/docker/telemetry/grafana/provisioning/datasources/tempo.yaml +++ b/docker/telemetry/grafana/provisioning/datasources/tempo.yaml @@ -47,7 +47,7 @@ datasources: tag: service.instance.id operator: "=" scope: resource - type: static + type: dynamic # service.version: xrpld build version (e.g., "2.4.0-b1"). # Filter traces from specific software releases. - id: node-version @@ -69,7 +69,7 @@ datasources: tag: xrpl.network.type operator: "=" scope: resource - type: static + type: dynamic # --- Span intrinsic filters --- # name: the span operation name (e.g., "rpc.command.server_info"). # Use to find traces for a specific RPC command or subsystem. @@ -77,21 +77,21 @@ datasources: tag: name operator: "=" scope: intrinsic - type: static + type: dynamic # status: span completion status ("ok", "error", "unset"). # Filter for failed operations to diagnose errors. - id: span-status tag: status operator: "=" scope: intrinsic - type: static + type: dynamic # duration: span wall-clock duration. Use with ">" operator # to find slow operations (e.g., duration > 500ms). - id: span-duration tag: duration operator: ">" scope: intrinsic - type: static + type: dynamic # Phase 2: RPC tracing filters - id: rpc-command tag: command @@ -144,12 +144,12 @@ datasources: tag: close_time_correct operator: "=" scope: span - type: dynamic + type: static - id: consensus-state tag: consensus_state operator: "=" scope: span - type: dynamic + type: static - id: consensus-close-resolution tag: close_resolution_ms operator: "=" diff --git a/docker/telemetry/otel-collector-config.yaml b/docker/telemetry/otel-collector-config.yaml index 96c44421ec..cccdce919e 100644 --- a/docker/telemetry/otel-collector-config.yaml +++ b/docker/telemetry/otel-collector-config.yaml @@ -51,6 +51,15 @@ processors: action: delete - key: telemetry.sdk.version action: delete + # Defense-in-depth: hash path-finding account attributes. The xrpld SDK + # already hashes these before export, but a node that emitted raw values + # is caught here so raw addresses never reach the backend. + attributes/hash: + actions: + - key: pathfind_source_account + action: hash + - key: pathfind_dest_account + action: hash connectors: spanmetrics: @@ -115,7 +124,7 @@ service: pipelines: traces: receivers: [otlp] - processors: [resource/tier, resource/stripsdk, batch] + processors: [resource/tier, resource/stripsdk, attributes/hash, batch] exporters: [debug, otlp/tempo, spanmetrics] metrics: receivers: [otlp, spanmetrics] diff --git a/docker/telemetry/xrpld-telemetry.cfg b/docker/telemetry/xrpld-telemetry.cfg index d233ad751a..74b5fecb72 100644 --- a/docker/telemetry/xrpld-telemetry.cfg +++ b/docker/telemetry/xrpld-telemetry.cfg @@ -7,7 +7,7 @@ # ./xrpld --conf docker/telemetry/xrpld-telemetry.cfg -a --start # 3. Send RPC commands to exercise tracing: # curl -s http://localhost:5005 -d '{"method":"server_info"}' -# 4. View traces in Jaeger UI: http://localhost:16686 +# 4. View traces in Grafana Explore: http://localhost:3000/explore [server] port_rpc_admin_local @@ -57,3 +57,11 @@ trace_transactions=1 trace_consensus=1 trace_peer=1 trace_ledger=1 + +# --- StatsD metrics (beast::insight) --- +# Sends UDP metrics to the collector's StatsD receiver so the StatsD +# dashboards have data. The collector binds this port on the host loopback. +[insight] +server=statsd +address=127.0.0.1:8125 +prefix=rippled diff --git a/docs/telemetry-runbook.md b/docs/telemetry-runbook.md index b462aabe70..8a7b97b082 100644 --- a/docs/telemetry-runbook.md +++ b/docs/telemetry-runbook.md @@ -104,6 +104,15 @@ RED metrics on the _Transaction Overview_ dashboard. | `txq.accept_tx` | TxQ.cpp | `tx_hash`, `retries_remaining`, `ter_code`, `txq_status` | Per-transaction apply during accept | | `txq.cleanup` | TxQ.cpp | `ledger_seq` | Post-close cleanup of expired queue entries | +### PathFinding Spans + +| Span Name | Source File | Attributes | Description | +| --------------------- | --------------------------------- | -------------------------------------------------- | ------------------------------------------------------- | +| `pathfind.request` | PathFind.cpp / RipplePathFind.cpp | `pathfind_source_account`, `pathfind_dest_account` | Path-find RPC entry (accounts hashed; set when present) | +| `pathfind.compute` | PathRequest.cpp | `pathfind_fast`, `pathfind_dest_currency` | Path computation for one request (`doUpdate`) | +| `pathfind.discover` | PathRequest.cpp | `pathfind_search_level`, `pathfind_num_paths` | Graph exploration (one per RPC call in `findPaths`) | +| `pathfind.update_all` | PathRequestManager.cpp | `pathfind_ledger_index`, `pathfind_num_requests` | Async recomputation of active requests on ledger close | + ### Consensus Spans | Span Name | Source File | Attributes | Description | @@ -114,7 +123,7 @@ RED metrics on the _Transaction Overview_ dashboard. | `consensus.ledger_close` | RCLConsensus.cpp | `ledger_seq`, `consensus_mode` | Ledger close event | | `consensus.establish` | Consensus.h | `converge_percent`, `establish_count`, `proposers` | Establish phase duration (child of round) | | `consensus.update_positions` | Consensus.h | `converge_percent`, `proposers`, `disputes_count` | Position update and dispute resolution (see Events below) | -| `consensus.check` | Consensus.h | `agree_count`, `disagree_count`, `converge_percent`, `have_close_time_consensus`, `threshold_percent`, `consensus_result` | Consensus threshold check | +| `consensus.check` | Consensus.h | `agree_count`, `disagree_count`, `converge_percent`, `have_close_time_consensus`, `threshold_percent`, `proposers_finished`, `consensus_stalled`, `establish_count`, `consensus_result` | Consensus threshold check | | `consensus.accept` | RCLConsensus.cpp | `proposers`, `round_time_ms`, `quorum`, `disputes_count`, `consensus_state` | Ledger accepted by consensus | | `consensus.accept.apply` | RCLConsensus.cpp | `ledger_seq`, `close_time`, `close_time_correct`, `close_resolution_ms`, `consensus_state`, `proposing`, `round_time_ms`, `parent_close_time`, `close_time_self`, `close_time_vote_bins`, `resolution_direction`, `tx_count` | Ledger application with close time details (see Events below) | | `consensus.validation.send` | RCLConsensus.cpp | `ledger_seq`, `proposing`, `ledger_hash`, `full_validation`, `validation_sign_time` | Validation sent after accept (follows-from link) | @@ -402,7 +411,7 @@ all its normal attributes, it just lacks a cross-node parent link. {name="tx.receive"} && status != error # Find proposals received with cross-node parent context -{name="consensus.proposal.receive"} && nestedSetParent > 0 +{} >> {name="consensus.proposal.receive"} # Trace a transaction across the network by its hash {name=~"tx\\..*"} | tx_hash = "" @@ -438,7 +447,12 @@ Every metric carries these standard labels: | `service_name` | Resource attribute | `xrpld` | | `span_kind` | Span kind | `SPAN_KIND_INTERNAL` | -Additionally, span attributes configured as dimensions in the collector become metric labels. The collector dimensions use the bare attribute keys emitted by the code, so the label name equals the attribute name: +Additionally, span attributes configured as dimensions in the collector +become metric labels. The span attribute keys are already underscore form +(the naming convention forbids dots), so the label name matches the attribute +name verbatim. Prometheus' dots → underscores sanitization only fires for +dotted attribute names (e.g. resource attributes like `service.name`), which +does not apply to these dimensions. | Span Attribute | Metric Label | Applies To | | -------------------- | -------------------- | ------------------------------- | diff --git a/include/xrpl/telemetry/Redaction.h b/include/xrpl/telemetry/Redaction.h new file mode 100644 index 0000000000..ed84ae2964 --- /dev/null +++ b/include/xrpl/telemetry/Redaction.h @@ -0,0 +1,64 @@ +#pragma once + +/** Account-address redaction for telemetry span attributes. + + Path-finding RPC handlers would otherwise emit the caller's raw + account addresses as span attributes. To keep plaintext addresses out + of the telemetry backend, they are hashed at the point of emission. + This header exposes a single pure helper that turns an address into a + short, stable, obfuscated token. + + Data flow: + + handler -> redactAccount(addr) -> span attribute -> OTLP export + + The returned token is the first 16 hex characters (lowercase) of the + SHA-512Half digest of the address. It is deterministic (same address + always maps to the same token) so operators can still correlate spans + for a given account across nodes and restarts. + + The hash is unsalted, so it is obfuscation, not a secrecy guarantee: + XRP account addresses are a public, enumerable set, so a determined + observer with the telemetry stream could rebuild the address->token + mapping. The goal here is to keep plaintext addresses out of traces + and dashboards, not to defend against a precomputation attack. A salt + is intentionally omitted because it would break cross-node/restart + correlation, which is the reason for hashing rather than dropping. + + A second, independent hashing layer runs in the OpenTelemetry + Collector (an `attributes/hash` processor) as defense-in-depth for + any node that emits a raw value. + + @note This function is pure and reentrant: it holds no global state, + performs no I/O, and is safe to call concurrently from any thread. + + Usage example: + @code + #include + using namespace xrpl::telemetry; + + span.setAttribute( + pathfind_span::attr::sourceAccount, redactAccount(src.asString())); + @endcode + + Edge case (empty input yields empty output): + @code + assert(redactAccount("") == ""); + @endcode +*/ + +#include +#include + +namespace xrpl::telemetry { + +/** Hash an account address into a short, stable, obfuscated token. + + @param addr The account address to redact (e.g. an r-address). + @return The first 16 lowercase hex characters of sha512Half(addr), + or an empty string when @p addr is empty. +*/ +[[nodiscard]] std::string +redactAccount(std::string_view addr); + +} // namespace xrpl::telemetry diff --git a/include/xrpl/tx/detail/TxApplySpanNames.h b/include/xrpl/tx/detail/TxApplySpanNames.h index c007ca3c03..2084458cf5 100644 --- a/include/xrpl/tx/detail/TxApplySpanNames.h +++ b/include/xrpl/tx/detail/TxApplySpanNames.h @@ -51,9 +51,11 @@ * @endcode * * @code - * // Transactor::operator() uses span() with prefix + suffix: - * auto span = SpanGuard::span( - * TraceCategory::Transactions, seg::tx, tx_apply_span::op::transactor); + * // Transactor::operator() also uses hashSpan on the same txID so it + * // co-traces with preflight and preclaim under one trace_id: + * auto span = SpanGuard::hashSpan( + * TraceCategory::Transactions, tx_apply_span::transactor, + * txID.data(), txID.kBytes); * span.setAttribute(tx_apply_span::attr::stage, tx_apply_span::val::apply); * @endcode */ @@ -79,6 +81,9 @@ inline constexpr auto transactor = makeStr("transactor"); inline constexpr auto preflight = join(seg::tx, op::preflight); /// "tx.preclaim" — full name for hashSpan() at the preclaim stage. inline constexpr auto preclaim = join(seg::tx, op::preclaim); +/// "tx.transactor" — full name for hashSpan() at the apply stage. Shares the +/// txID-derived trace_id so it co-traces with tx.preflight and tx.preclaim. +inline constexpr auto transactor = join(seg::tx, op::transactor); // ===== Attribute keys ====================================================== diff --git a/src/libxrpl/telemetry/Redaction.cpp b/src/libxrpl/telemetry/Redaction.cpp new file mode 100644 index 0000000000..ee15b8812e --- /dev/null +++ b/src/libxrpl/telemetry/Redaction.cpp @@ -0,0 +1,32 @@ +#include + +#include +#include +#include + +#include +#include +#include +#include + +namespace xrpl::telemetry { + +std::string +redactAccount(std::string_view addr) +{ + // Empty in, empty out: nothing to hash and no token to emit. + if (addr.empty()) + return {}; + + // sha512Half yields a uint256; to_string renders it as uppercase hex. + // Keep the first 16 chars (64 bits) — enough to correlate spans while + // staying non-reversible — and lowercase them for a stable token. + auto const digest = sha512Half(Slice(addr.data(), addr.size())); + std::string token = to_string(digest).substr(0, 16); + std::transform(token.begin(), token.end(), token.begin(), [](unsigned char c) { + return static_cast(std::tolower(c)); + }); + return token; +} + +} // namespace xrpl::telemetry diff --git a/src/libxrpl/telemetry/SpanGuard.cpp b/src/libxrpl/telemetry/SpanGuard.cpp index 2e7dc742c7..74206e46e1 100644 --- a/src/libxrpl/telemetry/SpanGuard.cpp +++ b/src/libxrpl/telemetry/SpanGuard.cpp @@ -238,6 +238,7 @@ SpanGuard::linkedSpan(std::string_view name) const return SpanGuard( std::make_unique(tracer->StartSpan( std::string(name), {}, {{spanCtx, {{kLinkTypeKey, kLinkTypeFollowsFrom}}}}, opts))); + // LCOV_EXCL_STOP } SpanGuard diff --git a/src/libxrpl/telemetry/TelemetryConfig.cpp b/src/libxrpl/telemetry/TelemetryConfig.cpp index 00705aa4ed..17eacf6582 100644 --- a/src/libxrpl/telemetry/TelemetryConfig.cpp +++ b/src/libxrpl/telemetry/TelemetryConfig.cpp @@ -112,6 +112,16 @@ makeTelemetrySetup( "(set both for mutual TLS, or neither for one-way TLS)."); } + // Mutual TLS only takes effect when TLS is on. Certificate paths set with + // use_tls=0 would be silently ignored and the exporter would connect in + // plaintext, so reject that contradiction instead of failing open. + if (!setup.tlsClientCertPath.empty() && !setup.useTls) + { + Throw( + "[telemetry] tls_client_cert/tls_client_key require use_tls=1 " + "(set use_tls=1 to enable mutual TLS, or remove the cert paths)."); + } + // Head sampling is intentionally fixed at 1.0 (sample everything) and is // not read from config. A per-node ratio would let nodes make divergent // keep/drop decisions for the same distributed trace, producing broken diff --git a/src/libxrpl/tx/Transactor.cpp b/src/libxrpl/tx/Transactor.cpp index 0c33995db2..e2752ba67b 100644 --- a/src/libxrpl/tx/Transactor.cpp +++ b/src/libxrpl/tx/Transactor.cpp @@ -1294,10 +1294,15 @@ Transactor::checkInvariants(TER result, XRPAmount fee) ApplyResult Transactor::operator()() { - auto span = telemetry::SpanGuard::span( + // Derive the trace_id from the transaction id so this apply-stage span + // shares one trace with the preflight and preclaim spans (which also use + // hashSpan on the same id). + auto const txID = ctx_.tx.getTransactionID(); + auto span = telemetry::SpanGuard::hashSpan( telemetry::TraceCategory::Transactions, - telemetry::seg::tx, - telemetry::tx_apply_span::op::transactor); + telemetry::tx_apply_span::transactor, + txID.data(), + txID.kBytes); // "apply" — the third apply-pipeline stage, after preflight and preclaim. span.setAttribute(telemetry::tx_apply_span::attr::stage, telemetry::tx_apply_span::val::apply); if (auto const* fmt = TxFormats::getInstance().findByType(ctx_.tx.getTxnType())) @@ -1428,6 +1433,11 @@ Transactor::operator()() span.setAttribute(telemetry::tx_apply_span::attr::terResult, transToken(result).c_str()); span.setAttribute(telemetry::tx_apply_span::attr::applied, applied); + // Mark the span as errored when the transaction was not applied or the + // engine result is not a success, so failed applies surface in span-status + // error counts alongside preflight and preclaim. + if (!applied || !isTesSuccess(result)) + span.setError(transToken(result)); return {result, applied, metadata}; } diff --git a/src/libxrpl/tx/applySteps.cpp b/src/libxrpl/tx/applySteps.cpp index 77c39d6f49..8a2c97c550 100644 --- a/src/libxrpl/tx/applySteps.cpp +++ b/src/libxrpl/tx/applySteps.cpp @@ -190,6 +190,10 @@ invokePreflight(PreflightContext const& ctx) { span.setAttribute( telemetry::tx_apply_span::attr::terResult, transToken(result.first).c_str()); + // Mark the span as errored when preflight rejects the transaction so + // failed stages surface in span-status error counts. + if (!isTesSuccess(result.first)) + span.setError(transToken(result.first)); } return result; } diff --git a/src/tests/libxrpl/telemetry/Redaction.cpp b/src/tests/libxrpl/telemetry/Redaction.cpp new file mode 100644 index 0000000000..7f16c0f1dc --- /dev/null +++ b/src/tests/libxrpl/telemetry/Redaction.cpp @@ -0,0 +1,44 @@ +#include + +#include + +#include +#include +#include + +using namespace xrpl; +using namespace xrpl::telemetry; + +// Empty input must produce empty output (edge / negative path). +TEST(Redaction, empty_input_returns_empty) +{ + EXPECT_EQ(redactAccount(""), ""); +} + +// Success path: assert the EXACT expected token. The value is the first +// 16 chars of sha512Half(addr) rendered as lowercase hex, computed once +// and hard-coded here so any change to the hashing recipe is caught. +TEST(Redaction, known_account_exact_hash) +{ + EXPECT_EQ(redactAccount("rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh"), "a513895f49311f54"); +} + +// Structural invariants: length, lowercase-hex alphabet, and stability. +TEST(Redaction, token_is_16_lowercase_hex_and_stable) +{ + auto const token = redactAccount("rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh"); + EXPECT_EQ(token.size(), 16u); + EXPECT_TRUE(std::all_of(token.begin(), token.end(), [](unsigned char c) { + return std::isdigit(c) || (c >= 'a' && c <= 'f'); + })); + // Deterministic: hashing the same input twice yields the same token. + EXPECT_EQ(token, redactAccount("rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh")); +} + +// Distinct inputs must produce distinct tokens (no accidental constant). +TEST(Redaction, distinct_inputs_distinct_tokens) +{ + EXPECT_NE( + redactAccount("rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh"), + redactAccount("rDifferentAccount1234567890abcdefgh")); +} diff --git a/src/tests/libxrpl/telemetry/TraceContextPropagator.cpp b/src/tests/libxrpl/telemetry/TraceContextPropagator.cpp index 67b4428b75..8755284630 100644 --- a/src/tests/libxrpl/telemetry/TraceContextPropagator.cpp +++ b/src/tests/libxrpl/telemetry/TraceContextPropagator.cpp @@ -81,10 +81,8 @@ TEST(TraceContextPropagator, extract_empty_protobuf) protocol::TraceContext const proto; auto ctx = xrpl::telemetry::extractFromProtobuf(proto); auto span = trace::GetSpan(ctx); - if (span) - { - EXPECT_FALSE(span->GetContext().IsValid()); - } + ASSERT_NE(span, nullptr); + EXPECT_FALSE(span->GetContext().IsValid()); } TEST(TraceContextPropagator, extract_wrong_size_trace_id) @@ -95,10 +93,8 @@ TEST(TraceContextPropagator, extract_wrong_size_trace_id) auto ctx = xrpl::telemetry::extractFromProtobuf(proto); auto span = trace::GetSpan(ctx); - if (span) - { - EXPECT_FALSE(span->GetContext().IsValid()); - } + ASSERT_NE(span, nullptr); + EXPECT_FALSE(span->GetContext().IsValid()); } TEST(TraceContextPropagator, extract_wrong_size_span_id) @@ -109,10 +105,8 @@ TEST(TraceContextPropagator, extract_wrong_size_span_id) auto ctx = xrpl::telemetry::extractFromProtobuf(proto); auto span = trace::GetSpan(ctx); - if (span) - { - EXPECT_FALSE(span->GetContext().IsValid()); - } + ASSERT_NE(span, nullptr); + EXPECT_FALSE(span->GetContext().IsValid()); } TEST(TraceContextPropagator, inject_invalid_span) diff --git a/src/xrpld/app/consensus/RCLConsensus.cpp b/src/xrpld/app/consensus/RCLConsensus.cpp index 51cc8bcb94..978d9c9ee2 100644 --- a/src/xrpld/app/consensus/RCLConsensus.cpp +++ b/src/xrpld/app/consensus/RCLConsensus.cpp @@ -1260,8 +1260,25 @@ RCLConsensus::Adaptor::startRoundTracing(RCLCxLedger const& prevLgr) telemetry::SpanContext const* const link = prevRoundSpanContext_.isValid() ? &prevRoundSpanContext_ : nullptr; - if (strategy == "deterministic") + if (strategy == "attribute") { + // Non-deterministic strategy: each node gets a random trace_id, + // correlated via the consensus_ledger_id attribute rather than a + // shared trace_id. Still attach a follows-from link to the prior + // round so consecutive rounds stay navigable. linkedSpan is not + // TraceCategory-aware, so gate it explicitly to match the gating + // of the hashSpan/span factories used below. + if (link != nullptr && app_.getTelemetry().shouldTraceConsensus()) + roundSpan_.emplace(telemetry::SpanGuard::linkedSpan(cs::round, *link)); + else + roundSpan_.emplace( + telemetry::SpanGuard::span( + telemetry::TraceCategory::Consensus, telemetry::seg::consensus, cs::op::round)); + } + else + { + // "deterministic" (the default): derive the trace_id from the previous + // ledger hash so all validators tracing the same round share one trace. roundSpan_.emplace( telemetry::SpanGuard::hashSpan( telemetry::TraceCategory::Consensus, @@ -1270,12 +1287,6 @@ RCLConsensus::Adaptor::startRoundTracing(RCLCxLedger const& prevLgr) prevLgr.id().kBytes, link)); } - else - { - roundSpan_.emplace( - telemetry::SpanGuard::span( - telemetry::TraceCategory::Consensus, telemetry::seg::consensus, cs::op::round)); - } if (!*roundSpan_) return; diff --git a/src/xrpld/app/misc/NetworkOPs.cpp b/src/xrpld/app/misc/NetworkOPs.cpp index 20e4d3758c..3dd384d605 100644 --- a/src/xrpld/app/misc/NetworkOPs.cpp +++ b/src/xrpld/app/misc/NetworkOPs.cpp @@ -17,7 +17,6 @@ #include #include #include -#include #include #include #include @@ -35,6 +34,7 @@ #include #include #include +#include #include #include diff --git a/src/xrpld/app/misc/detail/TxQ.cpp b/src/xrpld/app/misc/detail/TxQ.cpp index 124a115556..2e0b37f291 100644 --- a/src/xrpld/app/misc/detail/TxQ.cpp +++ b/src/xrpld/app/misc/detail/TxQ.cpp @@ -1449,9 +1449,6 @@ bool TxQ::accept(Application& app, OpenView& view) { using namespace telemetry; - auto span = - SpanGuard::span(TraceCategory::Transactions, txq_span::prefix::txq, txq_span::op::accept); - span.setAttribute(txq_span::attr::queueSize, static_cast(byFee_.size())); /* Move transactions from the queue from largest fee level to smallest. As we add more transactions, the required fee level will increase. @@ -1463,6 +1460,12 @@ TxQ::accept(Application& app, OpenView& view) std::scoped_lock const lock(mutex_); + // Create the span and read byFee_.size() only after taking the lock, since + // byFee_ is guarded by mutex_. + auto span = + SpanGuard::span(TraceCategory::Transactions, txq_span::prefix::txq, txq_span::op::accept); + span.setAttribute(txq_span::attr::queueSize, static_cast(byFee_.size())); + auto const metricsSnapshot = feeMetrics_.getSnapshot(); for (auto candidateIter = byFee_.begin(); candidateIter != byFee_.end();) diff --git a/src/xrpld/overlay/detail/PeerImp.cpp b/src/xrpld/overlay/detail/PeerImp.cpp index dd724f375d..6d960b33bd 100644 --- a/src/xrpld/overlay/detail/PeerImp.cpp +++ b/src/xrpld/overlay/detail/PeerImp.cpp @@ -7,7 +7,6 @@ #include #include #include -#include #include #include #include @@ -25,6 +24,7 @@ #include #include #include +#include #include #include diff --git a/src/xrpld/rpc/detail/PathFindSpanNames.h b/src/xrpld/rpc/detail/PathFindSpanNames.h index 3a12937fbc..71eb805ea3 100644 --- a/src/xrpld/rpc/detail/PathFindSpanNames.h +++ b/src/xrpld/rpc/detail/PathFindSpanNames.h @@ -84,8 +84,6 @@ inline constexpr auto numPaths = makeStr("pathfind_num_paths"); inline constexpr auto numRequests = makeStr("pathfind_num_requests"); /// "pathfind_ledger_index" — pathfind target ledger index. inline constexpr auto ledgerIndex = makeStr("pathfind_ledger_index"); -/// "pathfind_dest_amount" — requested destination amount as string. -inline constexpr auto destAmount = makeStr("pathfind_dest_amount"); /// "pathfind_dest_currency" — destination currency code. inline constexpr auto destCurrency = makeStr("pathfind_dest_currency"); /// "pathfind_num_source_assets" — candidate source assets count. diff --git a/src/xrpld/rpc/detail/PathRequest.cpp b/src/xrpld/rpc/detail/PathRequest.cpp index 324036d39b..4fb11ab10c 100644 --- a/src/xrpld/rpc/detail/PathRequest.cpp +++ b/src/xrpld/rpc/detail/PathRequest.cpp @@ -742,7 +742,6 @@ PathRequest::doUpdate( auto span = SpanGuard::span( TraceCategory::Rpc, pathfind_span::prefix::pathfind, pathfind_span::op::compute); span.setAttribute(pathfind_span::attr::fast, fast); - span.setAttribute(pathfind_span::attr::destAmount, saDstAmount_.getFullText().c_str()); span.setAttribute(pathfind_span::attr::destCurrency, to_string(saDstAmount_.asset()).c_str()); JLOG(journal_.debug()) << iIdentifier_ << " update " << (fast ? "fast" : "normal"); diff --git a/src/xrpld/rpc/detail/PathRequestManager.cpp b/src/xrpld/rpc/detail/PathRequestManager.cpp index b2831c5518..01df901ce3 100644 --- a/src/xrpld/rpc/detail/PathRequestManager.cpp +++ b/src/xrpld/rpc/detail/PathRequestManager.cpp @@ -73,15 +73,16 @@ PathRequestManager::updateAll(std::shared_ptr const& inLedger) cache = getAssetCache(inLedger, true); } - // updateAll runs on every ledger close; skip span emission entirely when - // there are no active path subscriptions to avoid a steady stream of empty - // spans at mainnet close cadence. - if (requests.empty()) - return; - using namespace telemetry; - auto span = SpanGuard::span( - TraceCategory::Rpc, pathfind_span::prefix::pathfind, pathfind_span::op::updateAll); + // updateAll runs on every ledger close. Skip span emission when there are + // no active path subscriptions, to avoid a steady stream of empty spans at + // mainnet close cadence. A null guard is used in that case; all other work + // still runs unchanged (notably the isNewPathRequest() flag reset below), + // so behaviour matches the pre-span code path. + auto span = requests.empty() + ? SpanGuard{} + : SpanGuard::span( + TraceCategory::Rpc, pathfind_span::prefix::pathfind, pathfind_span::op::updateAll); span.setAttribute(pathfind_span::attr::ledgerIndex, static_cast(inLedger->seq())); span.setAttribute(pathfind_span::attr::numRequests, static_cast(requests.size())); diff --git a/src/xrpld/rpc/detail/RPCHandler.cpp b/src/xrpld/rpc/detail/RPCHandler.cpp index 14082e3b88..1a918c343c 100644 --- a/src/xrpld/rpc/detail/RPCHandler.cpp +++ b/src/xrpld/rpc/detail/RPCHandler.cpp @@ -221,6 +221,31 @@ callMethod(JsonContext& context, Method method, std::string const& name, Object& } } +// Resolve the span suffix / command attribute for a request that failed in +// fillHandler. Returns the canonical handler name for a recognized command +// (a finite, bounded set) or the literal "unknown" for a request that omits +// both fields or names an unregistered command. The raw request value is +// deliberately NOT used: the command attribute is promoted to a Prometheus +// label by the spanmetrics connector, so an attacker-controlled string would +// let arbitrary request input drive unbounded span-name / label cardinality. +// Resolving against the registry keeps per-command error attribution for real +// commands (e.g. a submit rejected with rpcTOO_BUSY stays rpc.command.submit) +// while collapsing garbage input to a single series. +std::string_view +resolveCommandSpanName(JsonContext const& context) +{ + if (!context.params.isMember(jss::command) && !context.params.isMember(jss::method)) + return rpc_span::val::unknownCommand; + + std::string const cmd = context.params.isMember(jss::command) + ? context.params[jss::command].asString() + : context.params[jss::method].asString(); + + auto const* handler = getHandler(context.apiVersion, context.app.config().betaRpcApi, cmd); + return (handler != nullptr) ? std::string_view{handler->name} + : std::string_view{rpc_span::val::unknownCommand}; +} + } // namespace Status @@ -229,25 +254,12 @@ doCommand(RPC::JsonContext& context, json::Value& result) Handler const* handler = nullptr; if (auto error = fillHandler(context, handler)) { - std::string cmdName; - if (context.params.isMember(jss::command)) - { - cmdName = context.params[jss::command].asString(); - } - else if (context.params.isMember(jss::method)) - { - cmdName = context.params[jss::method].asString(); - } - else - { - cmdName = "unknown"; // LCOV_EXCL_LINE - } - // Use the resolved command name as the span suffix so dashboards - // can break out per-command error rates (e.g. rpc.command.submit - // for a submit that hit rpcTOO_BUSY). Falling back to a single - // "unknown" name only when the request truly omits both fields. + // Bound the span name and command attribute to the finite set of + // registered handler names (plus "unknown") — see the helper for why + // raw request input must not reach the telemetry pipeline. + auto const cmdName = resolveCommandSpanName(context); auto span = SpanGuard::span(TraceCategory::Rpc, rpc_span::prefix::command, cmdName); - span.setAttribute(rpc_span::attr::command, cmdName.c_str()); + span.setAttribute(rpc_span::attr::command, cmdName); span.setAttribute(rpc_span::attr::rpcStatus, rpc_span::val::error); span.setError(getErrorInfo(error).token.cStr()); diff --git a/src/xrpld/rpc/handlers/orderbook/PathFind.cpp b/src/xrpld/rpc/handlers/orderbook/PathFind.cpp index cecd1083c9..f04756be15 100644 --- a/src/xrpld/rpc/handlers/orderbook/PathFind.cpp +++ b/src/xrpld/rpc/handlers/orderbook/PathFind.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include namespace xrpl { @@ -20,10 +21,11 @@ doPathFind(RPC::JsonContext& context) using namespace telemetry; auto span = SpanGuard::span( TraceCategory::Rpc, pathfind_span::prefix::pathfind, pathfind_span::op::request); + // Addresses are hashed before emission for privacy. if (auto const& src = context.params[jss::source_account]; src.isString()) - span.setAttribute(pathfind_span::attr::sourceAccount, src.asString()); + span.setAttribute(pathfind_span::attr::sourceAccount, redactAccount(src.asString())); if (auto const& dst = context.params[jss::destination_account]; dst.isString()) - span.setAttribute(pathfind_span::attr::destAccount, dst.asString()); + span.setAttribute(pathfind_span::attr::destAccount, redactAccount(dst.asString())); if (context.app.config().pathSearchMax == 0) return rpcError(RpcNotSupported); diff --git a/src/xrpld/rpc/handlers/orderbook/RipplePathFind.cpp b/src/xrpld/rpc/handlers/orderbook/RipplePathFind.cpp index b55ba2515f..e08ef5445c 100644 --- a/src/xrpld/rpc/handlers/orderbook/RipplePathFind.cpp +++ b/src/xrpld/rpc/handlers/orderbook/RipplePathFind.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -28,10 +29,11 @@ doRipplePathFind(RPC::JsonContext& context) using namespace telemetry; auto span = SpanGuard::span( TraceCategory::Rpc, pathfind_span::prefix::pathfind, pathfind_span::op::request); + // Addresses are hashed before emission for privacy. if (auto const& src = context.params[jss::source_account]; src.isString()) - span.setAttribute(pathfind_span::attr::sourceAccount, src.asString()); + span.setAttribute(pathfind_span::attr::sourceAccount, redactAccount(src.asString())); if (auto const& dst = context.params[jss::destination_account]; dst.isString()) - span.setAttribute(pathfind_span::attr::destAccount, dst.asString()); + span.setAttribute(pathfind_span::attr::destAccount, redactAccount(dst.asString())); if (context.app.config().pathSearchMax == 0) return rpcError(RpcNotSupported); diff --git a/src/xrpld/app/misc/TxSpanNames.h b/src/xrpld/telemetry/TxSpanNames.h similarity index 100% rename from src/xrpld/app/misc/TxSpanNames.h rename to src/xrpld/telemetry/TxSpanNames.h diff --git a/src/xrpld/telemetry/TxTracing.h b/src/xrpld/telemetry/TxTracing.h index 47bb30eb54..066c018601 100644 --- a/src/xrpld/telemetry/TxTracing.h +++ b/src/xrpld/telemetry/TxTracing.h @@ -10,7 +10,7 @@ * no-op SpanGuard instances (zero overhead, zero dependencies). */ -#include +#include #include #include