diff --git a/.github/scripts/levelization/results/loops.txt b/.github/scripts/levelization/results/loops.txt index 35ed669ecb..fe92f35bd4 100644 --- a/.github/scripts/levelization/results/loops.txt +++ b/.github/scripts/levelization/results/loops.txt @@ -13,9 +13,6 @@ Loop: xrpld.app xrpld.rpc Loop: xrpld.app xrpld.shamap xrpld.shamap > xrpld.app -Loop: xrpld.app xrpld.telemetry - xrpld.telemetry ~= xrpld.app - Loop: xrpld.overlay xrpld.rpc xrpld.rpc ~= xrpld.overlay diff --git a/.github/scripts/levelization/results/ordering.txt b/.github/scripts/levelization/results/ordering.txt index 3668def3fd..bbb44307dd 100644 --- a/.github/scripts/levelization/results/ordering.txt +++ b/.github/scripts/levelization/results/ordering.txt @@ -47,6 +47,7 @@ libxrpl.shamap > xrpl.protocol libxrpl.shamap > xrpl.shamap libxrpl.telemetry > xrpl.basics libxrpl.telemetry > xrpl.config +libxrpl.telemetry > xrpl.protocol libxrpl.telemetry > xrpl.telemetry libxrpl.tx > xrpl.basics libxrpl.tx > xrpl.conditions @@ -263,6 +264,7 @@ xrpld.app > xrpl.config xrpld.app > xrpl.core xrpld.app > xrpld.consensus xrpld.app > xrpld.core +xrpld.app > xrpld.telemetry xrpld.app > xrpl.json xrpld.app > xrpl.ledger xrpld.app > xrpl.net diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c2aa11ba28..b787d06ad7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -321,7 +321,7 @@ other layer must match them. A CI check enforces this end to end. 3. Collision qualifier: `_` — only when a bare name would collide with a DIFFERENT concept in the shared spanmetrics label space, or with the OTel-reserved `status` key (e.g. `rpc_status`, `grpc_status`, - `consensus_state`, `consensus_round`). This disambiguates distinct concepts + `consensus_phase`, `consensus_round`). This disambiguates distinct concepts that share a word; it is NOT used to tag the same concept with the workflow that emitted it — that is rule 2 (one shared name). 4. Resource attribute: dotted `xrpl..` — reserved ONLY for diff --git a/OpenTelemetryPlan/02-design-decisions.md b/OpenTelemetryPlan/02-design-decisions.md index 3bee549eac..9bfee122c6 100644 --- a/OpenTelemetryPlan/02-design-decisions.md +++ b/OpenTelemetryPlan/02-design-decisions.md @@ -185,7 +185,7 @@ agree with the code. A CI check enforces this end to end. 3. **Collision qualifier** → `_`, only when a bare name would collide with a DIFFERENT concept in the shared spanmetrics label space or with the OTel-reserved `status` key (e.g. `rpc_status`, `grpc_status`, - `consensus_state`, `consensus_round`, `consensus_mode`). This disambiguates + `consensus_phase`, `consensus_round`, `consensus_mode`). This disambiguates distinct concepts that share a word; it is NOT used to tag the same concept with its emitting workflow — that is rule 2 (one shared name). 4. **Resource attribute** → dotted `xrpl..`, reserved ONLY @@ -245,6 +245,8 @@ keys (the dotted form is reserved for resource scope per §2.3.3). | `tx_fee` | int64 | Fee in drops | | `tx_result` | string | `"tesSUCCESS"`, `"tecPATH_DRY"`, etc. | | `ledger_index` | int64 | Ledger containing transaction | +| `relay_count` | int64 | Peers the transaction was relayed to | +| `suppressed` | bool | `true` when HashRouter dropped a dup | #### Consensus Attributes @@ -278,12 +280,14 @@ Establish-phase gap fill and cross-node correlation attributes (Phase 4a): #### RPC Attributes -| Key | Type | Description | -| ---------- | ------ | ----------------------------------------------------- | -| `command` | string | Command name (per-span unique on `rpc.command`) | -| `version` | int64 | API version | -| `rpc_role` | string | `"admin"` or `"user"` (qualified — `role` is generic) | -| `params` | string | Sanitized parameters (optional) | +| Key | Type | Description | +| ------------- | ------- | ----------------------------------------------------------------------------- | +| `command` | string | Command name (per-span unique on `rpc.command`) | +| `version` | int64 | API version | +| `rpc_role` | string | `"admin"` or `"user"` (qualified — `role` is generic) | +| `params` | string | Sanitized parameters (optional) | +| `rpc_status` | string | Response status: `success` \| `error` (qualified — `status` is OTel-reserved) | +| `duration_ms` | float64 | Request duration in milliseconds | #### Peer & Message Attributes @@ -395,32 +399,48 @@ The following data is explicitly **excluded** from telemetry collection: #### Privacy Protection Mechanisms -| Mechanism | Description | -| ----------------------------- | ------------------------------------------------------------------------- | -| **Account Hashing** | `tx_account` is hashed at collector level before storage | -| **Configurable Redaction** | Sensitive fields can be excluded via `[telemetry]` config section | -| **Sampling** | Only 10% of traces recorded by default, reducing data exposure | -| **Local Control** | Node operators have full control over what gets exported | -| **No Raw Payloads** | Transaction content is never recorded, only metadata (hash, type, result) | -| **Collector-Level Filtering** | Additional redaction/hashing can be configured at OTel Collector | +| Mechanism | Description | +| ----------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| **Account Hashing** | Account addresses are hashed both SDK-side (`pathfind_source_account`, `pathfind_dest_account` — always hashed before emission) and again at the collector level, so raw addresses never reach storage | +| **Configurable Redaction** | Sensitive fields can be excluded via `[telemetry]` config section | +| **Collector Tail Sampling** | xrpld head sampling is fixed at 1.0 (every span emitted); the collector retains ~10% of non-error traces, reducing stored data exposure | +| **Sampling** | Only 10% of traces recorded by default, reducing data exposure | +| **Local Control** | Node operators have full control over what gets exported | +| **No Raw Payloads** | Transaction content is never recorded, only metadata (hash, type, result) | +| **Collector-Level Filtering** | Additional redaction/hashing can be configured at OTel Collector | + +#### Account Address Hashing + +Account addresses are **always** hashed before they reach the telemetry +backend — there is no opt-out flag and therefore no insecure-by-default +failure mode. Protection is applied in two independent layers: + +1. **SDK-side** (this node): the path-finding RPC handlers call + `redactAccount()` (`xrpl::telemetry`, `Redaction.h`) before setting the + `pathfind_source_account` / `pathfind_dest_account` span attributes. The + helper emits the first 16 characters of `sha512Half(address)` as + lowercase hex — deterministic (spans for one account still correlate) + but non-reversible. +2. **Collector-side** (defense-in-depth): an `attributes/hash` processor in + the OpenTelemetry Collector re-hashes those same attributes, so any node + that emitted a raw value is still redacted before storage. #### Collector-Level Data Protection The OpenTelemetry Collector can be configured (via an `attributes` processor) to hash or redact sensitive attributes before export — for example, hashing -`tx_account`, deleting `peer_address` to drop IP addresses, and deleting -`params` to redact request parameters. +`pathfind_source_account` / `pathfind_dest_account`, deleting `peer_address` +to drop IP addresses, and deleting `params` to redact request parameters. #### Configuration Options for Privacy In `xrpld.cfg`, operators control data collection granularity through the `[telemetry]` section. Besides `enabled`, per-component toggles (`trace_transactions`, `trace_consensus`, `trace_rpc`, `trace_peer` — the last -often disabled due to high volume) select which spans are emitted, and -redaction flags (`redact_account` to hash account addresses, `redact_peer_address` -to remove peer IP addresses) control SDK-level redaction before export. - -> **Note**: The `redact_account` configuration in `xrpld.cfg` controls SDK-level redaction before export, while collector-level filtering (see [Collector-Level Data Protection](#collector-level-data-protection) above) provides an additional defense-in-depth layer. Both can operate independently. +often disabled due to high volume) select which spans are emitted. Account +address hashing is not configurable: addresses are hashed unconditionally by +the SDK helper described above, with collector-level hashing as a second +layer. > **Key Principle**: Telemetry collects **operational metadata** (timing, counts, hashes) — never **sensitive content** (keys, balances, amounts, raw payloads). @@ -646,7 +666,7 @@ flowchart TB - **xrpld Process (dark gray)**: The single xrpld node running all three observability frameworks side by side. Each framework operates independently with no interference. - **PerfLog to perf.log**: PerfLog writes JSON-formatted event logs to a local file. Grafana can ingest these via Loki or a file-based datasource. - **Beast Insight to StatsD Server**: Insight sends aggregated metrics (counters, gauges) over UDP to a StatsD server. Grafana reads from StatsD-compatible backends like Graphite or Prometheus (via StatsD exporter). -- **OpenTelemetry to OTLP Collector**: OTel exports spans over OTLP/gRPC to a Collector, which then forwards to a trace backend (Tempo). +- **OpenTelemetry to OTLP Collector**: OTel exports spans over OTLP/HTTP to a Collector, which then forwards to a trace backend (Tempo). (OTLP/gRPC is future work — §2.2.2.) - **Grafana (red, unified UI)**: All three data streams converge in Grafana, enabling operators to correlate logs, metrics, and traces in a single dashboard. **Phase 7 target state**: Beast Insight routes to `OTelCollector` (new `Collector` implementation) which exports via OTLP/HTTP to the same collector endpoint as traces. StatsD UDP path becomes a deprecated fallback (`[insight] server=statsd`). See [06-implementation-phases.md §6.8](./06-implementation-phases.md) and [Phase7_taskList.md](./Phase7_taskList.md) for details. diff --git a/OpenTelemetryPlan/03-implementation-strategy.md b/OpenTelemetryPlan/03-implementation-strategy.md index eecd144617..922d3d4999 100644 --- a/OpenTelemetryPlan/03-implementation-strategy.md +++ b/OpenTelemetryPlan/03-implementation-strategy.md @@ -180,14 +180,14 @@ pie showData | TracerProvider singleton | ~64 KB | At startup | | BatchSpanProcessor (circular buffer) | ~16 KB | At startup | | BatchSpanProcessor (worker thread) | ~8 MB | At startup | -| OTLP exporter (gRPC channel init) | ~256 KB | At startup | +| OTLP/HTTP exporter (client init) | ~64 KB | At startup | | Propagator registry | ~8 KB | At startup | -| **Total static** | **~8.3 MB** | | +| **Total static** | **~8.1 MB** | | > **Why higher than earlier estimate**: The BatchSpanProcessor's circular buffer itself is only ~16 KB > (2049 x 8-byte `AtomicUniquePtr` entries), but it spawns a dedicated worker thread whose default -> stack size on Linux is ~8 MB. The OTLP gRPC exporter allocates memory for channel stubs and TLS -> initialization. The worker thread stack dominates the static footprint. +> stack size on Linux is ~8 MB. The OTLP/HTTP exporter allocates a small client and TLS +> initialization buffer. The worker thread stack dominates the static footprint. ### 3.5.2 Dynamic Memory diff --git a/OpenTelemetryPlan/05-configuration-reference.md b/OpenTelemetryPlan/05-configuration-reference.md index a48e410b49..04b51d45a6 100644 --- a/OpenTelemetryPlan/05-configuration-reference.md +++ b/OpenTelemetryPlan/05-configuration-reference.md @@ -21,8 +21,8 @@ The authoritative `[telemetry]` example lives in `cfg/xrpld-example.cfg`. Teleme | `endpoint` | string | `http://localhost:4318/v1/traces` | OTLP/HTTP collector endpoint | | `use_tls` | bool | `false` | Enable TLS for exporter connection | | `tls_ca_cert` | string | `""` | Path to CA certificate file | -| `tls_client_cert` | string | `""` | Path to node's client certificate (PEM) for mutual TLS; empty = one-way TLS | -| `tls_client_key` | string | `""` | Path to private key (PEM) for `tls_client_cert`; required when it is set | +| `tls_client_cert` | string | `""` | Path to node's client certificate (PEM) for mutual TLS; requires `use_tls=1`; empty = one-way TLS | +| `tls_client_key` | string | `""` | Path to private key (PEM) for `tls_client_cert`; requires `use_tls=1`; required when the cert is set | | `batch_size` | uint | `512` | Spans per export batch | | `batch_delay_ms` | uint | `5000` | Max delay before sending batch (ms) | | `max_queue_size` | uint | `2048` | Maximum queued spans | @@ -168,10 +168,10 @@ flowchart TB **Reading the diagram:** -- **Configuration Sources**: `xrpld.cfg` provides runtime settings (endpoint, sampling) while the CMake flag controls whether telemetry is compiled in at all. +- **Configuration Sources**: `xrpld.cfg` provides runtime settings (endpoint, per-component trace toggles) while the CMake flag controls whether telemetry is compiled in at all. Head sampling is fixed at 1.0 and is not a config option; volume reduction happens via tail sampling in the collector. - **Initialization**: `setupTelemetry()` parses config values, then `makeTelemetry()` constructs the provider, processor, and exporter objects. - **Runtime Components**: The `TracerProvider` creates spans, the `BatchProcessor` buffers them, and the `OTLP Exporter` serializes and sends them over the wire. -- **OTLP arrow to Collector**: Trace data leaves the xrpld process via OTLP (gRPC or HTTP) and enters the external Collector pipeline. +- **OTLP arrow to Collector**: Trace data leaves the xrpld process via OTLP/HTTP and enters the external Collector pipeline. (OTLP/gRPC is future work — see design decisions §2.2.2.) - **Collector Pipeline**: `Receivers` ingest OTLP data, `Processors` apply sampling/filtering/enrichment, and `Exporters` forward traces to storage backends (Tempo, etc.). --- diff --git a/OpenTelemetryPlan/06-implementation-phases.md b/OpenTelemetryPlan/06-implementation-phases.md index 57c4748024..da0d24b6de 100644 --- a/OpenTelemetryPlan/06-implementation-phases.md +++ b/OpenTelemetryPlan/06-implementation-phases.md @@ -296,9 +296,11 @@ See [Phase4_taskList.md](./Phase4_taskList.md) for full task details. **Objective**: Wire `TraceContextPropagator` for P2P messages (proposals, validations) to enable true distributed tracing between nodes. -**Status**: Design documented, NOT implemented. Protobuf fields (field 1001) -and `TraceContextPropagator` free functions exist. Wiring deferred until Phase 4a is -validated in a multi-node environment. +**Status**: Partially implemented. Send-side injection (proposals and +validations) and receive-side extraction (`consensus.{proposal,validation}. +receive` spans parented on the sender's context) are wired in Phase 4a. +Remaining Phase 4b work: relay spans in `share(RCLCxPeerPos)` and multi-node +validation of the propagation path. **Prerequisites**: Phase 4a complete and validated. diff --git a/OpenTelemetryPlan/07-observability-backends.md b/OpenTelemetryPlan/07-observability-backends.md index cb17d2817a..4ebb6028fd 100644 --- a/OpenTelemetryPlan/07-observability-backends.md +++ b/OpenTelemetryPlan/07-observability-backends.md @@ -237,7 +237,7 @@ A Tempo-backed dashboard (uid `xrpld-consensus-health`) with four panels, all dr - **Proposers per Round** (stat): average of the `span.proposers` attribute on `consensus.round` spans. - **Recent Slow Rounds (>5s)** (table): `consensus.round` spans filtered to `duration > 5s`. -The underlying TraceQL queries are listed in section 7.7.3 and used throughout this doc. +Each panel's TraceQL query is described inline in its bullet above. ### 7.6.2 Node Overview Dashboard @@ -261,9 +261,9 @@ Grafana provisions three TraceQL-based alert rules (group `xrpld-tracing-alerts` - **RPC Error Rate Spike** (critical, `for: 2m`): fires when the error rate across `rpc.command.*` spans exceeds 5%. Error _rate_ is a ratio, so it must divide the error-span rate by the total-span rate — a single TraceQL `rate()` returns spans/second, not a percentage, and would fire on traffic volume alone. This uses span metrics emitted by the collector's `spanmetrics` connector (Prometheus datasource), not a TraceQL query: ``` - sum(rate(calls_total{service_name="xrpld", span_name=~"rpc.command.*", status_code="STATUS_CODE_ERROR"}[5m])) + sum(rate(traces_spanmetrics_calls_total{service_name="xrpld", span_name=~"rpc.command.*", status_code="STATUS_CODE_ERROR"}[5m])) / - sum(rate(calls_total{service_name="xrpld", span_name=~"rpc.command.*"}[5m])) + sum(rate(traces_spanmetrics_calls_total{service_name="xrpld", span_name=~"rpc.command.*"}[5m])) > 0.05 ``` diff --git a/OpenTelemetryPlan/08-appendix.md b/OpenTelemetryPlan/08-appendix.md index 07144805a3..53d7b01e42 100644 --- a/OpenTelemetryPlan/08-appendix.md +++ b/OpenTelemetryPlan/08-appendix.md @@ -211,7 +211,6 @@ flowchart TB | [Phase9_taskList.md](./Phase9_taskList.md) | Internal metric instrumentation gap fill (future) | | [Phase10_taskList.md](./Phase10_taskList.md) | Synthetic workload generation & validation (future) | | [Phase11_taskList.md](./Phase11_taskList.md) | Third-party data collection pipelines (future) | -| [presentation.md](./presentation.md) | Presentation slides for OpenTelemetry plan overview | > **Note**: Phases 1 and 6 do not have separate task list files. Phase 1 tasks are documented in [06-implementation-phases.md §6.2](./06-implementation-phases.md). Phase 6 tasks are documented in [06-implementation-phases.md §6.7](./06-implementation-phases.md). diff --git a/OpenTelemetryPlan/OpenTelemetryPlan.md b/OpenTelemetryPlan/OpenTelemetryPlan.md index 8b794150cc..0d0100bb8f 100644 --- a/OpenTelemetryPlan/OpenTelemetryPlan.md +++ b/OpenTelemetryPlan/OpenTelemetryPlan.md @@ -1,4 +1,4 @@ -# [OpenTelemetry](00-tracing-fundamentals.md) Distributed Tracing Implementation Plan for xrpld (xrpld) +# [OpenTelemetry](00-tracing-fundamentals.md) Distributed Tracing Implementation Plan for xrpld ## Executive Summary @@ -134,7 +134,7 @@ Key trace points span across transaction submission via RPC, peer-to-peer messag > **OTLP** = OpenTelemetry Protocol | **CNCF** = Cloud Native Computing Foundation -The OpenTelemetry C++ SDK is selected for its CNCF backing, active development, and native performance characteristics. Traces are exported via OTLP/gRPC (primary) or OTLP/HTTP (fallback) to an OpenTelemetry Collector, which provides flexible routing and sampling. +The OpenTelemetry C++ SDK is selected for its CNCF backing, active development, and native performance characteristics. Traces are exported via OTLP/HTTP to an OpenTelemetry Collector, which provides flexible routing and sampling. OTLP/gRPC is planned future work (see design decisions §2.2.2). Span naming follows a hierarchical `.` convention (e.g., `rpc.submit`, `tx.relay`, `consensus.round`). Context propagation uses W3C Trace Context headers for HTTP and embedded Protocol Buffer fields for P2P messages. The implementation coexists with existing PerfLog and Insight observability systems through correlation IDs. @@ -158,7 +158,7 @@ Performance optimization strategies include head sampling fixed at 100% (intenti > **OTLP** = OpenTelemetry Protocol | **APM** = Application Performance Monitoring -Configuration is handled through the `[telemetry]` section in `xrpld.cfg` with options for enabling/disabling, exporter selection, endpoint configuration, sampling ratios, and component-level filtering. CMake integration includes a `XRPL_ENABLE_TELEMETRY` option for compile-time control. +Configuration is handled through the `[telemetry]` section in `xrpld.cfg` with options for enabling/disabling, exporter selection, endpoint configuration, and component-level filtering. Head sampling is fixed at 1.0 (not operator-configurable); volume reduction is done by tail sampling in the collector. CMake integration includes a `XRPL_ENABLE_TELEMETRY` option for compile-time control. OpenTelemetry Collector configurations are provided for development and production (with tail-based sampling, Tempo, and Elastic APM). Docker Compose examples enable quick local development environment setup. diff --git a/OpenTelemetryPlan/Phase2_taskList.md b/OpenTelemetryPlan/Phase2_taskList.md index 1f572ecfa9..89d9ddd6e0 100644 --- a/OpenTelemetryPlan/Phase2_taskList.md +++ b/OpenTelemetryPlan/Phase2_taskList.md @@ -169,11 +169,12 @@ Node health (`amendment_blocked`, `server_state`) is not part of the telemetry s | `rpc.process` | `batch_size` | int64 | `params.size()` (only when batch) | | `rpc.ws_message` | `command` | string | `jv[command]` or `jv[method]` | | `rpc.command.*` | `load_type` | string | `context.loadType.label()` | -| `pathfind.compute` | `pathfind_dest_amount` | string | `saDstAmount_.getFullText()` | | `pathfind.compute` | `pathfind_dest_currency` | string | `to_string(saDstAmount_.asset())` | | `pathfind.discover` | `pathfind_num_source_assets` | int64 | `sourceAssets.size()` | -**New attr keys**: `RpcSpanNames.h` (`isBatch`, `batchSize`, `loadType`), `PathFindSpanNames.h` (`destAmount`, `destCurrency`, `numSourceAssets`). +_Note: `pathfind_dest_amount` was removed — the destination amount is a financial value excluded by the privacy policy (design §2.4.4)._ + +**New attr keys**: `RpcSpanNames.h` (`isBatch`, `batchSize`, `loadType`), `PathFindSpanNames.h` (`destCurrency`, `numSourceAssets`). **Modified files**: diff --git a/OpenTelemetryPlan/Phase3_taskList.md b/OpenTelemetryPlan/Phase3_taskList.md index ab7dd89022..82dbb4ffb3 100644 --- a/OpenTelemetryPlan/Phase3_taskList.md +++ b/OpenTelemetryPlan/Phase3_taskList.md @@ -166,7 +166,8 @@ ## Task 3.6: Context Propagation in Transaction Relay -**Status**: COMPLETE +**Status**: COMPLETE (transaction relay). Consensus proposal/validation +propagation is deferred to Phase 4 — see "Planned (Phase 4)" below. **Objective**: Ensure trace context flows correctly when transactions are relayed between peers, creating linked spans across nodes. @@ -178,17 +179,6 @@ wired in PeerImp) extracts the parent span_id and creates the tx.receive span as a child of the sender's tx.process span. -- **Proposal send/receive**: `RCLConsensus::Adaptor::propose()` injects the - current thread's active span context into the `TMProposeSet` protobuf via - `telemetry::injectToProtobuf()`. PeerImp creates a - `consensus.proposal.receive` span that extracts the sender's trace context - as parent (via `ConsensusReceiveTracing.h`). - -- **Validation send/receive**: `RCLConsensus::Adaptor::validate()` injects - the current thread's active span context into the `TMValidation` protobuf. - PeerImp creates a `consensus.validation.receive` span that extracts the - sender's trace context as parent. - - **Edge cases**: Missing trace context (older peers) degrades gracefully to standalone spans. Invalid/corrupted context is treated as absent. Trace flags are propagated and respected. @@ -200,20 +190,36 @@ - `PropagationHelpers.h` — `injectSpanContext(SpanGuard&, proto)` bridge between SpanGuard and protobuf TraceContext. - `TraceContextPropagator.h` — `injectToProtobuf(ctx, proto)` for - same-thread injection via OTel RuntimeContext (used in propose/validate). -- `ConsensusReceiveTracing.h` — `proposalReceiveSpan()` and - `validationReceiveSpan()` helper functions that create receive spans with - optional parent context extraction from incoming protobuf messages. + same-thread injection via OTel RuntimeContext. **Key modified files**: - `src/xrpld/app/misc/NetworkOPs.cpp` — tx relay injection -- `src/xrpld/app/consensus/RCLConsensus.cpp` — proposal/validation send injection -- `src/xrpld/overlay/detail/PeerImp.cpp` — proposal/validation receive spans - `include/xrpl/telemetry/SpanGuard.h` — `TraceBytes` struct, `getTraceBytes()` - `src/libxrpl/telemetry/SpanGuard.cpp` — `getTraceBytes()` implementation - `src/xrpld/telemetry/PropagationHelpers.h` — inject helpers (new file) -- `src/xrpld/telemetry/ConsensusReceiveTracing.h` — receive span helpers (new file) + +**Planned (Phase 4 — not in this PR)**: + +The consensus proposal/validation propagation below is Phase 4 scope and is +not implemented on this branch. It is listed here only to record the intended +design. + +- **Proposal send/receive**: `RCLConsensus::Adaptor::propose()` injects the + current thread's active span context into the `TMProposeSet` protobuf via + `telemetry::injectToProtobuf()`. PeerImp creates a + `consensus.proposal.receive` span that extracts the sender's trace context + as parent (via `ConsensusReceiveTracing.h`). + +- **Validation send/receive**: `RCLConsensus::Adaptor::validate()` injects + the current thread's active span context into the `TMValidation` protobuf. + PeerImp creates a `consensus.validation.receive` span that extracts the + sender's trace context as parent. + +- Planned files: `src/xrpld/app/consensus/RCLConsensus.cpp` (send injection), + `src/xrpld/overlay/detail/PeerImp.cpp` (receive spans), + `src/xrpld/telemetry/ConsensusReceiveTracing.h` (receive span helpers, + new file). **Reference**: @@ -396,7 +402,7 @@ This gives the best of both worlds: guaranteed cross-node correlation via determ - `src/xrpld/overlay/detail/PeerImp.cpp` — restructured span creation - `src/xrpld/app/misc/NetworkOPs.cpp` — deterministic context for tx.process -- `src/xrpld/app/misc/TxSpanNames.h` — new `traceStrategy` attribute constant +- `src/xrpld/telemetry/TxSpanNames.h` — new `traceStrategy` attribute constant - New or shared utility for `createDeterministicTxContext()` (location TBD: could be a shared header like `include/xrpl/telemetry/DeterministicContext.h`, or file-local if only used in two places) @@ -431,7 +437,7 @@ This gives the best of both worlds: guaranteed cross-node correlation via determ - `txSpan(category, group, name, txHash, parentCtx)` — child span (deterministic trace_id combined with protobuf-extracted parent span_id for relay ordering). -- **`TxTracing.h` helper functions** (`src/xrpld/overlay/detail/TxTracing.h`): +- **`TxTracing.h` helper functions** (`src/xrpld/telemetry/TxTracing.h`): File-local helpers that wrap `SpanGuard::txSpan()` for the two main PeerImp call sites: - `txReceiveSpan(txHash, parentCtx)` — creates `tx.receive` span with @@ -495,7 +501,7 @@ This gives the best of both worlds: guaranteed cross-node correlation via determ **Modified files**: -- `src/xrpld/app/misc/TxSpanNames.h` +- `src/xrpld/telemetry/TxSpanNames.h` - `src/xrpld/app/misc/detail/TxQSpanNames.h` - `src/xrpld/app/misc/NetworkOPs.cpp` - `src/xrpld/overlay/detail/PeerImp.cpp` diff --git a/OpenTelemetryPlan/Phase4_taskList.md b/OpenTelemetryPlan/Phase4_taskList.md index 9973331651..028a7a7e85 100644 --- a/OpenTelemetryPlan/Phase4_taskList.md +++ b/OpenTelemetryPlan/Phase4_taskList.md @@ -95,10 +95,16 @@ - Creates `consensus.proposal.receive` span - Sets `trusted` attribute (bool) -**Not implemented** (deferred to Phase 4b — cross-node propagation): +**Done here** (cross-node propagation, send + receive): -- `consensus.proposal.relay` span in `share(RCLCxPeerPos)` — requires trace context injection -- Trace context injection/extraction for `TMProposeSet::trace_context` +- Trace context injection for `TMProposeSet::trace_context` in `propose()` +- Receive-side extraction in `PeerImp::onMessage(TMProposeSet)` via + `telemetry::proposalReceiveSpan()` (parents the receive span on the + sender's context when a valid `trace_context` is present) + +**Not implemented** (deferred to Phase 4b): + +- `consensus.proposal.relay` span in `share(RCLCxPeerPos)` **Key modified files**: diff --git a/OpenTelemetryPlan/Phase5_IntegrationTest_taskList.md b/OpenTelemetryPlan/Phase5_IntegrationTest_taskList.md index 6f9df2f969..2bf33a390c 100644 --- a/OpenTelemetryPlan/Phase5_IntegrationTest_taskList.md +++ b/OpenTelemetryPlan/Phase5_IntegrationTest_taskList.md @@ -107,13 +107,13 @@ Tempo/Prometheus. - `consensus.validation.send` (Adaptor::validate) - Verify attributes: - `xrpl.consensus.mode` on `consensus.ledger_close` - - `xrpl.consensus.proposers` on `consensus.accept` + - `proposers` on `consensus.accept` - `xrpl.consensus.ledger.seq` on `consensus.validation.send` **Verification**: - [ ] Tempo shows `consensus.ledger_close` traces with `xrpl.consensus.mode` -- [ ] Tempo shows `consensus.accept` traces with `xrpl.consensus.proposers` +- [ ] Tempo shows `consensus.accept` traces with `proposers` - [ ] Tempo shows `consensus.proposal.send` traces - [ ] Tempo shows `consensus.validation.send` traces @@ -147,7 +147,7 @@ Tempo/Prometheus. - Prerequisites section - Single-node standalone test (quick verification) - 6-node consensus test (full verification) - - Expected span catalog (all 12 span names with attributes) + - Expected span catalog (all 11 span names with attributes) - Verification queries (Tempo API, Prometheus API) - Troubleshooting guide @@ -156,7 +156,7 @@ Tempo/Prometheus. **Verification**: - [ ] Document covers both single-node and multi-node testing -- [ ] All 12 span names documented with source file and attributes +- [ ] All 11 span names documented with source file and attributes - [ ] Troubleshooting section covers common failure modes --- diff --git a/OpenTelemetryPlan/presentation.md b/OpenTelemetryPlan/presentation.md deleted file mode 100644 index c1505eff59..0000000000 --- a/OpenTelemetryPlan/presentation.md +++ /dev/null @@ -1,374 +0,0 @@ -# OpenTelemetry Observability for xrpld - -> Status: Phases 1-8 shipped. Traces, metrics, logs all live via OTel. - ---- - -## Slide 1: Introduction - -> **CNCF** = Cloud Native Computing Foundation | **OTel** = OpenTelemetry - -### What is OpenTelemetry? - -CNCF-backed, vendor-neutral framework for **traces, metrics, and logs** with a single SDK and wire protocol (OTLP). - -### Why OTel for xrpld? - -- **End-to-end TX visibility** — submission → consensus → ledger inclusion -- **Cross-node correlation** — shared `trace_id` stitches hops without a central coordinator -- **Consensus round analysis** — phase timing across validators -- **Incident debugging** — correlated traces, metrics, logs for one query - -```mermaid -flowchart LR - A["Node A
tx.receive
trace_id: abc123"] --> B["Node B
tx.relay
trace_id: abc123"] --> C["Node C
tx.validate
trace_id: abc123"] --> D["Node D
ledger.apply
trace_id: abc123"] - - style A fill:#1565c0,stroke:#0d47a1,color:#fff - style B fill:#2e7d32,stroke:#1b5e20,color:#fff - style C fill:#2e7d32,stroke:#1b5e20,color:#fff - style D fill:#e65100,stroke:#bf360c,color:#fff -``` - -> One trace, four nodes, full lifecycle. - ---- - -## Slide 2: Old Stack vs New OTel Stack - -### Side-by-Side - -| Aspect | Before (StatsD + Debug Logs) | After (OTel: Traces + Metrics + Logs) | -| ------------------------- | --------------------------------------------------------------------------------- | ------------------------------------------------------------ | -| **Metrics** | Beast Insight → StatsD UDP → Graphite | `MetricsRegistry` → OTLP/HTTP → Prometheus | -| **Metric inventory** | **~250 metric series** at runtime (28 registrations × overlay traffic categories) | **23 native instruments** × dimensions + RED via spanmetrics | -| **Logs** | `beast::Journal` → `debug.log` (grep / tail) | Journal → filelog tail → Loki (structured, queryable) | -| **Traces** | None | Telemetry SDK → OTLP → Tempo (cross-node) | -| **Correlation** | Timestamp + grep across files | Shared `trace_id` across all 3 signals | -| **Format** | Counter/gauge names; free-form log lines | OTLP protobuf; structured records | -| **Backend choice** | Locked to StatsD daemon + log files | Vendor-neutral via Collector exporters | -| **Cross-node view** | ❌ Not possible | ✅ Native via trace context propagation | -| **Histogram p50/p95/p99** | ❌ Counters/gauges only | ✅ Native histograms + spanmetrics | - -### Legacy StatsD Metric Series (~250 total) - -| Category | Series | Notes | -| --------------------------- | -------- | ----------------------------------------------------------------------------------- | -| **Overlay traffic gauges** | ~224 | 56 `TrafficCount::category` enum × 4 gauges (`Bytes_{In,Out}`, `Messages_{In,Out}`) | -| **Peer Finder** | 2 | `Active_{In,Out}bound_Peers` | -| **State Accounting** | 10 | `{Disconnected,Connected,Syncing,Tracking,Full}_{duration,transitions}` | -| **Ledger** | 4 | `Validated/Published_Ledger_Age`, `mismatch`, `ledger_fetches` | -| **RPC / Pathfinding** | 5 | `requests`, `size`, `time`, `pathfind_{fast,full}` | -| **JobQueue / IO / Disconn** | 3 | `job_count`, `ios_latency`, `Peer_Disconnects` | -| **Total** | **~248** | 28 `make_*` call sites; series count balloons via overlay-category fan-out | - -### Use Case Matrix - -| Scenario | StatsD | Debug Logs | OTel Traces | OTel Metrics | OTel Logs | -| ---------------------------------- | ------ | ---------- | ----------- | ------------ | --------- | -| "TXs per second?" | ✅ | ❌ | ❌ | ✅ | ❌ | -| "Why was this specific TX slow?" | ❌ | ⚠️ | ✅ | ❌ | ⚠️ | -| "Which node delayed consensus?" | ❌ | ❌ | ✅ | ❌ | ❌ | -| "TX journey across 5 nodes" | ❌ | ❌ | ✅ | ❌ | ❌ | -| "Validator error at 14:02" | ❌ | ✅ | ⚠️ | ❌ | ✅ | -| "Reproduce rare assertion / crash" | ❌ | ✅ | ❌ | ❌ | ✅ | -| "p99 RPC latency by method" | ⚠️ | ❌ | ⚠️ | ✅ | ❌ | - -> Old stack: 2 signals, no correlation, single node. New stack: 3 signals, `trace_id` everywhere, cross-node native. - ---- - -## Slide 3: OTel vs Open-Source Alternatives - -| Feature | OpenTelemetry | Jaeger | Zipkin | SkyWalking | Pinpoint | Prometheus | -| ------------------- | --------------- | ------------- | --------------- | ---------- | ---------- | ---------- | -| **Tracing** | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | -| **Metrics** | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | -| **Logs** | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | -| **C++ SDK** | ✅ Official | ⚠️ Deprecated | ⚠️ Unmaintained | ❌ | ❌ | ✅ | -| **Vendor neutral** | ✅ Primary goal | ❌ | ❌ | ❌ | ❌ | ❌ | -| **Instrumentation** | Manual + Auto | Manual | Manual | Auto-first | Auto-first | Manual | -| **Backend** | Any (exporters) | Self | Self | Self | Self | Self | -| **CNCF Status** | Incubating | Graduated | — | Incubating | — | Graduated | - -> Only actively maintained, full-signal C++ option. Backend-agnostic — Tempo/Prometheus/Loki/Elastic/commercial all work without code change. - ---- - -## Slide 4: Architecture (Current) - -> **OTLP** = OpenTelemetry Protocol over HTTP/gRPC - -```mermaid -flowchart TB - subgraph xrpld["xrpld Node"] - direction TB - Surfaces["RPC · TX · Consensus · Peer · Ledger · Job"] - SDK["Telemetry SDK + MetricsRegistry"] - Journal["beast::Journal → debug.log
(trace_id/span_id injected)"] - Surfaces --> SDK - Surfaces --> Journal - end - - SDK -->|"OTLP/HTTP :4318
traces + metrics"| Collector["OTel Collector"] - Journal -->|"filelog tail"| Collector - - Collector --> Tempo["Tempo
(traces)"] - Collector --> Prom["Prometheus
(metrics)"] - Collector --> Loki["Loki
(logs)"] - - Tempo --> Grafana["Grafana
(15 dashboards)"] - Prom --> Grafana - Loki --> Grafana - - style xrpld fill:#424242,stroke:#212121,color:#fff - style SDK fill:#2e7d32,stroke:#1b5e20,color:#fff - style Journal fill:#1565c0,stroke:#0d47a1,color:#fff - style Collector fill:#e65100,stroke:#bf360c,color:#fff - style Grafana fill:#4a148c,stroke:#2e0d57,color:#fff -``` - -| Component | Role | -| ---------------------- | --------------------------------------------------- | -| Telemetry SDK | Span creation, trace context, OTLP traces export | -| MetricsRegistry | RPC/job/peer/consensus counters, gauges, histograms | -| beast::Journal filelog | `debug.log` tailed by Collector, parsed → Loki | -| OTel Collector | Receive OTLP + filelog; route to Tempo/Prom/Loki | -| Spanmetrics connector | Derives RED metrics from spans (Prometheus) | - ---- - -## Slide 5: Signal Coverage - -| Surface | Traces (Spans) | Metrics (OTLP) | Logs (Journal Partition) | -| ------------------ | --------------------------------------------------------------- | ---------------------------------------------- | ------------------------------ | -| **RPC** | `rpc.request` + handler spans | request count, latency p50/p95/p99, error rate | `RPC*` | -| **Transactions** | `tx.receive`, `tx.validate`, `tx.relay`, `tx.apply` | TX/sec by result, fee escalation gauges | `TxQ`, `LedgerMaster` | -| **Consensus** | `consensus.round`, `proposal.send/recv`, `validation.send/recv` | round duration, phase histograms, mode gauge | `Consensus`, `LedgerConsensus` | -| **Peer / Overlay** | `peer.send`, `peer.receive` per message type | peer count, bytes/sec by msg type, suppression | `Overlay`, `PeerImp` | -| **Ledger** | `ledger.close`, `ledger.apply` | close time, TX count, ledger index gauge | `LedgerMaster` | -| **Job Queue** | (sampled per type) | queue depth, queue/run duration histograms | `JobQueue` | - -> ~30 distinct span kinds, ~80 metric series, structured logs from 50+ partitions. - ---- - -## Slide 6: Context Propagation - -```mermaid -sequenceDiagram - participant Client - participant NodeA as Node A - participant NodeB as Node B - - Client->>NodeA: Submit TX (no context) - Note over NodeA: Create trace_id: abc123
span: tx.receive - NodeA->>NodeB: Relay TX (TraceContext field, ~29B) - Note over NodeB: Link trace_id: abc123
span: tx.relay (parent: A) -``` - -| Carrier | Mechanism | -| --------------------- | ------------------------------------------ | -| HTTP / WebSocket RPC | W3C `traceparent` header | -| P2P protobuf | `TraceContext` extension field per message | -| Internal job dispatch | Thread-local context + `SpanGuard` | - -| Field | Size | Description | -| ------------- | --------- | ------------------------------------- | -| `trace_id` | 16 bytes | Trace correlation key | -| `span_id` | 8 bytes | Parent span on receiver | -| `trace_flags` | 1 byte | Sampling decision | -| `trace_state` | 0-4 bytes | Optional vendor data | -| **Total** | **~29 B** | Per traced P2P message (~1-6% of msg) | - ---- - -## Slide 7: Performance Overhead - -| Metric | Overhead | Driver | -| ----------------- | ---------- | --------------------------------------------------- | -| **CPU** | 1-3% | ~4 μs/TX span work (~2% at 25 TPS baseline) | -| **Memory** | ~10 MB | SDK statics + worker stack + 2048-span export queue | -| **Network** | 10-50 KB/s | OTLP export + 29 B P2P context per traced msg | -| **Latency (p99)** | <2% | TX path dominates; RPC and consensus negligible | - -### Kill Switches - -1. `enabled=0` in `xrpld.cfg` → instant disable, no restart -2. Build with `XRPL_ENABLE_TELEMETRY=OFF` → zero overhead (no-op stubs) -3. Reduce `sampling_ratio` → linear export reduction - -> Derivations and per-component cost tables: see [03-implementation-strategy.md §3.5.4](./03-implementation-strategy.md#354-performance-data-sources). - ---- - -## Slide 8: Sampling — Head vs Tail - -| | Head Sampling | Tail Sampling | -| ------------------------ | --------------------------------- | -------------------------------------- | -| **Where** | Inside xrpld (SDK) | OTel Collector (external) | -| **Decision time** | Trace start (random coin flip) | Trace end (after all spans buffered) | -| **Knows trace content?** | No | Yes — error, latency, span kind | -| **xrpld overhead** | Lowest (drop = no-op) | Higher (export 100%) | -| **Captures all errors?** | No | **Yes** (status_code policy) | -| **Captures slow ops?** | No | **Yes** (latency policy) | -| **Config** | `xrpld.cfg`: `sampling_ratio=0.1` | `tail_sampling` processor in collector | -| **Best for** | Steady-state high volume | Anomaly + error retention | - -### Recommended Layered Strategy - -```mermaid -flowchart LR - xrpld["xrpld
sampling_ratio=1.0
(export all)"] -->|"100%"| col["Collector
tail_sampling:
errors + slow + 10% random"] - col -->|"~15-20% kept"| tempo["Tempo storage"] - - style xrpld fill:#424242,stroke:#212121,color:#fff - style col fill:#1565c0,stroke:#0d47a1,color:#fff - style tempo fill:#2e7d32,stroke:#1b5e20,color:#fff -``` - -> If Collector resource pressure: drop `sampling_ratio` to 0.5 — still enough trace volume for tail decisions. - ---- - -## Slide 9: Data Collection & Privacy - -### Collected (operational metadata) - -| Category | Attributes | -| ----------- | -------------------------------------------------------------------- | -| Transaction | `tx.hash`, `tx.type`, `tx.result`, `tx.fee`, `ledger_index` | -| Consensus | `round`, `phase`, `mode`, `proposers`, `duration_ms` | -| RPC | `command`, `version`, `status`, `duration_ms` | -| Peer | `peer.id` (public key), `latency_ms`, `message.type`, `message.size` | -| Ledger | `ledger.hash`, `ledger.index`, `close_time`, `tx_count` | -| Job | `job.type`, `queue_ms`, `worker` | - -### NOT Collected (hard exclusions) - -> ❌ Private keys · ❌ Account balances · ❌ Transaction amounts · ❌ Raw payloads · ❌ Personal data · ⚙️ IP addresses (configurable) - -### Privacy Mechanisms - -| Mechanism | Description | -| ---------------------- | --------------------------------------------------------- | -| Account hashing | `xrpl.tx.account` hashed at Collector before storage | -| Configurable redaction | Sensitive attributes excluded via Collector config | -| Sampling | 10% default reduces exposure | -| Local control | Operator owns Collector → backend pipeline | -| No raw payloads | Span attributes are metadata only, never message contents | - -> Principle: telemetry records **operational metadata** — never financial or personal content. - ---- - -## Slide 10: Implementation Timeline - -```mermaid -gantt - title OpenTelemetry Rollout - dateFormat YYYY-MM-DD - axisFormat Week %W - - section Done - Phase 1 Core Infra :done, p1, 2024-01-01, 2w - Phase 2 RPC Tracing :done, p2, after p1, 2w - Phase 3 TX Tracing :done, p3, after p2, 2w - Phase 4 Consensus :done, p4, after p3, 2w - Phase 5 Docs/Deploy :done, p5, after p4, 1w - Phase 6 StatsD Bridge :done, p6, after p5, 1w - Phase 7 Native OTel Metrics :done, p7, after p6, 2w - Phase 8 Log-Trace Correlation :done, p8, after p7, 1w - Phase 9 Metric Gap Fill :active, p9, after p8, 2w - - section Future - Phase 10 Workload Validation :p10, after p9, 2w - Phase 11 3rd-Party Pipelines :p11, after p10, 3w -``` - -| Phase | Focus | Status | -| ----- | ------------------------------------------- | ------- | -| 1 | SDK integration, Telemetry, Config | ✅ Done | -| 2 | RPC handler spans, HTTP context | ✅ Done | -| 3 | TX spans, P2P protobuf context | ✅ Done | -| 4 | Consensus rounds, proposal/validation | ✅ Done | -| 5 | Runbook, dashboards, deployment | ✅ Done | -| 6 | StatsD bridge (interim) | ✅ Done | -| 7 | Native OTel metrics (replace Beast Insight) | ✅ Done | -| 8 | Log-trace correlation (Loki) | ✅ Done | -| 9 | Internal metric gap fill | ✅ Done | - ---- - -## Slide 11: Current State — What Shipped - -### By Signal - -| Signal | Backend | Status | Notes | -| ----------- | ---------- | ------ | -------------------------------------------------------- | -| **Traces** | Tempo | ✅ | All 6 surfaces instrumented; cross-node propagation live | -| **Metrics** | Prometheus | ✅ | Native OTLP; Beast Insight retired | -| **Logs** | Loki | ✅ | filelog tailing `debug.log`; `trace_id` injected | - -### By Surface - -| Surface | Spans Live | Metrics Live | Notes | -| -------------- | ---------- | ------------ | --------------------------------------------------- | -| RPC | ✅ | ✅ | Handler + pathfinding + TxQ | -| Transactions | ✅ | ✅ | Receive, validate, relay, apply | -| Consensus | ✅ | ✅ | Round + proposal/validation send+receive (Phase 4a) | -| Peer / Overlay | ✅ | ✅ | Per-msg-type send/receive | -| Ledger | ✅ | ✅ | Close + apply | -| Job Queue | ✅ | ✅ | Queue depth + duration histograms | - -### Stack Live - -| Component | Version | -| -------------------------- | ------- | -| OTel Collector (contrib) | 0.121.0 | -| Grafana Tempo | 2.7.2 | -| Grafana Loki | 3.4.2 | -| Prometheus | latest | -| Grafana | 11.5.2 | -| **Dashboards provisioned** | **15** | - ---- - -## Slide 12: Future Phases - -### Phase 10 — Synthetic Workload Validation - -| Aspect | Detail | -| ----------- | ------------------------------------------------------------------ | -| Goal | Drive instrumented surfaces under reproducible load | -| Why | Validate dashboards, catch regressions, measure overhead at scale | -| Deliverable | Workload generator + assertion suite (RPC/TX/peer churn scenarios) | -| Effort | ~2 weeks | - -### Phase 11 — Admin-RPC Receiver (`xrpl_*` metrics) - -| Aspect | Detail | -| ----------- | --------------------------------------------------------------------------------------------------------------------------------------------- | -| Goal | Custom Go OTel Collector receiver polls xrpld admin RPC, emits `xrpl_*` Prometheus metrics | -| Why | Admin-RPC-only data has no native export — every consumer reinvents JSON-RPC polling | -| Scope | `validators` (UNL, listed keys), `feature` (amendments), `peers` (per-peer detail), `amm_info`, `book_offers`, `fee` (detail tiers) | -| Excluded | `server_info` / `get_counts` basics — Phase 9 (#6513) already ships `xrpld_server_info` + 14 gauges/histograms natively from in-process state | -| Deliverable | Go receiver plugin + custom Collector binary + 4 Grafana dashboards (UNL, amendments, AMM, DEX) + Prometheus alerts | -| Effort | ~3 weeks | - -```mermaid -flowchart LR - rpc["xrpld admin RPC
(validators, feature, peers,
amm_info, book_offers, fee)"] -->|JSON-RPC poll| recv["Custom Go receiver
(in Collector)"] - recv -->|xrpl_* metrics| prom["Prometheus"] - prom --> graf["Grafana dashboards"] - - style rpc fill:#2e7d32,stroke:#1b5e20,color:#fff - style recv fill:#1565c0,stroke:#0d47a1,color:#fff - style prom fill:#e65100,stroke:#bf360c,color:#fff - style graf fill:#6a1b9a,stroke:#4a148c,color:#fff -``` - -> Phase 11 fills the gap above Phase 9 — data only reachable via admin RPC, not via in-process metric callbacks. - ---- - -_End of Presentation_ diff --git a/OpenTelemetryPlan/secure-OTel.md b/OpenTelemetryPlan/secure-OTel.md index c570a06ecf..3d22de5452 100644 --- a/OpenTelemetryPlan/secure-OTel.md +++ b/OpenTelemetryPlan/secure-OTel.md @@ -20,7 +20,7 @@ The most effective way to prevent spoofing from external sources is to treat you server { listen 80; - location { + location / { # Clear out untrusted incoming trace headers proxy_set_header traceparent ""; proxy_set_header tracestate ""; @@ -92,10 +92,11 @@ receivers: grpc: endpoint: 0.0.0.0:4317 tls: + # Setting client_ca_file makes the collector require and verify a + # client cert, rejecting connections without a trusted one. client_ca_file: /certs/client_ca.pem # CA that signs trusted client certs cert_file: /certs/collector.pem key_file: /certs/collector.key - auth_type: require_and_verify_client_cert # Rejects unauthorized clients ``` ### **Approach C: Application Layer Authentication (Basic Auth Extension)** @@ -175,7 +176,7 @@ xrpld has **two distinct attack surfaces**, not one. The original guide conflate │ │ OTLP/gRPC │ │ + mTLS │ ▼ - └───────────────────────────────────────── [require_and_verify_client_cert] + └───────────────────────────────────────── [client_ca_file: verify client cert] OTel Collector (in private subnet, NetPol) ``` @@ -196,15 +197,15 @@ The guide's NGINX header stripping and OTTL stale-span filtering target HTTP gat Evaluated for the across-network deployment shape: -| Approach | Across-network fit | Cost | Verdict | -| ------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------- | ---------------------------------- | -| **A. NetworkPolicy / firewall** | Necessary baseline (don't expose `4317`/`4318` to the internet), but insufficient on its own when traffic genuinely crosses networks — you cannot NetworkPolicy the public internet. | Cheap. | **Defense-in-depth, not primary.** | -| **B. mTLS** | Strongest fit. Every xrpld node holds a client cert; collector verifies with `require_and_verify_client_cert`. Encrypts in transit (raw OTLP over the internet leaks transaction patterns and validator identity). Compromised node = revoke one cert, no shared secret to rotate everywhere. | Cert issuance + rotation pipeline. | **Primary.** | -| **C. Basic Auth** | Worst shape for this topology. Single shared password across all xrpld nodes — one leaked node config compromises the whole fleet. Doesn't encrypt; you'd need TLS underneath anyway, at which point you're 80% of the way to mTLS. | Cheap to set up, expensive to operate (rotation across N operators). | **Skip.** | +| Approach | Across-network fit | Cost | Verdict | +| ------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------- | ---------------------------------- | +| **A. NetworkPolicy / firewall** | Necessary baseline (don't expose `4317`/`4318` to the internet), but insufficient on its own when traffic genuinely crosses networks — you cannot NetworkPolicy the public internet. | Cheap. | **Defense-in-depth, not primary.** | +| **B. mTLS** | Strongest fit. Every xrpld node holds a client cert; the collector verifies it via `client_ca_file` in the receiver's `tls` block. Encrypts in transit (raw OTLP over the internet leaks transaction patterns and validator identity). Compromised node = revoke one cert, no shared secret to rotate everywhere. | Cert issuance + rotation pipeline. | **Primary.** | +| **C. Basic Auth** | Worst shape for this topology. Single shared password across all xrpld nodes — one leaked node config compromises the whole fleet. Doesn't encrypt; you'd need TLS underneath anyway, at which point you're 80% of the way to mTLS. | Cheap to set up, expensive to operate (rotation across N operators). | **Skip.** | ## Decision -**Primary defense:** mTLS (Approach B) on the collector's OTLP receivers, with `auth_type: require_and_verify_client_cert`. +**Primary defense:** mTLS (Approach B) on the collector's OTLP receivers. The collector requires and verifies each client certificate when `client_ca_file` is set in the receiver's `tls` block (there is no `auth_type` field — setting `client_ca_file` is what enforces client-cert verification). **Defense-in-depth:** NetworkPolicy / firewall rules (Approach A) so `4317`/`4318` are never reachable from outside the expected operator subnets even if mTLS were misconfigured. diff --git a/cmake/XrplCore.cmake b/cmake/XrplCore.cmake index f59c67510c..fa54ca7255 100644 --- a/cmake/XrplCore.cmake +++ b/cmake/XrplCore.cmake @@ -210,10 +210,13 @@ target_link_libraries( # When telemetry=ON, links the Conan-provided umbrella target # opentelemetry-cpp::opentelemetry-cpp (individual component targets like # ::api, ::sdk are not available in the Conan package). +# +# Links xrpl.libxrpl.protocol PRIVATELY for sha512Half (digest.h) add_module(xrpl telemetry) target_link_libraries( xrpl.libxrpl.telemetry PUBLIC xrpl.libxrpl.basics xrpl.libxrpl.beast xrpl.libxrpl.config + PRIVATE xrpl.libxrpl.protocol ) if(telemetry) target_link_libraries( diff --git a/docker/telemetry/docker-compose.yml b/docker/telemetry/docker-compose.yml index e93338515e..28ee64336c 100644 --- a/docker/telemetry/docker-compose.yml +++ b/docker/telemetry/docker-compose.yml @@ -78,7 +78,8 @@ services: - xrpld-telemetry prometheus: - image: prom/prometheus:latest + # Pinned to the v2.53 LTS line for reproducible, config-stable runs. + image: prom/prometheus:v2.53.3 ports: - "9090:9090" volumes: diff --git a/docker/telemetry/grafana/provisioning/datasources/tempo.yaml b/docker/telemetry/grafana/provisioning/datasources/tempo.yaml index 313375039d..67749a3bc2 100644 --- a/docker/telemetry/grafana/provisioning/datasources/tempo.yaml +++ b/docker/telemetry/grafana/provisioning/datasources/tempo.yaml @@ -55,7 +55,7 @@ datasources: tag: service.instance.id operator: "=" scope: resource - type: static + type: dynamic # service.version: xrpld build version (e.g., "2.4.0-b1"). # Filter traces from specific software releases. - id: node-version @@ -77,7 +77,7 @@ datasources: tag: xrpl.network.type operator: "=" scope: resource - type: static + type: dynamic # --- Span intrinsic filters --- # name: the span operation name (e.g., "rpc.command.server_info"). # Use to find traces for a specific RPC command or subsystem. @@ -85,21 +85,21 @@ datasources: tag: name operator: "=" scope: intrinsic - type: static + type: dynamic # status: span completion status ("ok", "error", "unset"). # Filter for failed operations to diagnose errors. - id: span-status tag: status operator: "=" scope: intrinsic - type: static + type: dynamic # duration: span wall-clock duration. Use with ">" operator # to find slow operations (e.g., duration > 500ms). - id: span-duration tag: duration operator: ">" scope: intrinsic - type: static + type: dynamic # Phase 2: RPC tracing filters - id: rpc-command tag: command @@ -164,12 +164,12 @@ datasources: tag: close_time_correct operator: "=" scope: span - type: dynamic + type: static - id: consensus-state tag: consensus_state operator: "=" scope: span - type: dynamic + type: static - id: consensus-close-resolution tag: close_resolution_ms operator: "=" diff --git a/docker/telemetry/otel-collector-config.yaml b/docker/telemetry/otel-collector-config.yaml index aa57680d78..e7c989bf8a 100644 --- a/docker/telemetry/otel-collector-config.yaml +++ b/docker/telemetry/otel-collector-config.yaml @@ -103,6 +103,15 @@ processors: action: delete - key: telemetry.sdk.version action: delete + # Defense-in-depth: hash path-finding account attributes. The xrpld SDK + # already hashes these before export, but a node that emitted raw values + # is caught here so raw addresses never reach the backend. + attributes/hash: + actions: + - key: pathfind_source_account + action: hash + - key: pathfind_dest_account + action: hash connectors: spanmetrics: @@ -176,7 +185,7 @@ service: pipelines: traces: receivers: [otlp] - processors: [resource/tier, resource/stripsdk, batch] + processors: [resource/tier, resource/stripsdk, attributes/hash, batch] exporters: [debug, otlp/tempo, spanmetrics] metrics: receivers: [otlp, spanmetrics] diff --git a/docs/telemetry-runbook.md b/docs/telemetry-runbook.md index 88c1a80f1c..a9cb25cae1 100644 --- a/docs/telemetry-runbook.md +++ b/docs/telemetry-runbook.md @@ -106,6 +106,15 @@ RED metrics on the _Transaction Overview_ dashboard. | `txq.accept_tx` | TxQ.cpp | `tx_hash`, `retries_remaining`, `ter_code`, `txq_status` | Per-transaction apply during accept | | `txq.cleanup` | TxQ.cpp | `ledger_seq` | Post-close cleanup of expired queue entries | +### PathFinding Spans + +| Span Name | Source File | Attributes | Description | +| --------------------- | --------------------------------- | -------------------------------------------------- | ------------------------------------------------------- | +| `pathfind.request` | PathFind.cpp / RipplePathFind.cpp | `pathfind_source_account`, `pathfind_dest_account` | Path-find RPC entry (accounts hashed; set when present) | +| `pathfind.compute` | PathRequest.cpp | `pathfind_fast`, `pathfind_dest_currency` | Path computation for one request (`doUpdate`) | +| `pathfind.discover` | PathRequest.cpp | `pathfind_search_level`, `pathfind_num_paths` | Graph exploration (one per RPC call in `findPaths`) | +| `pathfind.update_all` | PathRequestManager.cpp | `pathfind_ledger_index`, `pathfind_num_requests` | Async recomputation of active requests on ledger close | + ### Consensus Spans | Span Name | Source File | Attributes | Description | @@ -116,7 +125,7 @@ RED metrics on the _Transaction Overview_ dashboard. | `consensus.ledger_close` | RCLConsensus.cpp | `ledger_seq`, `consensus_mode` | Ledger close event | | `consensus.establish` | Consensus.h | `converge_percent`, `establish_count`, `proposers` | Establish phase duration (child of round) | | `consensus.update_positions` | Consensus.h | `converge_percent`, `proposers`, `disputes_count` | Position update and dispute resolution (see Events below) | -| `consensus.check` | Consensus.h | `agree_count`, `disagree_count`, `converge_percent`, `have_close_time_consensus`, `threshold_percent`, `consensus_result` | Consensus threshold check | +| `consensus.check` | Consensus.h | `agree_count`, `disagree_count`, `converge_percent`, `have_close_time_consensus`, `threshold_percent`, `proposers_finished`, `consensus_stalled`, `establish_count`, `consensus_result` | Consensus threshold check | | `consensus.accept` | RCLConsensus.cpp | `proposers`, `round_time_ms`, `quorum`, `disputes_count`, `consensus_state` | Ledger accepted by consensus | | `consensus.accept.apply` | RCLConsensus.cpp | `ledger_seq`, `close_time`, `close_time_correct`, `close_resolution_ms`, `consensus_state`, `proposing`, `round_time_ms`, `parent_close_time`, `close_time_self`, `close_time_vote_bins`, `resolution_direction`, `tx_count` | Ledger application with close time details (see Events below) | | `consensus.validation.send` | RCLConsensus.cpp | `ledger_seq`, `proposing`, `ledger_hash`, `full_validation`, `validation_sign_time` | Validation sent after accept (follows-from link) | @@ -488,7 +497,7 @@ all its normal attributes, it just lacks a cross-node parent link. {name="tx.receive" && status != error} # Find proposals received with cross-node parent context -{name="consensus.proposal.receive"} +{} >> {name="consensus.proposal.receive"} # Trace a transaction across the network by its hash {name =~ "tx.*" && span.tx_hash = ""} @@ -524,7 +533,12 @@ Every metric carries these standard labels: | `service_name` | Resource attribute | `xrpld` | | `span_kind` | Span kind | `SPAN_KIND_INTERNAL` | -Additionally, span attributes configured as dimensions in the collector become metric labels. The collector dimensions use the bare attribute keys emitted by the code, so the label name equals the attribute name: +Additionally, span attributes configured as dimensions in the collector +become metric labels. The span attribute keys are already underscore form +(the naming convention forbids dots), so the label name matches the attribute +name verbatim. Prometheus' dots → underscores sanitization only fires for +dotted attribute names (e.g. resource attributes like `service.name`), which +does not apply to these dimensions. | Span Attribute | Metric Label | Applies To | | -------------------- | -------------------- | ------------------------------- | diff --git a/include/xrpl/telemetry/Redaction.h b/include/xrpl/telemetry/Redaction.h new file mode 100644 index 0000000000..ed84ae2964 --- /dev/null +++ b/include/xrpl/telemetry/Redaction.h @@ -0,0 +1,64 @@ +#pragma once + +/** Account-address redaction for telemetry span attributes. + + Path-finding RPC handlers would otherwise emit the caller's raw + account addresses as span attributes. To keep plaintext addresses out + of the telemetry backend, they are hashed at the point of emission. + This header exposes a single pure helper that turns an address into a + short, stable, obfuscated token. + + Data flow: + + handler -> redactAccount(addr) -> span attribute -> OTLP export + + The returned token is the first 16 hex characters (lowercase) of the + SHA-512Half digest of the address. It is deterministic (same address + always maps to the same token) so operators can still correlate spans + for a given account across nodes and restarts. + + The hash is unsalted, so it is obfuscation, not a secrecy guarantee: + XRP account addresses are a public, enumerable set, so a determined + observer with the telemetry stream could rebuild the address->token + mapping. The goal here is to keep plaintext addresses out of traces + and dashboards, not to defend against a precomputation attack. A salt + is intentionally omitted because it would break cross-node/restart + correlation, which is the reason for hashing rather than dropping. + + A second, independent hashing layer runs in the OpenTelemetry + Collector (an `attributes/hash` processor) as defense-in-depth for + any node that emits a raw value. + + @note This function is pure and reentrant: it holds no global state, + performs no I/O, and is safe to call concurrently from any thread. + + Usage example: + @code + #include + using namespace xrpl::telemetry; + + span.setAttribute( + pathfind_span::attr::sourceAccount, redactAccount(src.asString())); + @endcode + + Edge case (empty input yields empty output): + @code + assert(redactAccount("") == ""); + @endcode +*/ + +#include +#include + +namespace xrpl::telemetry { + +/** Hash an account address into a short, stable, obfuscated token. + + @param addr The account address to redact (e.g. an r-address). + @return The first 16 lowercase hex characters of sha512Half(addr), + or an empty string when @p addr is empty. +*/ +[[nodiscard]] std::string +redactAccount(std::string_view addr); + +} // namespace xrpl::telemetry diff --git a/include/xrpl/tx/detail/TxApplySpanNames.h b/include/xrpl/tx/detail/TxApplySpanNames.h index c007ca3c03..2084458cf5 100644 --- a/include/xrpl/tx/detail/TxApplySpanNames.h +++ b/include/xrpl/tx/detail/TxApplySpanNames.h @@ -51,9 +51,11 @@ * @endcode * * @code - * // Transactor::operator() uses span() with prefix + suffix: - * auto span = SpanGuard::span( - * TraceCategory::Transactions, seg::tx, tx_apply_span::op::transactor); + * // Transactor::operator() also uses hashSpan on the same txID so it + * // co-traces with preflight and preclaim under one trace_id: + * auto span = SpanGuard::hashSpan( + * TraceCategory::Transactions, tx_apply_span::transactor, + * txID.data(), txID.kBytes); * span.setAttribute(tx_apply_span::attr::stage, tx_apply_span::val::apply); * @endcode */ @@ -79,6 +81,9 @@ inline constexpr auto transactor = makeStr("transactor"); inline constexpr auto preflight = join(seg::tx, op::preflight); /// "tx.preclaim" — full name for hashSpan() at the preclaim stage. inline constexpr auto preclaim = join(seg::tx, op::preclaim); +/// "tx.transactor" — full name for hashSpan() at the apply stage. Shares the +/// txID-derived trace_id so it co-traces with tx.preflight and tx.preclaim. +inline constexpr auto transactor = join(seg::tx, op::transactor); // ===== Attribute keys ====================================================== diff --git a/src/libxrpl/telemetry/Redaction.cpp b/src/libxrpl/telemetry/Redaction.cpp new file mode 100644 index 0000000000..ee15b8812e --- /dev/null +++ b/src/libxrpl/telemetry/Redaction.cpp @@ -0,0 +1,32 @@ +#include + +#include +#include +#include + +#include +#include +#include +#include + +namespace xrpl::telemetry { + +std::string +redactAccount(std::string_view addr) +{ + // Empty in, empty out: nothing to hash and no token to emit. + if (addr.empty()) + return {}; + + // sha512Half yields a uint256; to_string renders it as uppercase hex. + // Keep the first 16 chars (64 bits) — enough to correlate spans while + // staying non-reversible — and lowercase them for a stable token. + auto const digest = sha512Half(Slice(addr.data(), addr.size())); + std::string token = to_string(digest).substr(0, 16); + std::transform(token.begin(), token.end(), token.begin(), [](unsigned char c) { + return static_cast(std::tolower(c)); + }); + return token; +} + +} // namespace xrpl::telemetry diff --git a/src/libxrpl/telemetry/SpanGuard.cpp b/src/libxrpl/telemetry/SpanGuard.cpp index 6c37a495bb..05d0ca8067 100644 --- a/src/libxrpl/telemetry/SpanGuard.cpp +++ b/src/libxrpl/telemetry/SpanGuard.cpp @@ -238,6 +238,7 @@ SpanGuard::linkedSpan(std::string_view name) const return SpanGuard( std::make_unique(tracer->StartSpan( std::string(name), {}, {{spanCtx, {{kLinkTypeKey, kLinkTypeFollowsFrom}}}}, opts))); + // LCOV_EXCL_STOP } SpanGuard diff --git a/src/libxrpl/telemetry/TelemetryConfig.cpp b/src/libxrpl/telemetry/TelemetryConfig.cpp index 00705aa4ed..17eacf6582 100644 --- a/src/libxrpl/telemetry/TelemetryConfig.cpp +++ b/src/libxrpl/telemetry/TelemetryConfig.cpp @@ -112,6 +112,16 @@ makeTelemetrySetup( "(set both for mutual TLS, or neither for one-way TLS)."); } + // Mutual TLS only takes effect when TLS is on. Certificate paths set with + // use_tls=0 would be silently ignored and the exporter would connect in + // plaintext, so reject that contradiction instead of failing open. + if (!setup.tlsClientCertPath.empty() && !setup.useTls) + { + Throw( + "[telemetry] tls_client_cert/tls_client_key require use_tls=1 " + "(set use_tls=1 to enable mutual TLS, or remove the cert paths)."); + } + // Head sampling is intentionally fixed at 1.0 (sample everything) and is // not read from config. A per-node ratio would let nodes make divergent // keep/drop decisions for the same distributed trace, producing broken diff --git a/src/libxrpl/tx/Transactor.cpp b/src/libxrpl/tx/Transactor.cpp index 0c33995db2..e2752ba67b 100644 --- a/src/libxrpl/tx/Transactor.cpp +++ b/src/libxrpl/tx/Transactor.cpp @@ -1294,10 +1294,15 @@ Transactor::checkInvariants(TER result, XRPAmount fee) ApplyResult Transactor::operator()() { - auto span = telemetry::SpanGuard::span( + // Derive the trace_id from the transaction id so this apply-stage span + // shares one trace with the preflight and preclaim spans (which also use + // hashSpan on the same id). + auto const txID = ctx_.tx.getTransactionID(); + auto span = telemetry::SpanGuard::hashSpan( telemetry::TraceCategory::Transactions, - telemetry::seg::tx, - telemetry::tx_apply_span::op::transactor); + telemetry::tx_apply_span::transactor, + txID.data(), + txID.kBytes); // "apply" — the third apply-pipeline stage, after preflight and preclaim. span.setAttribute(telemetry::tx_apply_span::attr::stage, telemetry::tx_apply_span::val::apply); if (auto const* fmt = TxFormats::getInstance().findByType(ctx_.tx.getTxnType())) @@ -1428,6 +1433,11 @@ Transactor::operator()() span.setAttribute(telemetry::tx_apply_span::attr::terResult, transToken(result).c_str()); span.setAttribute(telemetry::tx_apply_span::attr::applied, applied); + // Mark the span as errored when the transaction was not applied or the + // engine result is not a success, so failed applies surface in span-status + // error counts alongside preflight and preclaim. + if (!applied || !isTesSuccess(result)) + span.setError(transToken(result)); return {result, applied, metadata}; } diff --git a/src/libxrpl/tx/applySteps.cpp b/src/libxrpl/tx/applySteps.cpp index 77c39d6f49..8a2c97c550 100644 --- a/src/libxrpl/tx/applySteps.cpp +++ b/src/libxrpl/tx/applySteps.cpp @@ -190,6 +190,10 @@ invokePreflight(PreflightContext const& ctx) { span.setAttribute( telemetry::tx_apply_span::attr::terResult, transToken(result.first).c_str()); + // Mark the span as errored when preflight rejects the transaction so + // failed stages surface in span-status error counts. + if (!isTesSuccess(result.first)) + span.setError(transToken(result.first)); } return result; } diff --git a/src/tests/libxrpl/telemetry/Redaction.cpp b/src/tests/libxrpl/telemetry/Redaction.cpp new file mode 100644 index 0000000000..7f16c0f1dc --- /dev/null +++ b/src/tests/libxrpl/telemetry/Redaction.cpp @@ -0,0 +1,44 @@ +#include + +#include + +#include +#include +#include + +using namespace xrpl; +using namespace xrpl::telemetry; + +// Empty input must produce empty output (edge / negative path). +TEST(Redaction, empty_input_returns_empty) +{ + EXPECT_EQ(redactAccount(""), ""); +} + +// Success path: assert the EXACT expected token. The value is the first +// 16 chars of sha512Half(addr) rendered as lowercase hex, computed once +// and hard-coded here so any change to the hashing recipe is caught. +TEST(Redaction, known_account_exact_hash) +{ + EXPECT_EQ(redactAccount("rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh"), "a513895f49311f54"); +} + +// Structural invariants: length, lowercase-hex alphabet, and stability. +TEST(Redaction, token_is_16_lowercase_hex_and_stable) +{ + auto const token = redactAccount("rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh"); + EXPECT_EQ(token.size(), 16u); + EXPECT_TRUE(std::all_of(token.begin(), token.end(), [](unsigned char c) { + return std::isdigit(c) || (c >= 'a' && c <= 'f'); + })); + // Deterministic: hashing the same input twice yields the same token. + EXPECT_EQ(token, redactAccount("rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh")); +} + +// Distinct inputs must produce distinct tokens (no accidental constant). +TEST(Redaction, distinct_inputs_distinct_tokens) +{ + EXPECT_NE( + redactAccount("rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh"), + redactAccount("rDifferentAccount1234567890abcdefgh")); +} diff --git a/src/tests/libxrpl/telemetry/TraceContextPropagator.cpp b/src/tests/libxrpl/telemetry/TraceContextPropagator.cpp index efcb172845..fdc6c443cb 100644 --- a/src/tests/libxrpl/telemetry/TraceContextPropagator.cpp +++ b/src/tests/libxrpl/telemetry/TraceContextPropagator.cpp @@ -83,10 +83,8 @@ TEST(TraceContextPropagator, extract_empty_protobuf) protocol::TraceContext const proto; auto ctx = xrpl::telemetry::extractFromProtobuf(proto); auto span = trace::GetSpan(ctx); - if (span) - { - EXPECT_FALSE(span->GetContext().IsValid()); - } + ASSERT_NE(span, nullptr); + EXPECT_FALSE(span->GetContext().IsValid()); } TEST(TraceContextPropagator, extract_wrong_size_trace_id) @@ -97,10 +95,8 @@ TEST(TraceContextPropagator, extract_wrong_size_trace_id) auto ctx = xrpl::telemetry::extractFromProtobuf(proto); auto span = trace::GetSpan(ctx); - if (span) - { - EXPECT_FALSE(span->GetContext().IsValid()); - } + ASSERT_NE(span, nullptr); + EXPECT_FALSE(span->GetContext().IsValid()); } TEST(TraceContextPropagator, extract_wrong_size_span_id) @@ -111,10 +107,8 @@ TEST(TraceContextPropagator, extract_wrong_size_span_id) auto ctx = xrpl::telemetry::extractFromProtobuf(proto); auto span = trace::GetSpan(ctx); - if (span) - { - EXPECT_FALSE(span->GetContext().IsValid()); - } + ASSERT_NE(span, nullptr); + EXPECT_FALSE(span->GetContext().IsValid()); } TEST(TraceContextPropagator, inject_invalid_span) diff --git a/src/xrpld/app/consensus/RCLConsensus.cpp b/src/xrpld/app/consensus/RCLConsensus.cpp index 5bf89e0a7a..9ad0dcca65 100644 --- a/src/xrpld/app/consensus/RCLConsensus.cpp +++ b/src/xrpld/app/consensus/RCLConsensus.cpp @@ -1274,8 +1274,25 @@ RCLConsensus::Adaptor::startRoundTracing(RCLCxLedger const& prevLgr) telemetry::SpanContext const* const link = prevRoundSpanContext_.isValid() ? &prevRoundSpanContext_ : nullptr; - if (strategy == "deterministic") + if (strategy == "attribute") { + // Non-deterministic strategy: each node gets a random trace_id, + // correlated via the consensus_ledger_id attribute rather than a + // shared trace_id. Still attach a follows-from link to the prior + // round so consecutive rounds stay navigable. linkedSpan is not + // TraceCategory-aware, so gate it explicitly to match the gating + // of the hashSpan/span factories used below. + if (link != nullptr && app_.getTelemetry().shouldTraceConsensus()) + roundSpan_.emplace(telemetry::SpanGuard::linkedSpan(cs::round, *link)); + else + roundSpan_.emplace( + telemetry::SpanGuard::span( + telemetry::TraceCategory::Consensus, telemetry::seg::consensus, cs::op::round)); + } + else + { + // "deterministic" (the default): derive the trace_id from the previous + // ledger hash so all validators tracing the same round share one trace. roundSpan_.emplace( telemetry::SpanGuard::hashSpan( telemetry::TraceCategory::Consensus, @@ -1284,12 +1301,6 @@ RCLConsensus::Adaptor::startRoundTracing(RCLCxLedger const& prevLgr) prevLgr.id().kBytes, link)); } - else - { - roundSpan_.emplace( - telemetry::SpanGuard::span( - telemetry::TraceCategory::Consensus, telemetry::seg::consensus, cs::op::round)); - } if (!*roundSpan_) return; diff --git a/src/xrpld/app/misc/NetworkOPs.cpp b/src/xrpld/app/misc/NetworkOPs.cpp index bf7e0e8cb9..7bfa711ee7 100644 --- a/src/xrpld/app/misc/NetworkOPs.cpp +++ b/src/xrpld/app/misc/NetworkOPs.cpp @@ -17,7 +17,6 @@ #include #include #include -#include #include #include #include @@ -36,6 +35,7 @@ #include #include #include +#include #include #include diff --git a/src/xrpld/app/misc/detail/TxQ.cpp b/src/xrpld/app/misc/detail/TxQ.cpp index 8e341efc07..3336e9bd2b 100644 --- a/src/xrpld/app/misc/detail/TxQ.cpp +++ b/src/xrpld/app/misc/detail/TxQ.cpp @@ -1459,9 +1459,6 @@ bool TxQ::accept(Application& app, OpenView& view) { using namespace telemetry; - auto span = - SpanGuard::span(TraceCategory::Transactions, txq_span::prefix::txq, txq_span::op::accept); - span.setAttribute(txq_span::attr::queueSize, static_cast(byFee_.size())); /* Move transactions from the queue from largest fee level to smallest. As we add more transactions, the required fee level will increase. @@ -1473,6 +1470,12 @@ TxQ::accept(Application& app, OpenView& view) std::scoped_lock const lock(mutex_); + // Create the span and read byFee_.size() only after taking the lock, since + // byFee_ is guarded by mutex_. + auto span = + SpanGuard::span(TraceCategory::Transactions, txq_span::prefix::txq, txq_span::op::accept); + span.setAttribute(txq_span::attr::queueSize, static_cast(byFee_.size())); + auto const metricsSnapshot = feeMetrics_.getSnapshot(); for (auto candidateIter = byFee_.begin(); candidateIter != byFee_.end();) diff --git a/src/xrpld/overlay/detail/PeerImp.cpp b/src/xrpld/overlay/detail/PeerImp.cpp index dd724f375d..6d960b33bd 100644 --- a/src/xrpld/overlay/detail/PeerImp.cpp +++ b/src/xrpld/overlay/detail/PeerImp.cpp @@ -7,7 +7,6 @@ #include #include #include -#include #include #include #include @@ -25,6 +24,7 @@ #include #include #include +#include #include #include diff --git a/src/xrpld/rpc/detail/PathFindSpanNames.h b/src/xrpld/rpc/detail/PathFindSpanNames.h index 3a12937fbc..71eb805ea3 100644 --- a/src/xrpld/rpc/detail/PathFindSpanNames.h +++ b/src/xrpld/rpc/detail/PathFindSpanNames.h @@ -84,8 +84,6 @@ inline constexpr auto numPaths = makeStr("pathfind_num_paths"); inline constexpr auto numRequests = makeStr("pathfind_num_requests"); /// "pathfind_ledger_index" — pathfind target ledger index. inline constexpr auto ledgerIndex = makeStr("pathfind_ledger_index"); -/// "pathfind_dest_amount" — requested destination amount as string. -inline constexpr auto destAmount = makeStr("pathfind_dest_amount"); /// "pathfind_dest_currency" — destination currency code. inline constexpr auto destCurrency = makeStr("pathfind_dest_currency"); /// "pathfind_num_source_assets" — candidate source assets count. diff --git a/src/xrpld/rpc/detail/PathRequest.cpp b/src/xrpld/rpc/detail/PathRequest.cpp index 324036d39b..4fb11ab10c 100644 --- a/src/xrpld/rpc/detail/PathRequest.cpp +++ b/src/xrpld/rpc/detail/PathRequest.cpp @@ -742,7 +742,6 @@ PathRequest::doUpdate( auto span = SpanGuard::span( TraceCategory::Rpc, pathfind_span::prefix::pathfind, pathfind_span::op::compute); span.setAttribute(pathfind_span::attr::fast, fast); - span.setAttribute(pathfind_span::attr::destAmount, saDstAmount_.getFullText().c_str()); span.setAttribute(pathfind_span::attr::destCurrency, to_string(saDstAmount_.asset()).c_str()); JLOG(journal_.debug()) << iIdentifier_ << " update " << (fast ? "fast" : "normal"); diff --git a/src/xrpld/rpc/detail/PathRequestManager.cpp b/src/xrpld/rpc/detail/PathRequestManager.cpp index b2831c5518..01df901ce3 100644 --- a/src/xrpld/rpc/detail/PathRequestManager.cpp +++ b/src/xrpld/rpc/detail/PathRequestManager.cpp @@ -73,15 +73,16 @@ PathRequestManager::updateAll(std::shared_ptr const& inLedger) cache = getAssetCache(inLedger, true); } - // updateAll runs on every ledger close; skip span emission entirely when - // there are no active path subscriptions to avoid a steady stream of empty - // spans at mainnet close cadence. - if (requests.empty()) - return; - using namespace telemetry; - auto span = SpanGuard::span( - TraceCategory::Rpc, pathfind_span::prefix::pathfind, pathfind_span::op::updateAll); + // updateAll runs on every ledger close. Skip span emission when there are + // no active path subscriptions, to avoid a steady stream of empty spans at + // mainnet close cadence. A null guard is used in that case; all other work + // still runs unchanged (notably the isNewPathRequest() flag reset below), + // so behaviour matches the pre-span code path. + auto span = requests.empty() + ? SpanGuard{} + : SpanGuard::span( + TraceCategory::Rpc, pathfind_span::prefix::pathfind, pathfind_span::op::updateAll); span.setAttribute(pathfind_span::attr::ledgerIndex, static_cast(inLedger->seq())); span.setAttribute(pathfind_span::attr::numRequests, static_cast(requests.size())); diff --git a/src/xrpld/rpc/detail/RPCHandler.cpp b/src/xrpld/rpc/detail/RPCHandler.cpp index 14082e3b88..1a918c343c 100644 --- a/src/xrpld/rpc/detail/RPCHandler.cpp +++ b/src/xrpld/rpc/detail/RPCHandler.cpp @@ -221,6 +221,31 @@ callMethod(JsonContext& context, Method method, std::string const& name, Object& } } +// Resolve the span suffix / command attribute for a request that failed in +// fillHandler. Returns the canonical handler name for a recognized command +// (a finite, bounded set) or the literal "unknown" for a request that omits +// both fields or names an unregistered command. The raw request value is +// deliberately NOT used: the command attribute is promoted to a Prometheus +// label by the spanmetrics connector, so an attacker-controlled string would +// let arbitrary request input drive unbounded span-name / label cardinality. +// Resolving against the registry keeps per-command error attribution for real +// commands (e.g. a submit rejected with rpcTOO_BUSY stays rpc.command.submit) +// while collapsing garbage input to a single series. +std::string_view +resolveCommandSpanName(JsonContext const& context) +{ + if (!context.params.isMember(jss::command) && !context.params.isMember(jss::method)) + return rpc_span::val::unknownCommand; + + std::string const cmd = context.params.isMember(jss::command) + ? context.params[jss::command].asString() + : context.params[jss::method].asString(); + + auto const* handler = getHandler(context.apiVersion, context.app.config().betaRpcApi, cmd); + return (handler != nullptr) ? std::string_view{handler->name} + : std::string_view{rpc_span::val::unknownCommand}; +} + } // namespace Status @@ -229,25 +254,12 @@ doCommand(RPC::JsonContext& context, json::Value& result) Handler const* handler = nullptr; if (auto error = fillHandler(context, handler)) { - std::string cmdName; - if (context.params.isMember(jss::command)) - { - cmdName = context.params[jss::command].asString(); - } - else if (context.params.isMember(jss::method)) - { - cmdName = context.params[jss::method].asString(); - } - else - { - cmdName = "unknown"; // LCOV_EXCL_LINE - } - // Use the resolved command name as the span suffix so dashboards - // can break out per-command error rates (e.g. rpc.command.submit - // for a submit that hit rpcTOO_BUSY). Falling back to a single - // "unknown" name only when the request truly omits both fields. + // Bound the span name and command attribute to the finite set of + // registered handler names (plus "unknown") — see the helper for why + // raw request input must not reach the telemetry pipeline. + auto const cmdName = resolveCommandSpanName(context); auto span = SpanGuard::span(TraceCategory::Rpc, rpc_span::prefix::command, cmdName); - span.setAttribute(rpc_span::attr::command, cmdName.c_str()); + span.setAttribute(rpc_span::attr::command, cmdName); span.setAttribute(rpc_span::attr::rpcStatus, rpc_span::val::error); span.setError(getErrorInfo(error).token.cStr()); diff --git a/src/xrpld/rpc/handlers/orderbook/PathFind.cpp b/src/xrpld/rpc/handlers/orderbook/PathFind.cpp index cecd1083c9..f04756be15 100644 --- a/src/xrpld/rpc/handlers/orderbook/PathFind.cpp +++ b/src/xrpld/rpc/handlers/orderbook/PathFind.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include namespace xrpl { @@ -20,10 +21,11 @@ doPathFind(RPC::JsonContext& context) using namespace telemetry; auto span = SpanGuard::span( TraceCategory::Rpc, pathfind_span::prefix::pathfind, pathfind_span::op::request); + // Addresses are hashed before emission for privacy. if (auto const& src = context.params[jss::source_account]; src.isString()) - span.setAttribute(pathfind_span::attr::sourceAccount, src.asString()); + span.setAttribute(pathfind_span::attr::sourceAccount, redactAccount(src.asString())); if (auto const& dst = context.params[jss::destination_account]; dst.isString()) - span.setAttribute(pathfind_span::attr::destAccount, dst.asString()); + span.setAttribute(pathfind_span::attr::destAccount, redactAccount(dst.asString())); if (context.app.config().pathSearchMax == 0) return rpcError(RpcNotSupported); diff --git a/src/xrpld/rpc/handlers/orderbook/RipplePathFind.cpp b/src/xrpld/rpc/handlers/orderbook/RipplePathFind.cpp index b55ba2515f..e08ef5445c 100644 --- a/src/xrpld/rpc/handlers/orderbook/RipplePathFind.cpp +++ b/src/xrpld/rpc/handlers/orderbook/RipplePathFind.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -28,10 +29,11 @@ doRipplePathFind(RPC::JsonContext& context) using namespace telemetry; auto span = SpanGuard::span( TraceCategory::Rpc, pathfind_span::prefix::pathfind, pathfind_span::op::request); + // Addresses are hashed before emission for privacy. if (auto const& src = context.params[jss::source_account]; src.isString()) - span.setAttribute(pathfind_span::attr::sourceAccount, src.asString()); + span.setAttribute(pathfind_span::attr::sourceAccount, redactAccount(src.asString())); if (auto const& dst = context.params[jss::destination_account]; dst.isString()) - span.setAttribute(pathfind_span::attr::destAccount, dst.asString()); + span.setAttribute(pathfind_span::attr::destAccount, redactAccount(dst.asString())); if (context.app.config().pathSearchMax == 0) return rpcError(RpcNotSupported); diff --git a/src/xrpld/app/misc/TxSpanNames.h b/src/xrpld/telemetry/TxSpanNames.h similarity index 100% rename from src/xrpld/app/misc/TxSpanNames.h rename to src/xrpld/telemetry/TxSpanNames.h diff --git a/src/xrpld/telemetry/TxTracing.h b/src/xrpld/telemetry/TxTracing.h index 47bb30eb54..066c018601 100644 --- a/src/xrpld/telemetry/TxTracing.h +++ b/src/xrpld/telemetry/TxTracing.h @@ -10,7 +10,7 @@ * no-op SpanGuard instances (zero overhead, zero dependencies). */ -#include +#include #include #include