From 0505ca35aadde1750ff54dccd1d20892a9fb5d55 Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Mon, 6 Jul 2026 17:41:38 +0100 Subject: [PATCH 1/4] fix(telemetry): make Tempo search filters dynamic Change the six remaining `type: static` search filters in tempo.yaml to `type: dynamic` so Grafana queries Tempo for available tag values instead of requiring manual free-text entry. Aligns service-name, node-id, network-type, span-name, span-status, and span-duration with the existing node-version and network-id filters. Co-Authored-By: Claude Opus 4.8 --- .../grafana/provisioning/datasources/tempo.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docker/telemetry/grafana/provisioning/datasources/tempo.yaml b/docker/telemetry/grafana/provisioning/datasources/tempo.yaml index 825d55453c..4214c72494 100644 --- a/docker/telemetry/grafana/provisioning/datasources/tempo.yaml +++ b/docker/telemetry/grafana/provisioning/datasources/tempo.yaml @@ -36,7 +36,7 @@ datasources: tag: service.name operator: "=" scope: resource - type: static + type: dynamic # service.instance.id: unique node identifier — defaults to the # node's public key (e.g., nHB1X37...). Distinguishes individual # nodes in a multi-node cluster or network. @@ -44,7 +44,7 @@ datasources: tag: service.instance.id operator: "=" scope: resource - type: static + type: dynamic # service.version: xrpld build version (e.g., "2.4.0-b1"). # Filter traces from specific software releases. - id: node-version @@ -66,7 +66,7 @@ datasources: tag: xrpl.network.type operator: "=" scope: resource - type: static + type: dynamic # --- Span intrinsic filters --- # name: the span operation name (e.g., "rpc.command.server_info"). # Use to find traces for a specific RPC command or subsystem. @@ -74,18 +74,18 @@ datasources: tag: name operator: "=" scope: intrinsic - type: static + type: dynamic # status: span completion status ("ok", "error", "unset"). # Filter for failed operations to diagnose errors. - id: span-status tag: status operator: "=" scope: intrinsic - type: static + type: dynamic # duration: span wall-clock duration. Use with ">" operator # to find slow operations (e.g., duration > 500ms). - id: span-duration tag: duration operator: ">" scope: intrinsic - type: static + type: dynamic From 26a85c764e099cb8036650c11b24e8aa7095dd16 Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Mon, 6 Jul 2026 18:15:23 +0100 Subject: [PATCH 2/4] code review comment addressed Signed-off-by: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> --- CONTRIBUTING.md | 2 +- OpenTelemetryPlan/02-design-decisions.md | 36 +- .../03-implementation-strategy.md | 8 +- .../05-configuration-reference.md | 4 +- .../07-observability-backends.md | 6 +- OpenTelemetryPlan/08-appendix.md | 6 - OpenTelemetryPlan/OpenTelemetryPlan.md | 6 +- OpenTelemetryPlan/presentation.md | 682 ------------------ 8 files changed, 33 insertions(+), 717 deletions(-) delete mode 100644 OpenTelemetryPlan/presentation.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 14c8d60117..6e2c2e47b3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -321,7 +321,7 @@ other layer must match them. A CI check enforces this end to end. 3. Collision qualifier: `_` — only when a bare name would collide with a DIFFERENT concept in the shared spanmetrics label space, or with the OTel-reserved `status` key (e.g. `rpc_status`, `grpc_status`, - `consensus_state`, `consensus_round`). This disambiguates distinct concepts + `consensus_phase`, `consensus_round`). This disambiguates distinct concepts that share a word; it is NOT used to tag the same concept with the workflow that emitted it — that is rule 2 (one shared name). 4. Resource attribute: dotted `xrpl..` — reserved ONLY for diff --git a/OpenTelemetryPlan/02-design-decisions.md b/OpenTelemetryPlan/02-design-decisions.md index a30df9e755..b26b26b461 100644 --- a/OpenTelemetryPlan/02-design-decisions.md +++ b/OpenTelemetryPlan/02-design-decisions.md @@ -185,7 +185,7 @@ agree with the code. A CI check enforces this end to end. 3. **Collision qualifier** → `_`, only when a bare name would collide with a DIFFERENT concept in the shared spanmetrics label space or with the OTel-reserved `status` key (e.g. `rpc_status`, `grpc_status`, - `consensus_state`, `consensus_round`, `consensus_mode`). This disambiguates + `consensus_phase`, `consensus_round`, `consensus_mode`). This disambiguates distinct concepts that share a word; it is NOT used to tag the same concept with its emitting workflow — that is rule 2 (one shared name). 4. **Resource attribute** → dotted `xrpl..`, reserved ONLY @@ -245,6 +245,8 @@ keys (the dotted form is reserved for resource scope per §2.3.3). | `tx_fee` | int64 | Fee in drops | | `tx_result` | string | `"tesSUCCESS"`, `"tecPATH_DRY"`, etc. | | `ledger_index` | int64 | Ledger containing transaction | +| `relay_count` | int64 | Peers the transaction was relayed to | +| `suppressed` | bool | `true` when HashRouter dropped a dup | #### Consensus Attributes @@ -261,12 +263,14 @@ keys (the dotted form is reserved for resource scope per §2.3.3). #### RPC Attributes -| Key | Type | Description | -| ---------- | ------ | ----------------------------------------------------- | -| `command` | string | Command name (per-span unique on `rpc.command`) | -| `version` | int64 | API version | -| `rpc_role` | string | `"admin"` or `"user"` (qualified — `role` is generic) | -| `params` | string | Sanitized parameters (optional) | +| Key | Type | Description | +| ------------- | ------- | ----------------------------------------------------------------------------- | +| `command` | string | Command name (per-span unique on `rpc.command`) | +| `version` | int64 | API version | +| `rpc_role` | string | `"admin"` or `"user"` (qualified — `role` is generic) | +| `params` | string | Sanitized parameters (optional) | +| `rpc_status` | string | Response status: `success` \| `error` (qualified — `status` is OTel-reserved) | +| `duration_ms` | float64 | Request duration in milliseconds | #### Peer & Message Attributes @@ -378,14 +382,14 @@ The following data is explicitly **excluded** from telemetry collection: #### Privacy Protection Mechanisms -| Mechanism | Description | -| ----------------------------- | ------------------------------------------------------------------------- | -| **Account Hashing** | `tx_account` is hashed at collector level before storage | -| **Configurable Redaction** | Sensitive fields can be excluded via `[telemetry]` config section | -| **Sampling** | Only 10% of traces recorded by default, reducing data exposure | -| **Local Control** | Node operators have full control over what gets exported | -| **No Raw Payloads** | Transaction content is never recorded, only metadata (hash, type, result) | -| **Collector-Level Filtering** | Additional redaction/hashing can be configured at OTel Collector | +| Mechanism | Description | +| ----------------------------- | --------------------------------------------------------------------------------------------------------------------------------------- | +| **Account Hashing** | `tx_account` is hashed at collector level before storage | +| **Configurable Redaction** | Sensitive fields can be excluded via `[telemetry]` config section | +| **Collector Tail Sampling** | xrpld head sampling is fixed at 1.0 (every span emitted); the collector retains ~10% of non-error traces, reducing stored data exposure | +| **Local Control** | Node operators have full control over what gets exported | +| **No Raw Payloads** | Transaction content is never recorded, only metadata (hash, type, result) | +| **Collector-Level Filtering** | Additional redaction/hashing can be configured at OTel Collector | #### Collector-Level Data Protection @@ -546,7 +550,7 @@ flowchart TB - **xrpld Process (dark gray)**: The single xrpld node running all three observability frameworks side by side. Each framework operates independently with no interference. - **PerfLog to perf.log**: PerfLog writes JSON-formatted event logs to a local file. Grafana can ingest these via Loki or a file-based datasource. - **Beast Insight to StatsD Server**: Insight sends aggregated metrics (counters, gauges) over UDP to a StatsD server. Grafana reads from StatsD-compatible backends like Graphite or Prometheus (via StatsD exporter). -- **OpenTelemetry to OTLP Collector**: OTel exports spans over OTLP/gRPC to a Collector, which then forwards to a trace backend (Tempo). +- **OpenTelemetry to OTLP Collector**: OTel exports spans over OTLP/HTTP to a Collector, which then forwards to a trace backend (Tempo). (OTLP/gRPC is future work — §2.2.2.) - **Grafana (red, unified UI)**: All three data streams converge in Grafana, enabling operators to correlate logs, metrics, and traces in a single dashboard. ### 2.6.5 Correlation with PerfLog diff --git a/OpenTelemetryPlan/03-implementation-strategy.md b/OpenTelemetryPlan/03-implementation-strategy.md index d51e30b211..4edcef6194 100644 --- a/OpenTelemetryPlan/03-implementation-strategy.md +++ b/OpenTelemetryPlan/03-implementation-strategy.md @@ -184,14 +184,14 @@ pie showData | TracerProvider singleton | ~64 KB | At startup | | BatchSpanProcessor (circular buffer) | ~16 KB | At startup | | BatchSpanProcessor (worker thread) | ~8 MB | At startup | -| OTLP exporter (gRPC channel init) | ~256 KB | At startup | +| OTLP/HTTP exporter (client init) | ~64 KB | At startup | | Propagator registry | ~8 KB | At startup | -| **Total static** | **~8.3 MB** | | +| **Total static** | **~8.1 MB** | | > **Why higher than earlier estimate**: The BatchSpanProcessor's circular buffer itself is only ~16 KB > (2049 x 8-byte `AtomicUniquePtr` entries), but it spawns a dedicated worker thread whose default -> stack size on Linux is ~8 MB. The OTLP gRPC exporter allocates memory for channel stubs and TLS -> initialization. The worker thread stack dominates the static footprint. +> stack size on Linux is ~8 MB. The OTLP/HTTP exporter allocates a small client and TLS +> initialization buffer. The worker thread stack dominates the static footprint. ### 3.5.2 Dynamic Memory diff --git a/OpenTelemetryPlan/05-configuration-reference.md b/OpenTelemetryPlan/05-configuration-reference.md index a81add88d7..9ea29ade1d 100644 --- a/OpenTelemetryPlan/05-configuration-reference.md +++ b/OpenTelemetryPlan/05-configuration-reference.md @@ -162,10 +162,10 @@ flowchart TB **Reading the diagram:** -- **Configuration Sources**: `xrpld.cfg` provides runtime settings (endpoint, sampling) while the CMake flag controls whether telemetry is compiled in at all. +- **Configuration Sources**: `xrpld.cfg` provides runtime settings (endpoint, per-component trace toggles) while the CMake flag controls whether telemetry is compiled in at all. Head sampling is fixed at 1.0 and is not a config option; volume reduction happens via tail sampling in the collector. - **Initialization**: `setup_Telemetry()` parses config values, then `make_Telemetry()` constructs the provider, processor, and exporter objects. - **Runtime Components**: The `TracerProvider` creates spans, the `BatchProcessor` buffers them, and the `OTLP Exporter` serializes and sends them over the wire. -- **OTLP arrow to Collector**: Trace data leaves the xrpld process via OTLP (gRPC or HTTP) and enters the external Collector pipeline. +- **OTLP arrow to Collector**: Trace data leaves the xrpld process via OTLP/HTTP and enters the external Collector pipeline. (OTLP/gRPC is future work — see design decisions §2.2.2.) - **Collector Pipeline**: `Receivers` ingest OTLP data, `Processors` apply sampling/filtering/enrichment, and `Exporters` forward traces to storage backends (Tempo, etc.). --- diff --git a/OpenTelemetryPlan/07-observability-backends.md b/OpenTelemetryPlan/07-observability-backends.md index cb17d2817a..4ebb6028fd 100644 --- a/OpenTelemetryPlan/07-observability-backends.md +++ b/OpenTelemetryPlan/07-observability-backends.md @@ -237,7 +237,7 @@ A Tempo-backed dashboard (uid `xrpld-consensus-health`) with four panels, all dr - **Proposers per Round** (stat): average of the `span.proposers` attribute on `consensus.round` spans. - **Recent Slow Rounds (>5s)** (table): `consensus.round` spans filtered to `duration > 5s`. -The underlying TraceQL queries are listed in section 7.7.3 and used throughout this doc. +Each panel's TraceQL query is described inline in its bullet above. ### 7.6.2 Node Overview Dashboard @@ -261,9 +261,9 @@ Grafana provisions three TraceQL-based alert rules (group `xrpld-tracing-alerts` - **RPC Error Rate Spike** (critical, `for: 2m`): fires when the error rate across `rpc.command.*` spans exceeds 5%. Error _rate_ is a ratio, so it must divide the error-span rate by the total-span rate — a single TraceQL `rate()` returns spans/second, not a percentage, and would fire on traffic volume alone. This uses span metrics emitted by the collector's `spanmetrics` connector (Prometheus datasource), not a TraceQL query: ``` - sum(rate(calls_total{service_name="xrpld", span_name=~"rpc.command.*", status_code="STATUS_CODE_ERROR"}[5m])) + sum(rate(traces_spanmetrics_calls_total{service_name="xrpld", span_name=~"rpc.command.*", status_code="STATUS_CODE_ERROR"}[5m])) / - sum(rate(calls_total{service_name="xrpld", span_name=~"rpc.command.*"}[5m])) + sum(rate(traces_spanmetrics_calls_total{service_name="xrpld", span_name=~"rpc.command.*"}[5m])) > 0.05 ``` diff --git a/OpenTelemetryPlan/08-appendix.md b/OpenTelemetryPlan/08-appendix.md index ddff360695..c40988f4fd 100644 --- a/OpenTelemetryPlan/08-appendix.md +++ b/OpenTelemetryPlan/08-appendix.md @@ -182,12 +182,6 @@ flowchart TB | [07-observability-backends.md](./07-observability-backends.md) | Backend selection and architecture | | [08-appendix.md](./08-appendix.md) | Glossary, references, version history | -### Task Lists - -| Document | Description | -| ------------------------------------ | --------------------------------------------------- | -| [presentation.md](./presentation.md) | Presentation slides for OpenTelemetry plan overview | - --- _Previous: [Observability Backends](./07-observability-backends.md)_ | _Back to: [Overview](./OpenTelemetryPlan.md)_ diff --git a/OpenTelemetryPlan/OpenTelemetryPlan.md b/OpenTelemetryPlan/OpenTelemetryPlan.md index eec093e3e0..ff1f0abfb1 100644 --- a/OpenTelemetryPlan/OpenTelemetryPlan.md +++ b/OpenTelemetryPlan/OpenTelemetryPlan.md @@ -1,4 +1,4 @@ -# [OpenTelemetry](00-tracing-fundamentals.md) Distributed Tracing Implementation Plan for xrpld (xrpld) +# [OpenTelemetry](00-tracing-fundamentals.md) Distributed Tracing Implementation Plan for xrpld ## Executive Summary @@ -126,7 +126,7 @@ Key trace points span across transaction submission via RPC, peer-to-peer messag > **OTLP** = OpenTelemetry Protocol | **CNCF** = Cloud Native Computing Foundation -The OpenTelemetry C++ SDK is selected for its CNCF backing, active development, and native performance characteristics. Traces are exported via OTLP/gRPC (primary) or OTLP/HTTP (fallback) to an OpenTelemetry Collector, which provides flexible routing and sampling. +The OpenTelemetry C++ SDK is selected for its CNCF backing, active development, and native performance characteristics. Traces are exported via OTLP/HTTP to an OpenTelemetry Collector, which provides flexible routing and sampling. OTLP/gRPC is planned future work (see design decisions §2.2.2). Span naming follows a hierarchical `.` convention (e.g., `rpc.submit`, `tx.relay`, `consensus.round`). Context propagation uses W3C Trace Context headers for HTTP and embedded Protocol Buffer fields for P2P messages. The implementation coexists with existing PerfLog and Insight observability systems through correlation IDs. @@ -150,7 +150,7 @@ Performance optimization strategies include head sampling fixed at 100% (intenti > **OTLP** = OpenTelemetry Protocol | **APM** = Application Performance Monitoring -Configuration is handled through the `[telemetry]` section in `xrpld.cfg` with options for enabling/disabling, exporter selection, endpoint configuration, sampling ratios, and component-level filtering. CMake integration includes a `XRPL_ENABLE_TELEMETRY` option for compile-time control. +Configuration is handled through the `[telemetry]` section in `xrpld.cfg` with options for enabling/disabling, exporter selection, endpoint configuration, and component-level filtering. Head sampling is fixed at 1.0 (not operator-configurable); volume reduction is done by tail sampling in the collector. CMake integration includes a `XRPL_ENABLE_TELEMETRY` option for compile-time control. OpenTelemetry Collector configurations are provided for development and production (with tail-based sampling, Tempo, and Elastic APM). Docker Compose examples enable quick local development environment setup. diff --git a/OpenTelemetryPlan/presentation.md b/OpenTelemetryPlan/presentation.md deleted file mode 100644 index bc83f82fc2..0000000000 --- a/OpenTelemetryPlan/presentation.md +++ /dev/null @@ -1,682 +0,0 @@ -# OpenTelemetry Distributed Tracing for xrpld - ---- - -## Slide 1: Introduction - -> **CNCF** = Cloud Native Computing Foundation - -### What is OpenTelemetry? - -OpenTelemetry is an open-source, CNCF-backed observability framework for distributed tracing, metrics, and logs. - -### Why OpenTelemetry for xrpld? - -- **End-to-End Transaction Visibility**: Track transactions from submission → consensus → ledger inclusion -- **Cross-Node Correlation**: Follow requests across multiple independent nodes using a unique `trace_id` -- **Consensus Round Analysis**: Understand timing and behavior across validators -- **Incident Debugging**: Correlate events across distributed nodes during issues - -```mermaid -flowchart LR - A["Node A
tx.receive
trace_id: abc123"] --> B["Node B
tx.relay
trace_id: abc123"] --> C["Node C
tx.validate
trace_id: abc123"] --> D["Node D
ledger.apply
trace_id: abc123"] - - style A fill:#1565c0,stroke:#0d47a1,color:#fff - style B fill:#2e7d32,stroke:#1b5e20,color:#fff - style C fill:#2e7d32,stroke:#1b5e20,color:#fff - style D fill:#e65100,stroke:#bf360c,color:#fff -``` - -**Reading the diagram:** - -- **Node A (blue, leftmost)**: The originating node that first receives the transaction and assigns a new `trace_id: abc123`; this ID becomes the correlation key for the entire distributed trace. -- **Node B and Node C (green, middle)**: Relay and validation nodes — each creates its own span but carries the same `trace_id`, so their work is linked to the original submission without any central coordinator. -- **Node D (orange, rightmost)**: The final node that applies the transaction to the ledger; the trace now spans the full lifecycle from submission to ledger inclusion. -- **Left-to-right flow**: The horizontal progression shows the real-world message path — a transaction hops from node to node, and the shared `trace_id` stitches all hops into a single queryable trace. - -> **Trace ID: abc123** — All nodes share the same trace, enabling cross-node correlation. - ---- - -## Slide 2: OpenTelemetry vs Open Source Alternatives - -> **CNCF** = Cloud Native Computing Foundation - -| Feature | OpenTelemetry | Jaeger | Zipkin | SkyWalking | Pinpoint | Prometheus | -| ------------------- | ---------------- | ---------------- | ------------------ | ---------- | ---------- | ---------- | -| **Tracing** | YES | YES | YES | YES | YES | NO | -| **Metrics** | YES | NO | NO | YES | YES | YES | -| **Logs** | YES | NO | NO | YES | NO | NO | -| **C++ SDK** | YES Official | YES (Deprecated) | YES (Unmaintained) | NO | NO | YES | -| **Vendor Neutral** | YES Primary goal | NO | NO | NO | NO | NO | -| **Instrumentation** | Manual + Auto | Manual | Manual | Auto-first | Auto-first | Manual | -| **Backend** | Any (exporters) | Self | Self | Self | Self | Self | -| **CNCF Status** | Incubating | Graduated | NO | Incubating | NO | Graduated | - -> **Why OpenTelemetry?** It's the only actively maintained, full-featured C++ option with vendor neutrality — allowing export to Tempo, Prometheus, Grafana, or any commercial backend without changing instrumentation. - ---- - -## Slide 3: Adoption Scope — Traces Only (Current Plan) - -OpenTelemetry supports three signal types: **Traces**, **Metrics**, and **Logs**. xrpld already captures metrics (StatsD via Beast Insight) and logs (Journal/PerfLog). The question is: how much of OTel do we adopt? - -> **Scenario A**: Add distributed tracing. Keep StatsD for metrics and Journal for logs. - -```mermaid -flowchart LR - subgraph xrpld["xrpld Process"] - direction TB - OTel["OTel SDK
(Traces)"] - Insight["Beast Insight
(StatsD Metrics)"] - Journal["Journal + PerfLog
(Logging)"] - end - - OTel -->|"OTLP"| Collector["OTel Collector"] - Insight -->|"UDP"| StatsD["StatsD Server"] - Journal -->|"File I/O"| LogFile["perf.log / debug.log"] - - Collector --> Tempo["Tempo"] - StatsD --> Graphite["Graphite / Grafana"] - LogFile --> Loki["Loki (optional)"] - - style xrpld fill:#424242,stroke:#212121,color:#fff - style OTel fill:#2e7d32,stroke:#1b5e20,color:#fff - style Insight fill:#1565c0,stroke:#0d47a1,color:#fff - style Journal fill:#e65100,stroke:#bf360c,color:#fff - style Collector fill:#2e7d32,stroke:#1b5e20,color:#fff -``` - -| Aspect | Details | -| ------------------------------ | --------------------------------------------------------------------------------------------------------------- | -| **What changes for operators** | Deploy OTel Collector + trace backend. Existing StatsD and log pipelines stay as-is. | -| **Codebase impact** | New `Telemetry` module (~1500 LOC). Beast Insight and Journal untouched. | -| **New capabilities** | Cross-node trace correlation, span-based debugging, request lifecycle visibility. | -| **What we still can't do** | Correlate metrics with specific traces natively. StatsD metrics remain fire-and-forget with no trace exemplars. | -| **Maintenance burden** | Three separate observability systems to maintain (OTel + StatsD + Journal). | -| **Risk** | Lowest — additive change, no existing systems disturbed. | - ---- - -## Slide 4: Future Adoption — Metrics & Logs via OTel - -### Scenario B: + OTel Metrics (Replace StatsD) - -> Migrate StatsD to OTel Metrics API, exposing Prometheus-compatible metrics. Remove Beast Insight. - -```mermaid -flowchart LR - subgraph xrpld["xrpld Process"] - direction TB - OTel["OTel SDK
(Traces + Metrics)"] - Journal["Journal + PerfLog
(Logging)"] - end - - OTel -->|"OTLP"| Collector["OTel Collector"] - Journal -->|"File I/O"| LogFile["perf.log / debug.log"] - - Collector --> Tempo["Tempo
(Traces)"] - Collector --> Prom["Prometheus
(Metrics)"] - LogFile --> Loki["Loki (optional)"] - - style xrpld fill:#424242,stroke:#212121,color:#fff - style OTel fill:#2e7d32,stroke:#1b5e20,color:#fff - style Journal fill:#e65100,stroke:#bf360c,color:#fff - style Collector fill:#2e7d32,stroke:#1b5e20,color:#fff -``` - -- **Better metrics?** Yes — Prometheus gives native histograms (p50/p95/p99), multi-dimensional labels, and exemplars linking metric spikes to traces. -- **Codebase**: Remove `Beast::Insight` + `StatsDCollector` (~2000 LOC). Single SDK for traces and metrics. -- **Operator effort**: Rewrite dashboards from StatsD/Graphite queries to PromQL. Run both in parallel during transition. -- **Risk**: Medium — operators must migrate monitoring infrastructure. - -### Scenario C: + OTel Logs (Full Stack) - -> Also replace Journal logging with OTel Logs API. Single SDK for everything. - -```mermaid -flowchart LR - subgraph xrpld["xrpld Process"] - OTel["OTel SDK
(Traces + Metrics + Logs)"] - end - - OTel -->|"OTLP"| Collector["OTel Collector"] - - Collector --> Tempo["Tempo
(Traces)"] - Collector --> Prom["Prometheus
(Metrics)"] - Collector --> Loki["Loki / Elastic
(Logs)"] - - style xrpld fill:#424242,stroke:#212121,color:#fff - style OTel fill:#2e7d32,stroke:#1b5e20,color:#fff - style Collector fill:#2e7d32,stroke:#1b5e20,color:#fff -``` - -- **Structured logging**: OTel Logs API outputs structured records with `trace_id`, `span_id`, severity, and attributes by design. -- **Full correlation**: Every log line carries `trace_id`. Click trace → see logs. Click metric spike → see trace → see logs. -- **Codebase**: Remove Beast Insight (~2000 LOC) + simplify Journal/PerfLog (~3000 LOC). One dependency instead of three. -- **Risk**: Highest — `beast::Journal` is deeply embedded in every component. Large refactor. OTel C++ Logs API is newer (stable since v1.11, less battle-tested). - -### Recommendation - -```mermaid -flowchart LR - A["Phase 1
Traces Only
(Current Plan)"] --> B["Phase 2
+ Metrics
(Replace StatsD)"] --> C["Phase 3
+ Logs
(Full OTel)"] - - style A fill:#2e7d32,stroke:#1b5e20,color:#fff - style B fill:#1565c0,stroke:#0d47a1,color:#fff - style C fill:#e65100,stroke:#bf360c,color:#fff -``` - -| Phase | Signal | Strategy | Risk | -| -------------------- | --------- | -------------------------------------------------------------- | ------ | -| **Phase 1** (now) | Traces | Add OTel traces. Keep StatsD and Journal. Prove value. | Low | -| **Phase 2** (future) | + Metrics | Migrate StatsD → Prometheus via OTel. Remove Beast Insight. | Medium | -| **Phase 3** (future) | + Logs | Adopt OTel Logs API. Align with structured logging initiative. | High | - -> **Key Takeaway**: Start with traces (unique value, lowest risk), then incrementally adopt metrics and logs as the OTel infrastructure proves itself. - ---- - -## Slide 5: Comparison with xrpld's Existing Solutions - -### Current Observability Stack - -| Aspect | PerfLog (JSON) | StatsD (Metrics) | OpenTelemetry (NEW) | -| --------------------- | --------------------- | --------------------- | --------------------------- | -| **Type** | Logging | Metrics | Distributed Tracing | -| **Scope** | Single node | Single node | **Cross-node** | -| **Data** | JSON log entries | Counters, gauges | Spans with context | -| **Correlation** | By timestamp | By metric name | By `trace_id` | -| **Overhead** | Low (file I/O) | Low (UDP) | Low-Medium (configurable) | -| **Question Answered** | "What happened here?" | "How many? How fast?" | **"What was the journey?"** | - -### Use Case Matrix - -| Scenario | PerfLog | StatsD | OpenTelemetry | -| -------------------------------- | ------- | ------ | ------------- | -| "How many TXs per second?" | ❌ | ✅ | ❌ | -| "Why was this specific TX slow?" | ⚠️ | ❌ | ✅ | -| "Which node delayed consensus?" | ❌ | ❌ | ✅ | -| "Show TX journey across 5 nodes" | ❌ | ❌ | ✅ | - -> **Key Insight**: In the **traces-only** approach (Phase 1), OpenTelemetry **complements** existing systems. In future phases, OTel metrics and logs could **replace** StatsD and Journal respectively — see Slides 3-4 for the full adoption roadmap. - ---- - -## Slide 6: Architecture - -> **OTLP** = OpenTelemetry Protocol | **WS** = WebSocket - -### High-Level Integration Architecture - -```mermaid -flowchart TB - subgraph xrpld["xrpld Node"] - subgraph services["Core Services"] - direction LR - RPC["RPC Server
(HTTP/WS)"] ~~~ Overlay["Overlay
(P2P Network)"] ~~~ Consensus["Consensus
(RCLConsensus)"] - end - - Telemetry["Telemetry Module
(OpenTelemetry SDK)"] - - services --> Telemetry - end - - Telemetry -->|OTLP/HTTP| Collector["OTel Collector"] - - Collector --> Tempo["Grafana Tempo"] - Collector --> Elastic["Elastic APM"] - - style xrpld fill:#424242,stroke:#212121,color:#fff - style services fill:#1565c0,stroke:#0d47a1,color:#fff - style Telemetry fill:#2e7d32,stroke:#1b5e20,color:#fff - style Collector fill:#e65100,stroke:#bf360c,color:#fff -``` - -**Reading the diagram:** - -- **Core Services (blue, top)**: RPC Server, Overlay, and Consensus are the three primary components that generate trace data — they represent the entry points for client requests, peer messages, and consensus rounds respectively. -- **Telemetry Module (green, middle)**: The OpenTelemetry SDK sits below the core services and receives span data from all three; it acts as a single collection point within the xrpld process. -- **OTel Collector (orange, center)**: An external process that receives spans over OTLP/HTTP from the Telemetry Module; it decouples xrpld from backend choices and handles batching, sampling, and routing. -- **Backends (bottom row)**: Tempo and Elastic APM are interchangeable — the Collector fans out to any combination, so operators can switch backends without modifying xrpld code. -- **Top-to-bottom flow**: Data flows from instrumented code down through the SDK, out over the network to the Collector, and finally into storage/visualization backends. - -### Context Propagation - -```mermaid -sequenceDiagram - participant Client - participant NodeA as Node A - participant NodeB as Node B - - Client->>NodeA: Submit TX (no context) - Note over NodeA: Creates trace_id: abc123
span: tx.receive - NodeA->>NodeB: Relay TX
(traceparent: abc123) - Note over NodeB: Links to trace_id: abc123
span: tx.relay -``` - -- **HTTP/RPC**: W3C Trace Context headers (`traceparent`) -- **P2P Messages**: Protocol Buffer extension fields - ---- - -## Slide 7: Implementation Plan - -### 5-Phase Rollout (9 Weeks) - -> **Note**: Dates shown are relative to project start, not calendar dates. - -```mermaid -gantt - title Implementation Timeline - dateFormat YYYY-MM-DD - axisFormat Week %W - - section Phase 1 - Core Infrastructure :p1, 2024-01-01, 2w - - section Phase 2 - RPC Tracing :p2, after p1, 2w - - section Phase 3 - Transaction Tracing :p3, after p2, 2w - - section Phase 4 - Consensus Tracing :p4, after p3, 2w - - section Phase 5 - Documentation :p5, after p4, 1w -``` - -### Phase Details - -| Phase | Focus | Key Deliverables | Effort | -| ----- | ------------------- | -------------------------------------------- | ------- | -| 1 | Core Infrastructure | SDK integration, Telemetry interface, Config | 10 days | -| 2 | RPC Tracing | HTTP context extraction, Handler spans | 10 days | -| 3 | Transaction Tracing | Protobuf context, P2P relay propagation | 10 days | -| 4 | Consensus Tracing | Round spans, Proposal/validation tracing | 10 days | -| 5 | Documentation | Runbook, Dashboards, Training | 7 days | - -**Total Effort**: ~47 developer-days (2 developers) - -> **Future Phases** (not in current scope): After traces are stable, OTel metrics can replace StatsD (~3 weeks), and OTel logs can replace Journal (~4 weeks, aligned with structured logging initiative). See Slides 3-4 for the full adoption roadmap. - ---- - -## Slide 8: Performance Overhead - -> **OTLP** = OpenTelemetry Protocol - -### Estimated System Impact - -| Metric | Overhead | Notes | -| ----------------- | ---------- | ------------------------------------------------ | -| **CPU** | 1-3% | Span creation and attribute setting | -| **Memory** | ~10 MB | SDK statics + batch buffer + worker thread stack | -| **Network** | 10-50 KB/s | Compressed OTLP export to collector | -| **Latency (p99)** | <2% | With proper sampling configuration | - -#### How We Arrived at These Numbers - -**Assumptions (XRPL mainnet baseline)**: - -| Parameter | Value | Source | -| ------------------------- | ---------------------- | --------------------------------------------------------------------------------------------------- | -| Transaction throughput | ~25 TPS (peaks to ~50) | Mainnet average | -| Default peers per node | 21 | `peerfinder/detail/Tuning.h` (`defaultMaxPeers`) | -| Consensus round frequency | ~1 round / 3-4 seconds | `ConsensusParms.h` (`ledgerMIN_CONSENSUS=1950ms`) | -| Proposers per round | ~20-35 | Mainnet UNL size | -| P2P message rate | ~160 msgs/sec | See message breakdown below | -| Avg TX processing time | ~200 μs | Profiled baseline | -| Single span creation cost | 500-1000 ns | OTel C++ SDK benchmarks (see [3.5.4](./03-implementation-strategy.md#354-performance-data-sources)) | - -**P2P message breakdown** (per node, mainnet): - -| Message Type | Rate | Derivation | -| ------------- | ------------ | --------------------------------------------------------------------- | -| TMTransaction | ~100/sec | ~25 TPS × ~4 relay hops per TX, deduplicated by HashRouter | -| TMValidation | ~50/sec | ~35 validators × ~1 validation/3s round ≈ ~12/sec, plus relay fan-out | -| TMProposeSet | ~10/sec | ~35 proposers / 3s round ≈ ~12/round, clustered in establish phase | -| **Total** | **~160/sec** | **Only traced message types counted** | - -**CPU (1-3%) — Calculation**: - -Per-transaction tracing cost breakdown: - -| Operation | Cost | Notes | -| ----------------------------------------------- | ----------- | ------------------------------------------ | -| `tx.receive` span (create + end + 4 attributes) | ~1400 ns | ~1000ns create + ~200ns end + 4×50ns attrs | -| `tx.validate` span | ~1200 ns | ~1000ns create + ~200ns for 2 attributes | -| `tx.relay` span | ~1200 ns | ~1000ns create + ~200ns for 2 attributes | -| Context injection into P2P message | ~200 ns | Serialize trace_id + span_id into protobuf | -| **Total per TX** | **~4.0 μs** | | - -> **CPU overhead**: 4.0 μs / 200 μs baseline = **~2.0% per transaction**. Under high load with consensus + RPC spans overlapping, reaches ~3%. Consensus itself adds only ~36 μs per 3-second round (~0.001%), so the TX path dominates. On production server hardware (3+ GHz Xeon), span creation drops to ~500-600 ns, bringing per-TX cost to ~2.6 μs (~1.3%). See [Section 3.5.4](./03-implementation-strategy.md#354-performance-data-sources) for benchmark sources. - -**Memory (~10 MB) — Calculation**: - -| Component | Size | Notes | -| --------------------------------------------- | ------------------ | ------------------------------------- | -| TracerProvider + Exporter (gRPC channel init) | ~320 KB | Allocated once at startup | -| BatchSpanProcessor (circular buffer) | ~16 KB | 2049 × 8-byte AtomicUniquePtr entries | -| BatchSpanProcessor (worker thread stack) | ~8 MB | Default Linux thread stack size | -| Active spans (in-flight, max ~1000) | ~500-800 KB | ~500-800 bytes/span × 1000 concurrent | -| Export queue (batch buffer, max 2048 spans) | ~1 MB | ~500 bytes/span × 2048 queue depth | -| Thread-local context storage (~100 threads) | ~6.4 KB | ~64 bytes/thread | -| **Total** | **~10 MB ceiling** | | - -> Memory plateaus once the export queue fills — the `max_queue_size=2048` config bounds growth. -> The worker thread stack (~8 MB) dominates the static footprint but is virtual memory; actual RSS -> depends on stack usage (typically much less). Active spans are larger than originally estimated -> (~500-800 bytes) because the OTel SDK `Span` object includes a mutex (~40 bytes), `SpanData` -> recordable (~250 bytes base), and `std::map`-based attribute storage (~200-500 bytes for 3-5 -> string attributes). See [Section 3.5.4](./03-implementation-strategy.md#354-performance-data-sources) for source references. - -> **Measured (perf-iac, telemetry on vs off, 9 nodes under payment load)**: the ~10 MB -> above is a theoretical SDK-footprint ceiling, dominated by virtual (not resident) thread-stack -> memory. In practice, per-node RSS showed **no measurable increase over the telemetry-off -> baseline** (~15 GiB mean / ~18–19 GiB peak on both sides), with no OOM, swap, or leak over the -> run. Treat memory overhead as negligible; the ceiling is a provisioning safety margin, not an -> expected increase. - -**Network (10-50 KB/s) — Calculation**: - -Two sources of network overhead: - -**(A) OTLP span export to Collector:** - -| Sampling Rate | Effective Spans/sec | Avg Span Size (compressed) | Bandwidth | -| -------------------------- | ------------------- | -------------------------- | ------------ | -| 100% (dev only) | ~500 | ~500 bytes | ~250 KB/s | -| **10% (recommended prod)** | **~50** | **~500 bytes** | **~25 KB/s** | -| 1% (minimal) | ~5 | ~500 bytes | ~2.5 KB/s | - -> The ~500 spans/sec at 100% comes from: ~100 TX spans + ~160 P2P context spans + ~23 consensus spans/round + ~50 RPC spans = ~500/sec. OTLP protobuf with gzip compression yields ~500 bytes/span average. - -**(B) P2P trace context overhead** (added to existing messages, always-on regardless of sampling): - -| Message Type | Rate | Context Size | Bandwidth | -| ------------- | -------- | ------------ | ------------- | -| TMTransaction | ~100/sec | 29 bytes | ~2.9 KB/s | -| TMValidation | ~50/sec | 29 bytes | ~1.5 KB/s | -| TMProposeSet | ~10/sec | 29 bytes | ~0.3 KB/s | -| **Total P2P** | | | **~4.7 KB/s** | - -> **Combined**: 25 KB/s (OTLP export at 10%) + 5 KB/s (P2P context) ≈ **~30 KB/s typical**. The 10-50 KB/s range covers 10-20% sampling under normal to peak mainnet load. - -**Latency (<2%) — Calculation**: - -| Path | Tracing Cost | Baseline | Overhead | -| ------------------------------ | ------------ | -------- | -------- | -| Fast RPC (e.g., `server_info`) | 2.75 μs | ~1 ms | 0.275% | -| Slow RPC (e.g., `path_find`) | 2.75 μs | ~100 ms | 0.003% | -| Transaction processing | 4.0 μs | ~200 μs | 2.0% | -| Consensus round | 36 μs | ~3 sec | 0.001% | - -> At p99, even the worst case (TX processing at 2.0%) is within the 1-3% range. RPC and consensus overhead are negligible. On production hardware, TX overhead drops to ~1.3%. - -### Per-Message Overhead (Context Propagation) - -Each P2P message carries trace context with the following overhead: - -| Field | Size | Description | -| ------------- | ------------- | ----------------------------------------- | -| `trace_id` | 16 bytes | Unique identifier for the entire trace | -| `span_id` | 8 bytes | Current span (becomes parent on receiver) | -| `trace_flags` | 1 byte | Sampling decision flags | -| `trace_state` | 0-4 bytes | Optional vendor-specific data | -| **Total** | **~29 bytes** | **Added per traced P2P message** | - -```mermaid -flowchart LR - subgraph msg["P2P Message with Trace Context"] - A["Original Message
(variable size)"] --> B["+ TraceContext
(~29 bytes)"] - end - - subgraph breakdown["Context Breakdown"] - C["trace_id
16 bytes"] - D["span_id
8 bytes"] - E["flags
1 byte"] - F["state
0-4 bytes"] - end - - B --> breakdown - - style A fill:#424242,stroke:#212121,color:#fff - style B fill:#2e7d32,stroke:#1b5e20,color:#fff - style C fill:#1565c0,stroke:#0d47a1,color:#fff - style D fill:#1565c0,stroke:#0d47a1,color:#fff - style E fill:#e65100,stroke:#bf360c,color:#fff - style F fill:#4a148c,stroke:#2e0d57,color:#fff -``` - -**Reading the diagram:** - -- **Original Message (gray, left)**: The existing P2P message payload of variable size — this is unchanged; trace context is appended, never modifying the original data. -- **+ TraceContext (green, right of message)**: The additional 29-byte context block attached to each traced message; the arrow from the original message shows it is a pure addition. -- **Context Breakdown (right subgraph)**: The four fields — `trace_id` (16 bytes), `span_id` (8 bytes), `flags` (1 byte), and `state` (0-4 bytes) — show exactly what is added and their individual sizes. -- **Color coding**: Blue fields (`trace_id`, `span_id`) are the core identifiers required for trace correlation; orange (`flags`) controls sampling decisions; purple (`state`) is optional vendor data typically omitted. - -> **Note**: 29 bytes represents ~1-6% overhead depending on message size (500B simple TX to 5KB proposal), which is acceptable for the observability benefits provided. - -### Mitigation Strategies - -```mermaid -flowchart LR - A["Head Sampling
fixed 1.0 (record all)"] --> B["Tail Sampling
Keep errors/slow"] --> C["Batch Export
Reduce I/O"] --> D["Conditional Compile
XRPL_ENABLE_TELEMETRY"] - - style A fill:#1565c0,stroke:#0d47a1,color:#fff - style B fill:#2e7d32,stroke:#1b5e20,color:#fff - style C fill:#e65100,stroke:#bf360c,color:#fff - style D fill:#4a148c,stroke:#2e0d57,color:#fff -``` - -> For a detailed explanation of head vs. tail sampling, see Slide 9. - -### Kill Switches (Rollback Options) - -1. **Config Disable**: Set `enabled=0` in config → instant disable, no restart needed for sampling -2. **Rebuild**: Compile with `XRPL_ENABLE_TELEMETRY=OFF` → zero overhead (no-op) -3. **Full Revert**: Clean separation allows easy commit reversion - ---- - -## Slide 9: Sampling Strategies — Head vs. Tail - -> Sampling controls **which traces are recorded and exported**. Without sampling, every operation generates a trace — at 500+ spans/sec, this overwhelms storage and network. Sampling lets you keep the signal, discard the noise. - -### Head Sampling (Decision at Start) - -The sampling decision is made **when a trace begins**, before any work is done. A random number is generated; if it falls within the configured ratio, the entire trace is recorded. Otherwise, the trace is silently dropped. - -```mermaid -flowchart LR - A["New Request
Arrives"] --> B{"Random < 10%?"} - B -->|"Yes (1 in 10)"| C["Record Entire Trace
(all spans)"] - B -->|"No (9 in 10)"| D["Drop Entire Trace
(zero overhead)"] - - style C fill:#2e7d32,stroke:#1b5e20,color:#fff - style D fill:#c62828,stroke:#8c2809,color:#fff - style B fill:#1565c0,stroke:#0d47a1,color:#fff -``` - -| Aspect | Details | -| ----------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **Where it runs** | Inside xrpld (SDK-level). In xrpld the ratio is fixed at 1.0 and not read from config (tail sampling in the collector needs every span) — the example below shows the general head-sampling mechanism. | -| **When the decision happens** | At trace creation time — before the first span is even populated. | -| **How it works** | `sampling_ratio=0.1` means each trace has a 10% probability of being recorded. Dropped traces incur near-zero overhead (no spans created, no attributes set, no export). | -| **Propagation** | Once a trace is sampled, the `trace_flags` field (1 byte in the context header) tells downstream nodes to also sample it. Unsampled traces propagate `trace_flags=0`, so downstream nodes skip them too. | -| **Pros** | Lowest overhead. Simple to configure. Predictable resource usage. | -| **Cons** | **Blind** — it doesn't know if the trace will be interesting. A rare error or slow consensus round has only a 10% chance of being captured. | -| **Best for** | High-volume, steady-state traffic where most traces look similar (e.g., routine RPC requests). | - -**xrpld configuration**: - -```ini -[telemetry] -# xrpld fixes head sampling at 1.0 (record every trace). This value is -# NOT read from config — the collector performs tail sampling instead, -# which needs all spans to arrive. See Slide 9 (tail sampling) and §7.4.2. -sampling_ratio=1.0 -``` - -### Tail Sampling (Decision at End) - -The sampling decision is made **after the trace completes**, based on its actual content — was it slow? Did it error? Was it a consensus round? This requires buffering complete traces before deciding. - -```mermaid -flowchart TB - A["All Traces
Buffered (100%)"] --> B["OTel Collector
Evaluates Rules"] - - B --> C{"Error?"} - C -->|Yes| K["KEEP"] - - C -->|No| D{"Slow?
(>5s consensus,
>1s RPC)"} - D -->|Yes| K - - D -->|No| E{"Random < 10%?"} - E -->|Yes| K - E -->|No| F["DROP"] - - style K fill:#2e7d32,stroke:#1b5e20,color:#fff - style F fill:#c62828,stroke:#8c2809,color:#fff - style B fill:#1565c0,stroke:#0d47a1,color:#fff - style C fill:#e65100,stroke:#bf360c,color:#fff - style D fill:#e65100,stroke:#bf360c,color:#fff - style E fill:#4a148c,stroke:#2e0d57,color:#fff -``` - -| Aspect | Details | -| ----------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **Where it runs** | In the **OTel Collector** (external process), not inside xrpld. xrpld exports 100% of traces; the Collector decides what to keep. | -| **When the decision happens** | After the Collector has received all spans for a trace (waits `decision_wait=10s` for stragglers). | -| **How it works** | Policy rules evaluate the completed trace: keep all errors, keep slow operations above a threshold, keep all consensus rounds, then probabilistically sample the rest at 10%. | -| **Pros** | **Never misses important traces**. Errors, slow requests, and consensus anomalies are always captured regardless of probability. | -| **Cons** | Higher resource usage — xrpld must export 100% of spans to the Collector, which buffers them in memory before deciding. The Collector needs more RAM (configured via `num_traces` and `decision_wait`). | -| **Best for** | Production troubleshooting where you can't afford to miss errors or anomalies. | - -**Collector configuration** (tail sampling rules for xrpld): - -```yaml -processors: - tail_sampling: - decision_wait: 10s # Wait for all spans in a trace - num_traces: 100000 # Buffer up to 100K concurrent traces - policies: - - name: errors # Always keep error traces - type: status_code - status_code: { status_codes: [ERROR] } - - - name: slow-consensus # Keep consensus rounds >5s - type: latency - latency: { threshold_ms: 5000 } - - - name: slow-rpc # Keep slow RPC requests >1s - type: latency - latency: { threshold_ms: 1000 } - - - name: probabilistic # Sample 10% of everything else - type: probabilistic - probabilistic: { sampling_percentage: 10 } -``` - -### Head vs. Tail — Side-by-Side - -| | Head Sampling | Tail Sampling | -| ----------------------------- | ----------------------------------------- | ------------------------------------------------ | -| **Decision point** | Trace start (inside xrpld) | Trace end (in OTel Collector) | -| **Knows trace content?** | No (random coin flip) | Yes (evaluates completed trace) | -| **Overhead on xrpld** | Lowest (dropped traces = no-op) | Higher (must export 100% to Collector) | -| **Collector resource usage** | Low (receives only sampled traces) | Higher (buffers all traces before deciding) | -| **Captures all errors?** | No (only if trace was randomly selected) | **Yes** (error policy catches them) | -| **Captures slow operations?** | No (random) | **Yes** (latency policy catches them) | -| **Configuration** | Fixed at `1.0` in xrpld (not config-read) | `otel-collector.yaml`: `tail_sampling` processor | -| **Best for** | High-throughput steady-state | Troubleshooting & anomaly detection | - -### Recommended Strategy for xrpld - -Use **both** in a layered approach: - -```mermaid -flowchart LR - subgraph xrpld["xrpld (Head Sampling)"] - HS["sampling_ratio=1.0
(export everything)"] - end - - subgraph collector["OTel Collector (Tail Sampling)"] - TS["Keep: errors + slow + 10% random
Drop: routine traces"] - end - - subgraph storage["Backend Storage"] - ST["Only interesting traces
stored long-term"] - end - - xrpld -->|"100% of spans"| collector -->|"~15-20% kept"| storage - - style xrpld fill:#424242,stroke:#212121,color:#fff - style collector fill:#1565c0,stroke:#0d47a1,color:#fff - style storage fill:#2e7d32,stroke:#1b5e20,color:#fff -``` - -> **Why this works**: xrpld exports everything (no blind drops), the Collector applies intelligent filtering (keep errors/slow/anomalies, sample the rest), and only ~15-20% of traces reach storage. xrpld's head sampling is fixed at 1.0 and not configurable, because tail sampling can only see traces that reach the Collector — any head drop would blind the error/slow policies. To reduce volume, tune the Collector's tail-sampling rules rather than adding head sampling. - ---- - -## Slide 10: Data Collection & Privacy - -### What Data is Collected - -| Category | Attributes Collected | Purpose | -| --------------- | -------------------------------------------------------------------------------------------------------------------- | --------------------------- | -| **Transaction** | `tx_hash`, `tx_type`, `tx_result`, `tx_fee`, `ledger_index` | Trace transaction lifecycle | -| **Consensus** | `consensus_round`, `consensus_phase`, `consensus_mode`, `proposers` (count of proposing validators), `round_time_ms` | Analyze consensus timing | -| **RPC** | `command`, `version`, `rpc_status`, `duration_ms` | Monitor RPC performance | -| **Peer** | `peer_id`(public key), `peer_latency_ms`, `message_type`, `message_size_bytes` | Network topology analysis | -| **Ledger** | `ledger_hash`, `ledger_index`, `close_time`, `ledger_tx_count` | Ledger progression tracking | -| **Job** | `job_type`, `job_queue_ms`, `job_worker` | JobQueue performance | - -### What is NOT Collected (Privacy Guarantees) - -```mermaid -flowchart LR - subgraph notCollected["❌ NOT Collected"] - direction LR - A["Private Keys"] ~~~ B["Account Balances"] ~~~ C["Transaction Amounts"] - end - - subgraph alsoNot["❌ Also Excluded"] - direction LR - D["IP Addresses
(configurable)"] ~~~ E["Personal Data"] ~~~ F["Raw TX Payloads"] - end - - style A fill:#c62828,stroke:#8c2809,color:#fff - style B fill:#c62828,stroke:#8c2809,color:#fff - style C fill:#c62828,stroke:#8c2809,color:#fff - style D fill:#c62828,stroke:#8c2809,color:#fff - style E fill:#c62828,stroke:#8c2809,color:#fff - style F fill:#c62828,stroke:#8c2809,color:#fff -``` - -**Reading the diagram:** - -- **NOT Collected (top row, red)**: Private Keys, Account Balances, and Transaction Amounts are explicitly excluded — these are financial/security-sensitive fields that telemetry never touches. -- **Also Excluded (bottom row, red)**: IP Addresses (configurable per deployment), Personal Data, and Raw TX Payloads are also excluded — these protect operator and user privacy. -- **All-red styling**: Every box is styled in red to visually reinforce that these are hard exclusions, not optional — the telemetry system has no code path to collect any of these fields. -- **Two-row layout**: The split between "NOT Collected" and "Also Excluded" distinguishes between financial data (top) and operational/personal data (bottom), making the privacy boundaries clear to auditors. - -### Privacy Protection Mechanisms - -| Mechanism | Description | -| -------------------------- | --------------------------------------------------------- | -| **Account Hashing** | `tx_account` is hashed at collector level before storage | -| **Configurable Redaction** | Sensitive fields can be excluded via config | -| **Sampling** | Only 10% of traces recorded by default (reduces exposure) | -| **Local Control** | Node operators control what gets exported | -| **No Raw Payloads** | Transaction content is never recorded, only metadata | - -> **Key Principle**: Telemetry collects **operational metadata** (timing, counts, hashes) — never **sensitive content** (keys, balances, amounts). - ---- - -_End of Presentation_ From 60a2ac7bfebce0d3e13bb08115766dbb45377352 Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Mon, 6 Jul 2026 19:17:34 +0100 Subject: [PATCH 3/4] fix(telemetry): bound RPC error-path span cardinality; balance LCOV markers Resolve the doCommand error-path span suffix and command attribute against the handler registry instead of using raw request input. A recognized command yields its canonical handler name; anything else (unregistered command, or neither command/method present) collapses to the literal "unknown". This bounds telemetry cardinality to the finite set of handler names plus "unknown" while preserving per-command error attribution for known-but-rejected commands (e.g. a submit that hit rpcTOO_BUSY stays rpc.command.submit). The command attribute is promoted to a Prometheus label by the spanmetrics connector, so attacker-controlled input previously enabled unbounded label cardinality. Also add the missing LCOV_EXCL_STOP in the first SpanGuard::linkedSpan overload so its coverage exclusion no longer bleeds into the second overload's guards. Co-Authored-By: Claude Opus 4.8 --- src/libxrpl/telemetry/SpanGuard.cpp | 1 + src/xrpld/rpc/detail/RPCHandler.cpp | 48 ++++++++++++++++++----------- 2 files changed, 31 insertions(+), 18 deletions(-) diff --git a/src/libxrpl/telemetry/SpanGuard.cpp b/src/libxrpl/telemetry/SpanGuard.cpp index 720750d2e5..e0b552ecc2 100644 --- a/src/libxrpl/telemetry/SpanGuard.cpp +++ b/src/libxrpl/telemetry/SpanGuard.cpp @@ -227,6 +227,7 @@ SpanGuard::linkedSpan(std::string_view name) const return SpanGuard( std::make_unique(tracer->StartSpan( std::string(name), {}, {{spanCtx, {{kLinkTypeKey, kLinkTypeFollowsFrom}}}}, opts))); + // LCOV_EXCL_STOP } SpanGuard diff --git a/src/xrpld/rpc/detail/RPCHandler.cpp b/src/xrpld/rpc/detail/RPCHandler.cpp index 109c991e98..f914fac548 100644 --- a/src/xrpld/rpc/detail/RPCHandler.cpp +++ b/src/xrpld/rpc/detail/RPCHandler.cpp @@ -220,6 +220,31 @@ callMethod(JsonContext& context, Method method, std::string const& name, Object& } } +// Resolve the span suffix / command attribute for a request that failed in +// fillHandler. Returns the canonical handler name for a recognized command +// (a finite, bounded set) or the literal "unknown" for a request that omits +// both fields or names an unregistered command. The raw request value is +// deliberately NOT used: the command attribute is promoted to a Prometheus +// label by the spanmetrics connector, so an attacker-controlled string would +// let arbitrary request input drive unbounded span-name / label cardinality. +// Resolving against the registry keeps per-command error attribution for real +// commands (e.g. a submit rejected with rpcTOO_BUSY stays rpc.command.submit) +// while collapsing garbage input to a single series. +std::string_view +resolveCommandSpanName(JsonContext const& context) +{ + if (!context.params.isMember(jss::command) && !context.params.isMember(jss::method)) + return rpc_span::val::unknownCommand; + + std::string const cmd = context.params.isMember(jss::command) + ? context.params[jss::command].asString() + : context.params[jss::method].asString(); + + auto const* handler = getHandler(context.apiVersion, context.app.config().betaRpcApi, cmd); + return (handler != nullptr) ? std::string_view{handler->name} + : std::string_view{rpc_span::val::unknownCommand}; +} + } // namespace Status @@ -228,25 +253,12 @@ doCommand(RPC::JsonContext& context, json::Value& result) Handler const* handler = nullptr; if (auto error = fillHandler(context, handler)) { - std::string cmdName; - if (context.params.isMember(jss::command)) - { - cmdName = context.params[jss::command].asString(); - } - else if (context.params.isMember(jss::method)) - { - cmdName = context.params[jss::method].asString(); - } - else - { - cmdName = "unknown"; // LCOV_EXCL_LINE - } - // Use the resolved command name as the span suffix so dashboards - // can break out per-command error rates (e.g. rpc.command.submit - // for a submit that hit rpcTOO_BUSY). Falling back to a single - // "unknown" name only when the request truly omits both fields. + // Bound the span name and command attribute to the finite set of + // registered handler names (plus "unknown") — see the helper for why + // raw request input must not reach the telemetry pipeline. + auto const cmdName = resolveCommandSpanName(context); auto span = SpanGuard::span(TraceCategory::Rpc, rpc_span::prefix::command, cmdName); - span.setAttribute(rpc_span::attr::command, cmdName.c_str()); + span.setAttribute(rpc_span::attr::command, cmdName); span.setAttribute(rpc_span::attr::rpcStatus, rpc_span::val::error); span.setError(getErrorInfo(error).token.cStr()); From 7a6ff619e5a5b66fa34418736d301bb100611664 Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Mon, 6 Jul 2026 20:37:08 +0100 Subject: [PATCH 4/4] code review changes Signed-off-by: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> --- .../scripts/levelization/results/ordering.txt | 1 + OpenTelemetryPlan/02-design-decisions.md | 44 ++++++++++----- OpenTelemetryPlan/Phase2_taskList.md | 5 +- cmake/XrplCore.cmake | 3 + docker/telemetry/otel-collector-config.yaml | 11 +++- include/xrpl/telemetry/Redaction.h | 56 +++++++++++++++++++ src/libxrpl/telemetry/Redaction.cpp | 32 +++++++++++ src/tests/libxrpl/telemetry/Redaction.cpp | 44 +++++++++++++++ src/xrpld/rpc/detail/PathFindSpanNames.h | 2 - src/xrpld/rpc/detail/PathRequest.cpp | 1 - src/xrpld/rpc/detail/PathRequestManager.cpp | 17 +++--- src/xrpld/rpc/handlers/orderbook/PathFind.cpp | 6 +- .../rpc/handlers/orderbook/RipplePathFind.cpp | 6 +- 13 files changed, 195 insertions(+), 33 deletions(-) create mode 100644 include/xrpl/telemetry/Redaction.h create mode 100644 src/libxrpl/telemetry/Redaction.cpp create mode 100644 src/tests/libxrpl/telemetry/Redaction.cpp diff --git a/.github/scripts/levelization/results/ordering.txt b/.github/scripts/levelization/results/ordering.txt index 94d2ef76d5..934fbc67a9 100644 --- a/.github/scripts/levelization/results/ordering.txt +++ b/.github/scripts/levelization/results/ordering.txt @@ -47,6 +47,7 @@ libxrpl.shamap > xrpl.protocol libxrpl.shamap > xrpl.shamap libxrpl.telemetry > xrpl.basics libxrpl.telemetry > xrpl.config +libxrpl.telemetry > xrpl.protocol libxrpl.telemetry > xrpl.telemetry libxrpl.tx > xrpl.basics libxrpl.tx > xrpl.conditions diff --git a/OpenTelemetryPlan/02-design-decisions.md b/OpenTelemetryPlan/02-design-decisions.md index 7beb89d096..aef9705223 100644 --- a/OpenTelemetryPlan/02-design-decisions.md +++ b/OpenTelemetryPlan/02-design-decisions.md @@ -378,32 +378,46 @@ The following data is explicitly **excluded** from telemetry collection: #### Privacy Protection Mechanisms -| Mechanism | Description | -| ----------------------------- | ------------------------------------------------------------------------- | -| **Account Hashing** | `tx_account` is hashed at collector level before storage | -| **Configurable Redaction** | Sensitive fields can be excluded via `[telemetry]` config section | -| **Sampling** | Only 10% of traces recorded by default, reducing data exposure | -| **Local Control** | Node operators have full control over what gets exported | -| **No Raw Payloads** | Transaction content is never recorded, only metadata (hash, type, result) | -| **Collector-Level Filtering** | Additional redaction/hashing can be configured at OTel Collector | +| Mechanism | Description | +| ----------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| **Account Hashing** | Account addresses are hashed both SDK-side (`pathfind_source_account`, `pathfind_dest_account` — always hashed before emission) and again at the collector level, so raw addresses never reach storage | +| **Sampling** | Only 10% of traces recorded by default, reducing data exposure | +| **Local Control** | Node operators have full control over what gets exported | +| **No Raw Payloads** | Transaction content is never recorded, only metadata (hash, type, result) | +| **Collector-Level Filtering** | Additional redaction/hashing can be configured at OTel Collector | + +#### Account Address Hashing + +Account addresses are **always** hashed before they reach the telemetry +backend — there is no opt-out flag and therefore no insecure-by-default +failure mode. Protection is applied in two independent layers: + +1. **SDK-side** (this node): the path-finding RPC handlers call + `redactAccount()` (`xrpl::telemetry`, `Redaction.h`) before setting the + `pathfind_source_account` / `pathfind_dest_account` span attributes. The + helper emits the first 16 characters of `sha512Half(address)` as + lowercase hex — deterministic (spans for one account still correlate) + but non-reversible. +2. **Collector-side** (defense-in-depth): an `attributes/hash` processor in + the OpenTelemetry Collector re-hashes those same attributes, so any node + that emitted a raw value is still redacted before storage. #### Collector-Level Data Protection The OpenTelemetry Collector can be configured (via an `attributes` processor) to hash or redact sensitive attributes before export — for example, hashing -`tx_account`, deleting `peer_address` to drop IP addresses, and deleting -`params` to redact request parameters. +`pathfind_source_account` / `pathfind_dest_account`, deleting `peer_address` +to drop IP addresses, and deleting `params` to redact request parameters. #### Configuration Options for Privacy In `xrpld.cfg`, operators control data collection granularity through the `[telemetry]` section. Besides `enabled`, per-component toggles (`trace_transactions`, `trace_consensus`, `trace_rpc`, `trace_peer` — the last -often disabled due to high volume) select which spans are emitted, and -redaction flags (`redact_account` to hash account addresses, `redact_peer_address` -to remove peer IP addresses) control SDK-level redaction before export. - -> **Note**: The `redact_account` configuration in `xrpld.cfg` controls SDK-level redaction before export, while collector-level filtering (see [Collector-Level Data Protection](#collector-level-data-protection) above) provides an additional defense-in-depth layer. Both can operate independently. +often disabled due to high volume) select which spans are emitted. Account +address hashing is not configurable: addresses are hashed unconditionally by +the SDK helper described above, with collector-level hashing as a second +layer. > **Key Principle**: Telemetry collects **operational metadata** (timing, counts, hashes) — never **sensitive content** (keys, balances, amounts, raw payloads). diff --git a/OpenTelemetryPlan/Phase2_taskList.md b/OpenTelemetryPlan/Phase2_taskList.md index d3afd8633b..0e19be9e9c 100644 --- a/OpenTelemetryPlan/Phase2_taskList.md +++ b/OpenTelemetryPlan/Phase2_taskList.md @@ -169,11 +169,12 @@ Node health (`amendment_blocked`, `server_state`) is not part of the telemetry s | `rpc.process` | `batch_size` | int64 | `params.size()` (only when batch) | | `rpc.ws_message` | `command` | string | `jv[command]` or `jv[method]` | | `rpc.command.*` | `load_type` | string | `context.loadType.label()` | -| `pathfind.compute` | `pathfind_dest_amount` | string | `saDstAmount_.getFullText()` | | `pathfind.compute` | `pathfind_dest_currency` | string | `to_string(saDstAmount_.asset())` | | `pathfind.discover` | `pathfind_num_source_assets` | int64 | `sourceAssets.size()` | -**New attr keys**: `RpcSpanNames.h` (`isBatch`, `batchSize`, `loadType`), `PathFindSpanNames.h` (`destAmount`, `destCurrency`, `numSourceAssets`). +_Note: `pathfind_dest_amount` was removed — the destination amount is a financial value excluded by the privacy policy (design §2.4.4)._ + +**New attr keys**: `RpcSpanNames.h` (`isBatch`, `batchSize`, `loadType`), `PathFindSpanNames.h` (`destCurrency`, `numSourceAssets`). **Modified files**: diff --git a/cmake/XrplCore.cmake b/cmake/XrplCore.cmake index b061a0b82a..99322d7fd0 100644 --- a/cmake/XrplCore.cmake +++ b/cmake/XrplCore.cmake @@ -206,10 +206,13 @@ target_link_libraries(xrpl.libxrpl.tx PUBLIC xrpl.libxrpl.ledger) # When telemetry=ON, links the Conan-provided umbrella target # opentelemetry-cpp::opentelemetry-cpp (individual component targets like # ::api, ::sdk are not available in the Conan package). +# +# Links xrpl.libxrpl.protocol PRIVATELY for sha512Half (digest.h) add_module(xrpl telemetry) target_link_libraries( xrpl.libxrpl.telemetry PUBLIC xrpl.libxrpl.basics xrpl.libxrpl.beast xrpl.libxrpl.config + PRIVATE xrpl.libxrpl.protocol ) if(telemetry) target_link_libraries( diff --git a/docker/telemetry/otel-collector-config.yaml b/docker/telemetry/otel-collector-config.yaml index 957adf036e..4f6f4f1ae4 100644 --- a/docker/telemetry/otel-collector-config.yaml +++ b/docker/telemetry/otel-collector-config.yaml @@ -48,6 +48,15 @@ processors: action: delete - key: telemetry.sdk.version action: delete + # Defense-in-depth: hash path-finding account attributes. The xrpld SDK + # already hashes these before export, but a node that emitted raw values + # is caught here so raw addresses never reach the backend. + attributes/hash: + actions: + - key: pathfind_source_account + action: hash + - key: pathfind_dest_account + action: hash exporters: debug: @@ -66,5 +75,5 @@ service: pipelines: traces: receivers: [otlp] - processors: [resource/tier, resource/stripsdk, batch] + processors: [resource/tier, resource/stripsdk, attributes/hash, batch] exporters: [debug, otlp/tempo] diff --git a/include/xrpl/telemetry/Redaction.h b/include/xrpl/telemetry/Redaction.h new file mode 100644 index 0000000000..50ab17efe2 --- /dev/null +++ b/include/xrpl/telemetry/Redaction.h @@ -0,0 +1,56 @@ +#pragma once + +/** Account-address redaction for telemetry span attributes. + + Path-finding RPC handlers would otherwise emit the caller's raw + account addresses as span attributes. To keep addresses out of the + telemetry backend, they are hashed at the point of emission. This + header exposes a single pure helper that turns an address into a + short, stable, non-reversible token. + + Data flow: + + handler -> redactAccount(addr) -> span attribute -> OTLP export + + The returned token is the first 16 hex characters (lowercase) of the + SHA-512Half digest of the address. It is deterministic (same address + always maps to the same token) so operators can still correlate spans + for a given account, but the original address cannot be recovered. + + A second, independent hashing layer runs in the OpenTelemetry + Collector (an `attributes/hash` processor) as defense-in-depth for + any node that emits a raw value. + + @note This function is pure and reentrant: it holds no global state, + performs no I/O, and is safe to call concurrently from any thread. + + Usage example: + @code + #include + using namespace xrpl::telemetry; + + span.setAttribute( + pathfind_span::attr::sourceAccount, redactAccount(src.asString())); + @endcode + + Edge case (empty input yields empty output): + @code + assert(redactAccount("") == ""); + @endcode +*/ + +#include +#include + +namespace xrpl::telemetry { + +/** Hash an account address into a short, stable, non-reversible token. + + @param addr The account address to redact (e.g. an r-address). + @return The first 16 lowercase hex characters of sha512Half(addr), + or an empty string when @p addr is empty. +*/ +[[nodiscard]] std::string +redactAccount(std::string_view addr); + +} // namespace xrpl::telemetry diff --git a/src/libxrpl/telemetry/Redaction.cpp b/src/libxrpl/telemetry/Redaction.cpp new file mode 100644 index 0000000000..ee15b8812e --- /dev/null +++ b/src/libxrpl/telemetry/Redaction.cpp @@ -0,0 +1,32 @@ +#include + +#include +#include +#include + +#include +#include +#include +#include + +namespace xrpl::telemetry { + +std::string +redactAccount(std::string_view addr) +{ + // Empty in, empty out: nothing to hash and no token to emit. + if (addr.empty()) + return {}; + + // sha512Half yields a uint256; to_string renders it as uppercase hex. + // Keep the first 16 chars (64 bits) — enough to correlate spans while + // staying non-reversible — and lowercase them for a stable token. + auto const digest = sha512Half(Slice(addr.data(), addr.size())); + std::string token = to_string(digest).substr(0, 16); + std::transform(token.begin(), token.end(), token.begin(), [](unsigned char c) { + return static_cast(std::tolower(c)); + }); + return token; +} + +} // namespace xrpl::telemetry diff --git a/src/tests/libxrpl/telemetry/Redaction.cpp b/src/tests/libxrpl/telemetry/Redaction.cpp new file mode 100644 index 0000000000..7f16c0f1dc --- /dev/null +++ b/src/tests/libxrpl/telemetry/Redaction.cpp @@ -0,0 +1,44 @@ +#include + +#include + +#include +#include +#include + +using namespace xrpl; +using namespace xrpl::telemetry; + +// Empty input must produce empty output (edge / negative path). +TEST(Redaction, empty_input_returns_empty) +{ + EXPECT_EQ(redactAccount(""), ""); +} + +// Success path: assert the EXACT expected token. The value is the first +// 16 chars of sha512Half(addr) rendered as lowercase hex, computed once +// and hard-coded here so any change to the hashing recipe is caught. +TEST(Redaction, known_account_exact_hash) +{ + EXPECT_EQ(redactAccount("rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh"), "a513895f49311f54"); +} + +// Structural invariants: length, lowercase-hex alphabet, and stability. +TEST(Redaction, token_is_16_lowercase_hex_and_stable) +{ + auto const token = redactAccount("rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh"); + EXPECT_EQ(token.size(), 16u); + EXPECT_TRUE(std::all_of(token.begin(), token.end(), [](unsigned char c) { + return std::isdigit(c) || (c >= 'a' && c <= 'f'); + })); + // Deterministic: hashing the same input twice yields the same token. + EXPECT_EQ(token, redactAccount("rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh")); +} + +// Distinct inputs must produce distinct tokens (no accidental constant). +TEST(Redaction, distinct_inputs_distinct_tokens) +{ + EXPECT_NE( + redactAccount("rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh"), + redactAccount("rDifferentAccount1234567890abcdefgh")); +} diff --git a/src/xrpld/rpc/detail/PathFindSpanNames.h b/src/xrpld/rpc/detail/PathFindSpanNames.h index 3a12937fbc..71eb805ea3 100644 --- a/src/xrpld/rpc/detail/PathFindSpanNames.h +++ b/src/xrpld/rpc/detail/PathFindSpanNames.h @@ -84,8 +84,6 @@ inline constexpr auto numPaths = makeStr("pathfind_num_paths"); inline constexpr auto numRequests = makeStr("pathfind_num_requests"); /// "pathfind_ledger_index" — pathfind target ledger index. inline constexpr auto ledgerIndex = makeStr("pathfind_ledger_index"); -/// "pathfind_dest_amount" — requested destination amount as string. -inline constexpr auto destAmount = makeStr("pathfind_dest_amount"); /// "pathfind_dest_currency" — destination currency code. inline constexpr auto destCurrency = makeStr("pathfind_dest_currency"); /// "pathfind_num_source_assets" — candidate source assets count. diff --git a/src/xrpld/rpc/detail/PathRequest.cpp b/src/xrpld/rpc/detail/PathRequest.cpp index 324036d39b..4fb11ab10c 100644 --- a/src/xrpld/rpc/detail/PathRequest.cpp +++ b/src/xrpld/rpc/detail/PathRequest.cpp @@ -742,7 +742,6 @@ PathRequest::doUpdate( auto span = SpanGuard::span( TraceCategory::Rpc, pathfind_span::prefix::pathfind, pathfind_span::op::compute); span.setAttribute(pathfind_span::attr::fast, fast); - span.setAttribute(pathfind_span::attr::destAmount, saDstAmount_.getFullText().c_str()); span.setAttribute(pathfind_span::attr::destCurrency, to_string(saDstAmount_.asset()).c_str()); JLOG(journal_.debug()) << iIdentifier_ << " update " << (fast ? "fast" : "normal"); diff --git a/src/xrpld/rpc/detail/PathRequestManager.cpp b/src/xrpld/rpc/detail/PathRequestManager.cpp index b2831c5518..01df901ce3 100644 --- a/src/xrpld/rpc/detail/PathRequestManager.cpp +++ b/src/xrpld/rpc/detail/PathRequestManager.cpp @@ -73,15 +73,16 @@ PathRequestManager::updateAll(std::shared_ptr const& inLedger) cache = getAssetCache(inLedger, true); } - // updateAll runs on every ledger close; skip span emission entirely when - // there are no active path subscriptions to avoid a steady stream of empty - // spans at mainnet close cadence. - if (requests.empty()) - return; - using namespace telemetry; - auto span = SpanGuard::span( - TraceCategory::Rpc, pathfind_span::prefix::pathfind, pathfind_span::op::updateAll); + // updateAll runs on every ledger close. Skip span emission when there are + // no active path subscriptions, to avoid a steady stream of empty spans at + // mainnet close cadence. A null guard is used in that case; all other work + // still runs unchanged (notably the isNewPathRequest() flag reset below), + // so behaviour matches the pre-span code path. + auto span = requests.empty() + ? SpanGuard{} + : SpanGuard::span( + TraceCategory::Rpc, pathfind_span::prefix::pathfind, pathfind_span::op::updateAll); span.setAttribute(pathfind_span::attr::ledgerIndex, static_cast(inLedger->seq())); span.setAttribute(pathfind_span::attr::numRequests, static_cast(requests.size())); diff --git a/src/xrpld/rpc/handlers/orderbook/PathFind.cpp b/src/xrpld/rpc/handlers/orderbook/PathFind.cpp index cecd1083c9..f04756be15 100644 --- a/src/xrpld/rpc/handlers/orderbook/PathFind.cpp +++ b/src/xrpld/rpc/handlers/orderbook/PathFind.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include namespace xrpl { @@ -20,10 +21,11 @@ doPathFind(RPC::JsonContext& context) using namespace telemetry; auto span = SpanGuard::span( TraceCategory::Rpc, pathfind_span::prefix::pathfind, pathfind_span::op::request); + // Addresses are hashed before emission for privacy. if (auto const& src = context.params[jss::source_account]; src.isString()) - span.setAttribute(pathfind_span::attr::sourceAccount, src.asString()); + span.setAttribute(pathfind_span::attr::sourceAccount, redactAccount(src.asString())); if (auto const& dst = context.params[jss::destination_account]; dst.isString()) - span.setAttribute(pathfind_span::attr::destAccount, dst.asString()); + span.setAttribute(pathfind_span::attr::destAccount, redactAccount(dst.asString())); if (context.app.config().pathSearchMax == 0) return rpcError(RpcNotSupported); diff --git a/src/xrpld/rpc/handlers/orderbook/RipplePathFind.cpp b/src/xrpld/rpc/handlers/orderbook/RipplePathFind.cpp index b55ba2515f..e08ef5445c 100644 --- a/src/xrpld/rpc/handlers/orderbook/RipplePathFind.cpp +++ b/src/xrpld/rpc/handlers/orderbook/RipplePathFind.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -28,10 +29,11 @@ doRipplePathFind(RPC::JsonContext& context) using namespace telemetry; auto span = SpanGuard::span( TraceCategory::Rpc, pathfind_span::prefix::pathfind, pathfind_span::op::request); + // Addresses are hashed before emission for privacy. if (auto const& src = context.params[jss::source_account]; src.isString()) - span.setAttribute(pathfind_span::attr::sourceAccount, src.asString()); + span.setAttribute(pathfind_span::attr::sourceAccount, redactAccount(src.asString())); if (auto const& dst = context.params[jss::destination_account]; dst.isString()) - span.setAttribute(pathfind_span::attr::destAccount, dst.asString()); + span.setAttribute(pathfind_span::attr::destAccount, redactAccount(dst.asString())); if (context.app.config().pathSearchMax == 0) return rpcError(RpcNotSupported);