From 0505ca35aadde1750ff54dccd1d20892a9fb5d55 Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Mon, 6 Jul 2026 17:41:38 +0100 Subject: [PATCH 1/2] fix(telemetry): make Tempo search filters dynamic Change the six remaining `type: static` search filters in tempo.yaml to `type: dynamic` so Grafana queries Tempo for available tag values instead of requiring manual free-text entry. Aligns service-name, node-id, network-type, span-name, span-status, and span-duration with the existing node-version and network-id filters. Co-Authored-By: Claude Opus 4.8 --- .../grafana/provisioning/datasources/tempo.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docker/telemetry/grafana/provisioning/datasources/tempo.yaml b/docker/telemetry/grafana/provisioning/datasources/tempo.yaml index 825d55453c..4214c72494 100644 --- a/docker/telemetry/grafana/provisioning/datasources/tempo.yaml +++ b/docker/telemetry/grafana/provisioning/datasources/tempo.yaml @@ -36,7 +36,7 @@ datasources: tag: service.name operator: "=" scope: resource - type: static + type: dynamic # service.instance.id: unique node identifier — defaults to the # node's public key (e.g., nHB1X37...). Distinguishes individual # nodes in a multi-node cluster or network. @@ -44,7 +44,7 @@ datasources: tag: service.instance.id operator: "=" scope: resource - type: static + type: dynamic # service.version: xrpld build version (e.g., "2.4.0-b1"). # Filter traces from specific software releases. - id: node-version @@ -66,7 +66,7 @@ datasources: tag: xrpl.network.type operator: "=" scope: resource - type: static + type: dynamic # --- Span intrinsic filters --- # name: the span operation name (e.g., "rpc.command.server_info"). # Use to find traces for a specific RPC command or subsystem. @@ -74,18 +74,18 @@ datasources: tag: name operator: "=" scope: intrinsic - type: static + type: dynamic # status: span completion status ("ok", "error", "unset"). # Filter for failed operations to diagnose errors. - id: span-status tag: status operator: "=" scope: intrinsic - type: static + type: dynamic # duration: span wall-clock duration. Use with ">" operator # to find slow operations (e.g., duration > 500ms). - id: span-duration tag: duration operator: ">" scope: intrinsic - type: static + type: dynamic From 26a85c764e099cb8036650c11b24e8aa7095dd16 Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Mon, 6 Jul 2026 18:15:23 +0100 Subject: [PATCH 2/2] code review comment addressed Signed-off-by: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> --- CONTRIBUTING.md | 2 +- OpenTelemetryPlan/02-design-decisions.md | 36 +- .../03-implementation-strategy.md | 8 +- .../05-configuration-reference.md | 4 +- .../07-observability-backends.md | 6 +- OpenTelemetryPlan/08-appendix.md | 6 - OpenTelemetryPlan/OpenTelemetryPlan.md | 6 +- OpenTelemetryPlan/presentation.md | 682 ------------------ 8 files changed, 33 insertions(+), 717 deletions(-) delete mode 100644 OpenTelemetryPlan/presentation.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 14c8d60117..6e2c2e47b3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -321,7 +321,7 @@ other layer must match them. A CI check enforces this end to end. 3. Collision qualifier: `_` — only when a bare name would collide with a DIFFERENT concept in the shared spanmetrics label space, or with the OTel-reserved `status` key (e.g. `rpc_status`, `grpc_status`, - `consensus_state`, `consensus_round`). This disambiguates distinct concepts + `consensus_phase`, `consensus_round`). This disambiguates distinct concepts that share a word; it is NOT used to tag the same concept with the workflow that emitted it — that is rule 2 (one shared name). 4. Resource attribute: dotted `xrpl..` — reserved ONLY for diff --git a/OpenTelemetryPlan/02-design-decisions.md b/OpenTelemetryPlan/02-design-decisions.md index a30df9e755..b26b26b461 100644 --- a/OpenTelemetryPlan/02-design-decisions.md +++ b/OpenTelemetryPlan/02-design-decisions.md @@ -185,7 +185,7 @@ agree with the code. A CI check enforces this end to end. 3. **Collision qualifier** → `_`, only when a bare name would collide with a DIFFERENT concept in the shared spanmetrics label space or with the OTel-reserved `status` key (e.g. `rpc_status`, `grpc_status`, - `consensus_state`, `consensus_round`, `consensus_mode`). This disambiguates + `consensus_phase`, `consensus_round`, `consensus_mode`). This disambiguates distinct concepts that share a word; it is NOT used to tag the same concept with its emitting workflow — that is rule 2 (one shared name). 4. **Resource attribute** → dotted `xrpl..`, reserved ONLY @@ -245,6 +245,8 @@ keys (the dotted form is reserved for resource scope per §2.3.3). | `tx_fee` | int64 | Fee in drops | | `tx_result` | string | `"tesSUCCESS"`, `"tecPATH_DRY"`, etc. | | `ledger_index` | int64 | Ledger containing transaction | +| `relay_count` | int64 | Peers the transaction was relayed to | +| `suppressed` | bool | `true` when HashRouter dropped a dup | #### Consensus Attributes @@ -261,12 +263,14 @@ keys (the dotted form is reserved for resource scope per §2.3.3). #### RPC Attributes -| Key | Type | Description | -| ---------- | ------ | ----------------------------------------------------- | -| `command` | string | Command name (per-span unique on `rpc.command`) | -| `version` | int64 | API version | -| `rpc_role` | string | `"admin"` or `"user"` (qualified — `role` is generic) | -| `params` | string | Sanitized parameters (optional) | +| Key | Type | Description | +| ------------- | ------- | ----------------------------------------------------------------------------- | +| `command` | string | Command name (per-span unique on `rpc.command`) | +| `version` | int64 | API version | +| `rpc_role` | string | `"admin"` or `"user"` (qualified — `role` is generic) | +| `params` | string | Sanitized parameters (optional) | +| `rpc_status` | string | Response status: `success` \| `error` (qualified — `status` is OTel-reserved) | +| `duration_ms` | float64 | Request duration in milliseconds | #### Peer & Message Attributes @@ -378,14 +382,14 @@ The following data is explicitly **excluded** from telemetry collection: #### Privacy Protection Mechanisms -| Mechanism | Description | -| ----------------------------- | ------------------------------------------------------------------------- | -| **Account Hashing** | `tx_account` is hashed at collector level before storage | -| **Configurable Redaction** | Sensitive fields can be excluded via `[telemetry]` config section | -| **Sampling** | Only 10% of traces recorded by default, reducing data exposure | -| **Local Control** | Node operators have full control over what gets exported | -| **No Raw Payloads** | Transaction content is never recorded, only metadata (hash, type, result) | -| **Collector-Level Filtering** | Additional redaction/hashing can be configured at OTel Collector | +| Mechanism | Description | +| ----------------------------- | --------------------------------------------------------------------------------------------------------------------------------------- | +| **Account Hashing** | `tx_account` is hashed at collector level before storage | +| **Configurable Redaction** | Sensitive fields can be excluded via `[telemetry]` config section | +| **Collector Tail Sampling** | xrpld head sampling is fixed at 1.0 (every span emitted); the collector retains ~10% of non-error traces, reducing stored data exposure | +| **Local Control** | Node operators have full control over what gets exported | +| **No Raw Payloads** | Transaction content is never recorded, only metadata (hash, type, result) | +| **Collector-Level Filtering** | Additional redaction/hashing can be configured at OTel Collector | #### Collector-Level Data Protection @@ -546,7 +550,7 @@ flowchart TB - **xrpld Process (dark gray)**: The single xrpld node running all three observability frameworks side by side. Each framework operates independently with no interference. - **PerfLog to perf.log**: PerfLog writes JSON-formatted event logs to a local file. Grafana can ingest these via Loki or a file-based datasource. - **Beast Insight to StatsD Server**: Insight sends aggregated metrics (counters, gauges) over UDP to a StatsD server. Grafana reads from StatsD-compatible backends like Graphite or Prometheus (via StatsD exporter). -- **OpenTelemetry to OTLP Collector**: OTel exports spans over OTLP/gRPC to a Collector, which then forwards to a trace backend (Tempo). +- **OpenTelemetry to OTLP Collector**: OTel exports spans over OTLP/HTTP to a Collector, which then forwards to a trace backend (Tempo). (OTLP/gRPC is future work — §2.2.2.) - **Grafana (red, unified UI)**: All three data streams converge in Grafana, enabling operators to correlate logs, metrics, and traces in a single dashboard. ### 2.6.5 Correlation with PerfLog diff --git a/OpenTelemetryPlan/03-implementation-strategy.md b/OpenTelemetryPlan/03-implementation-strategy.md index d51e30b211..4edcef6194 100644 --- a/OpenTelemetryPlan/03-implementation-strategy.md +++ b/OpenTelemetryPlan/03-implementation-strategy.md @@ -184,14 +184,14 @@ pie showData | TracerProvider singleton | ~64 KB | At startup | | BatchSpanProcessor (circular buffer) | ~16 KB | At startup | | BatchSpanProcessor (worker thread) | ~8 MB | At startup | -| OTLP exporter (gRPC channel init) | ~256 KB | At startup | +| OTLP/HTTP exporter (client init) | ~64 KB | At startup | | Propagator registry | ~8 KB | At startup | -| **Total static** | **~8.3 MB** | | +| **Total static** | **~8.1 MB** | | > **Why higher than earlier estimate**: The BatchSpanProcessor's circular buffer itself is only ~16 KB > (2049 x 8-byte `AtomicUniquePtr` entries), but it spawns a dedicated worker thread whose default -> stack size on Linux is ~8 MB. The OTLP gRPC exporter allocates memory for channel stubs and TLS -> initialization. The worker thread stack dominates the static footprint. +> stack size on Linux is ~8 MB. The OTLP/HTTP exporter allocates a small client and TLS +> initialization buffer. The worker thread stack dominates the static footprint. ### 3.5.2 Dynamic Memory diff --git a/OpenTelemetryPlan/05-configuration-reference.md b/OpenTelemetryPlan/05-configuration-reference.md index a81add88d7..9ea29ade1d 100644 --- a/OpenTelemetryPlan/05-configuration-reference.md +++ b/OpenTelemetryPlan/05-configuration-reference.md @@ -162,10 +162,10 @@ flowchart TB **Reading the diagram:** -- **Configuration Sources**: `xrpld.cfg` provides runtime settings (endpoint, sampling) while the CMake flag controls whether telemetry is compiled in at all. +- **Configuration Sources**: `xrpld.cfg` provides runtime settings (endpoint, per-component trace toggles) while the CMake flag controls whether telemetry is compiled in at all. Head sampling is fixed at 1.0 and is not a config option; volume reduction happens via tail sampling in the collector. - **Initialization**: `setup_Telemetry()` parses config values, then `make_Telemetry()` constructs the provider, processor, and exporter objects. - **Runtime Components**: The `TracerProvider` creates spans, the `BatchProcessor` buffers them, and the `OTLP Exporter` serializes and sends them over the wire. -- **OTLP arrow to Collector**: Trace data leaves the xrpld process via OTLP (gRPC or HTTP) and enters the external Collector pipeline. +- **OTLP arrow to Collector**: Trace data leaves the xrpld process via OTLP/HTTP and enters the external Collector pipeline. (OTLP/gRPC is future work — see design decisions §2.2.2.) - **Collector Pipeline**: `Receivers` ingest OTLP data, `Processors` apply sampling/filtering/enrichment, and `Exporters` forward traces to storage backends (Tempo, etc.). --- diff --git a/OpenTelemetryPlan/07-observability-backends.md b/OpenTelemetryPlan/07-observability-backends.md index cb17d2817a..4ebb6028fd 100644 --- a/OpenTelemetryPlan/07-observability-backends.md +++ b/OpenTelemetryPlan/07-observability-backends.md @@ -237,7 +237,7 @@ A Tempo-backed dashboard (uid `xrpld-consensus-health`) with four panels, all dr - **Proposers per Round** (stat): average of the `span.proposers` attribute on `consensus.round` spans. - **Recent Slow Rounds (>5s)** (table): `consensus.round` spans filtered to `duration > 5s`. -The underlying TraceQL queries are listed in section 7.7.3 and used throughout this doc. +Each panel's TraceQL query is described inline in its bullet above. ### 7.6.2 Node Overview Dashboard @@ -261,9 +261,9 @@ Grafana provisions three TraceQL-based alert rules (group `xrpld-tracing-alerts` - **RPC Error Rate Spike** (critical, `for: 2m`): fires when the error rate across `rpc.command.*` spans exceeds 5%. Error _rate_ is a ratio, so it must divide the error-span rate by the total-span rate — a single TraceQL `rate()` returns spans/second, not a percentage, and would fire on traffic volume alone. This uses span metrics emitted by the collector's `spanmetrics` connector (Prometheus datasource), not a TraceQL query: ``` - sum(rate(calls_total{service_name="xrpld", span_name=~"rpc.command.*", status_code="STATUS_CODE_ERROR"}[5m])) + sum(rate(traces_spanmetrics_calls_total{service_name="xrpld", span_name=~"rpc.command.*", status_code="STATUS_CODE_ERROR"}[5m])) / - sum(rate(calls_total{service_name="xrpld", span_name=~"rpc.command.*"}[5m])) + sum(rate(traces_spanmetrics_calls_total{service_name="xrpld", span_name=~"rpc.command.*"}[5m])) > 0.05 ``` diff --git a/OpenTelemetryPlan/08-appendix.md b/OpenTelemetryPlan/08-appendix.md index ddff360695..c40988f4fd 100644 --- a/OpenTelemetryPlan/08-appendix.md +++ b/OpenTelemetryPlan/08-appendix.md @@ -182,12 +182,6 @@ flowchart TB | [07-observability-backends.md](./07-observability-backends.md) | Backend selection and architecture | | [08-appendix.md](./08-appendix.md) | Glossary, references, version history | -### Task Lists - -| Document | Description | -| ------------------------------------ | --------------------------------------------------- | -| [presentation.md](./presentation.md) | Presentation slides for OpenTelemetry plan overview | - --- _Previous: [Observability Backends](./07-observability-backends.md)_ | _Back to: [Overview](./OpenTelemetryPlan.md)_ diff --git a/OpenTelemetryPlan/OpenTelemetryPlan.md b/OpenTelemetryPlan/OpenTelemetryPlan.md index eec093e3e0..ff1f0abfb1 100644 --- a/OpenTelemetryPlan/OpenTelemetryPlan.md +++ b/OpenTelemetryPlan/OpenTelemetryPlan.md @@ -1,4 +1,4 @@ -# [OpenTelemetry](00-tracing-fundamentals.md) Distributed Tracing Implementation Plan for xrpld (xrpld) +# [OpenTelemetry](00-tracing-fundamentals.md) Distributed Tracing Implementation Plan for xrpld ## Executive Summary @@ -126,7 +126,7 @@ Key trace points span across transaction submission via RPC, peer-to-peer messag > **OTLP** = OpenTelemetry Protocol | **CNCF** = Cloud Native Computing Foundation -The OpenTelemetry C++ SDK is selected for its CNCF backing, active development, and native performance characteristics. Traces are exported via OTLP/gRPC (primary) or OTLP/HTTP (fallback) to an OpenTelemetry Collector, which provides flexible routing and sampling. +The OpenTelemetry C++ SDK is selected for its CNCF backing, active development, and native performance characteristics. Traces are exported via OTLP/HTTP to an OpenTelemetry Collector, which provides flexible routing and sampling. OTLP/gRPC is planned future work (see design decisions §2.2.2). Span naming follows a hierarchical `.` convention (e.g., `rpc.submit`, `tx.relay`, `consensus.round`). Context propagation uses W3C Trace Context headers for HTTP and embedded Protocol Buffer fields for P2P messages. The implementation coexists with existing PerfLog and Insight observability systems through correlation IDs. @@ -150,7 +150,7 @@ Performance optimization strategies include head sampling fixed at 100% (intenti > **OTLP** = OpenTelemetry Protocol | **APM** = Application Performance Monitoring -Configuration is handled through the `[telemetry]` section in `xrpld.cfg` with options for enabling/disabling, exporter selection, endpoint configuration, sampling ratios, and component-level filtering. CMake integration includes a `XRPL_ENABLE_TELEMETRY` option for compile-time control. +Configuration is handled through the `[telemetry]` section in `xrpld.cfg` with options for enabling/disabling, exporter selection, endpoint configuration, and component-level filtering. Head sampling is fixed at 1.0 (not operator-configurable); volume reduction is done by tail sampling in the collector. CMake integration includes a `XRPL_ENABLE_TELEMETRY` option for compile-time control. OpenTelemetry Collector configurations are provided for development and production (with tail-based sampling, Tempo, and Elastic APM). Docker Compose examples enable quick local development environment setup. diff --git a/OpenTelemetryPlan/presentation.md b/OpenTelemetryPlan/presentation.md deleted file mode 100644 index bc83f82fc2..0000000000 --- a/OpenTelemetryPlan/presentation.md +++ /dev/null @@ -1,682 +0,0 @@ -# OpenTelemetry Distributed Tracing for xrpld - ---- - -## Slide 1: Introduction - -> **CNCF** = Cloud Native Computing Foundation - -### What is OpenTelemetry? - -OpenTelemetry is an open-source, CNCF-backed observability framework for distributed tracing, metrics, and logs. - -### Why OpenTelemetry for xrpld? - -- **End-to-End Transaction Visibility**: Track transactions from submission → consensus → ledger inclusion -- **Cross-Node Correlation**: Follow requests across multiple independent nodes using a unique `trace_id` -- **Consensus Round Analysis**: Understand timing and behavior across validators -- **Incident Debugging**: Correlate events across distributed nodes during issues - -```mermaid -flowchart LR - A["Node A
tx.receive
trace_id: abc123"] --> B["Node B
tx.relay
trace_id: abc123"] --> C["Node C
tx.validate
trace_id: abc123"] --> D["Node D
ledger.apply
trace_id: abc123"] - - style A fill:#1565c0,stroke:#0d47a1,color:#fff - style B fill:#2e7d32,stroke:#1b5e20,color:#fff - style C fill:#2e7d32,stroke:#1b5e20,color:#fff - style D fill:#e65100,stroke:#bf360c,color:#fff -``` - -**Reading the diagram:** - -- **Node A (blue, leftmost)**: The originating node that first receives the transaction and assigns a new `trace_id: abc123`; this ID becomes the correlation key for the entire distributed trace. -- **Node B and Node C (green, middle)**: Relay and validation nodes — each creates its own span but carries the same `trace_id`, so their work is linked to the original submission without any central coordinator. -- **Node D (orange, rightmost)**: The final node that applies the transaction to the ledger; the trace now spans the full lifecycle from submission to ledger inclusion. -- **Left-to-right flow**: The horizontal progression shows the real-world message path — a transaction hops from node to node, and the shared `trace_id` stitches all hops into a single queryable trace. - -> **Trace ID: abc123** — All nodes share the same trace, enabling cross-node correlation. - ---- - -## Slide 2: OpenTelemetry vs Open Source Alternatives - -> **CNCF** = Cloud Native Computing Foundation - -| Feature | OpenTelemetry | Jaeger | Zipkin | SkyWalking | Pinpoint | Prometheus | -| ------------------- | ---------------- | ---------------- | ------------------ | ---------- | ---------- | ---------- | -| **Tracing** | YES | YES | YES | YES | YES | NO | -| **Metrics** | YES | NO | NO | YES | YES | YES | -| **Logs** | YES | NO | NO | YES | NO | NO | -| **C++ SDK** | YES Official | YES (Deprecated) | YES (Unmaintained) | NO | NO | YES | -| **Vendor Neutral** | YES Primary goal | NO | NO | NO | NO | NO | -| **Instrumentation** | Manual + Auto | Manual | Manual | Auto-first | Auto-first | Manual | -| **Backend** | Any (exporters) | Self | Self | Self | Self | Self | -| **CNCF Status** | Incubating | Graduated | NO | Incubating | NO | Graduated | - -> **Why OpenTelemetry?** It's the only actively maintained, full-featured C++ option with vendor neutrality — allowing export to Tempo, Prometheus, Grafana, or any commercial backend without changing instrumentation. - ---- - -## Slide 3: Adoption Scope — Traces Only (Current Plan) - -OpenTelemetry supports three signal types: **Traces**, **Metrics**, and **Logs**. xrpld already captures metrics (StatsD via Beast Insight) and logs (Journal/PerfLog). The question is: how much of OTel do we adopt? - -> **Scenario A**: Add distributed tracing. Keep StatsD for metrics and Journal for logs. - -```mermaid -flowchart LR - subgraph xrpld["xrpld Process"] - direction TB - OTel["OTel SDK
(Traces)"] - Insight["Beast Insight
(StatsD Metrics)"] - Journal["Journal + PerfLog
(Logging)"] - end - - OTel -->|"OTLP"| Collector["OTel Collector"] - Insight -->|"UDP"| StatsD["StatsD Server"] - Journal -->|"File I/O"| LogFile["perf.log / debug.log"] - - Collector --> Tempo["Tempo"] - StatsD --> Graphite["Graphite / Grafana"] - LogFile --> Loki["Loki (optional)"] - - style xrpld fill:#424242,stroke:#212121,color:#fff - style OTel fill:#2e7d32,stroke:#1b5e20,color:#fff - style Insight fill:#1565c0,stroke:#0d47a1,color:#fff - style Journal fill:#e65100,stroke:#bf360c,color:#fff - style Collector fill:#2e7d32,stroke:#1b5e20,color:#fff -``` - -| Aspect | Details | -| ------------------------------ | --------------------------------------------------------------------------------------------------------------- | -| **What changes for operators** | Deploy OTel Collector + trace backend. Existing StatsD and log pipelines stay as-is. | -| **Codebase impact** | New `Telemetry` module (~1500 LOC). Beast Insight and Journal untouched. | -| **New capabilities** | Cross-node trace correlation, span-based debugging, request lifecycle visibility. | -| **What we still can't do** | Correlate metrics with specific traces natively. StatsD metrics remain fire-and-forget with no trace exemplars. | -| **Maintenance burden** | Three separate observability systems to maintain (OTel + StatsD + Journal). | -| **Risk** | Lowest — additive change, no existing systems disturbed. | - ---- - -## Slide 4: Future Adoption — Metrics & Logs via OTel - -### Scenario B: + OTel Metrics (Replace StatsD) - -> Migrate StatsD to OTel Metrics API, exposing Prometheus-compatible metrics. Remove Beast Insight. - -```mermaid -flowchart LR - subgraph xrpld["xrpld Process"] - direction TB - OTel["OTel SDK
(Traces + Metrics)"] - Journal["Journal + PerfLog
(Logging)"] - end - - OTel -->|"OTLP"| Collector["OTel Collector"] - Journal -->|"File I/O"| LogFile["perf.log / debug.log"] - - Collector --> Tempo["Tempo
(Traces)"] - Collector --> Prom["Prometheus
(Metrics)"] - LogFile --> Loki["Loki (optional)"] - - style xrpld fill:#424242,stroke:#212121,color:#fff - style OTel fill:#2e7d32,stroke:#1b5e20,color:#fff - style Journal fill:#e65100,stroke:#bf360c,color:#fff - style Collector fill:#2e7d32,stroke:#1b5e20,color:#fff -``` - -- **Better metrics?** Yes — Prometheus gives native histograms (p50/p95/p99), multi-dimensional labels, and exemplars linking metric spikes to traces. -- **Codebase**: Remove `Beast::Insight` + `StatsDCollector` (~2000 LOC). Single SDK for traces and metrics. -- **Operator effort**: Rewrite dashboards from StatsD/Graphite queries to PromQL. Run both in parallel during transition. -- **Risk**: Medium — operators must migrate monitoring infrastructure. - -### Scenario C: + OTel Logs (Full Stack) - -> Also replace Journal logging with OTel Logs API. Single SDK for everything. - -```mermaid -flowchart LR - subgraph xrpld["xrpld Process"] - OTel["OTel SDK
(Traces + Metrics + Logs)"] - end - - OTel -->|"OTLP"| Collector["OTel Collector"] - - Collector --> Tempo["Tempo
(Traces)"] - Collector --> Prom["Prometheus
(Metrics)"] - Collector --> Loki["Loki / Elastic
(Logs)"] - - style xrpld fill:#424242,stroke:#212121,color:#fff - style OTel fill:#2e7d32,stroke:#1b5e20,color:#fff - style Collector fill:#2e7d32,stroke:#1b5e20,color:#fff -``` - -- **Structured logging**: OTel Logs API outputs structured records with `trace_id`, `span_id`, severity, and attributes by design. -- **Full correlation**: Every log line carries `trace_id`. Click trace → see logs. Click metric spike → see trace → see logs. -- **Codebase**: Remove Beast Insight (~2000 LOC) + simplify Journal/PerfLog (~3000 LOC). One dependency instead of three. -- **Risk**: Highest — `beast::Journal` is deeply embedded in every component. Large refactor. OTel C++ Logs API is newer (stable since v1.11, less battle-tested). - -### Recommendation - -```mermaid -flowchart LR - A["Phase 1
Traces Only
(Current Plan)"] --> B["Phase 2
+ Metrics
(Replace StatsD)"] --> C["Phase 3
+ Logs
(Full OTel)"] - - style A fill:#2e7d32,stroke:#1b5e20,color:#fff - style B fill:#1565c0,stroke:#0d47a1,color:#fff - style C fill:#e65100,stroke:#bf360c,color:#fff -``` - -| Phase | Signal | Strategy | Risk | -| -------------------- | --------- | -------------------------------------------------------------- | ------ | -| **Phase 1** (now) | Traces | Add OTel traces. Keep StatsD and Journal. Prove value. | Low | -| **Phase 2** (future) | + Metrics | Migrate StatsD → Prometheus via OTel. Remove Beast Insight. | Medium | -| **Phase 3** (future) | + Logs | Adopt OTel Logs API. Align with structured logging initiative. | High | - -> **Key Takeaway**: Start with traces (unique value, lowest risk), then incrementally adopt metrics and logs as the OTel infrastructure proves itself. - ---- - -## Slide 5: Comparison with xrpld's Existing Solutions - -### Current Observability Stack - -| Aspect | PerfLog (JSON) | StatsD (Metrics) | OpenTelemetry (NEW) | -| --------------------- | --------------------- | --------------------- | --------------------------- | -| **Type** | Logging | Metrics | Distributed Tracing | -| **Scope** | Single node | Single node | **Cross-node** | -| **Data** | JSON log entries | Counters, gauges | Spans with context | -| **Correlation** | By timestamp | By metric name | By `trace_id` | -| **Overhead** | Low (file I/O) | Low (UDP) | Low-Medium (configurable) | -| **Question Answered** | "What happened here?" | "How many? How fast?" | **"What was the journey?"** | - -### Use Case Matrix - -| Scenario | PerfLog | StatsD | OpenTelemetry | -| -------------------------------- | ------- | ------ | ------------- | -| "How many TXs per second?" | ❌ | ✅ | ❌ | -| "Why was this specific TX slow?" | ⚠️ | ❌ | ✅ | -| "Which node delayed consensus?" | ❌ | ❌ | ✅ | -| "Show TX journey across 5 nodes" | ❌ | ❌ | ✅ | - -> **Key Insight**: In the **traces-only** approach (Phase 1), OpenTelemetry **complements** existing systems. In future phases, OTel metrics and logs could **replace** StatsD and Journal respectively — see Slides 3-4 for the full adoption roadmap. - ---- - -## Slide 6: Architecture - -> **OTLP** = OpenTelemetry Protocol | **WS** = WebSocket - -### High-Level Integration Architecture - -```mermaid -flowchart TB - subgraph xrpld["xrpld Node"] - subgraph services["Core Services"] - direction LR - RPC["RPC Server
(HTTP/WS)"] ~~~ Overlay["Overlay
(P2P Network)"] ~~~ Consensus["Consensus
(RCLConsensus)"] - end - - Telemetry["Telemetry Module
(OpenTelemetry SDK)"] - - services --> Telemetry - end - - Telemetry -->|OTLP/HTTP| Collector["OTel Collector"] - - Collector --> Tempo["Grafana Tempo"] - Collector --> Elastic["Elastic APM"] - - style xrpld fill:#424242,stroke:#212121,color:#fff - style services fill:#1565c0,stroke:#0d47a1,color:#fff - style Telemetry fill:#2e7d32,stroke:#1b5e20,color:#fff - style Collector fill:#e65100,stroke:#bf360c,color:#fff -``` - -**Reading the diagram:** - -- **Core Services (blue, top)**: RPC Server, Overlay, and Consensus are the three primary components that generate trace data — they represent the entry points for client requests, peer messages, and consensus rounds respectively. -- **Telemetry Module (green, middle)**: The OpenTelemetry SDK sits below the core services and receives span data from all three; it acts as a single collection point within the xrpld process. -- **OTel Collector (orange, center)**: An external process that receives spans over OTLP/HTTP from the Telemetry Module; it decouples xrpld from backend choices and handles batching, sampling, and routing. -- **Backends (bottom row)**: Tempo and Elastic APM are interchangeable — the Collector fans out to any combination, so operators can switch backends without modifying xrpld code. -- **Top-to-bottom flow**: Data flows from instrumented code down through the SDK, out over the network to the Collector, and finally into storage/visualization backends. - -### Context Propagation - -```mermaid -sequenceDiagram - participant Client - participant NodeA as Node A - participant NodeB as Node B - - Client->>NodeA: Submit TX (no context) - Note over NodeA: Creates trace_id: abc123
span: tx.receive - NodeA->>NodeB: Relay TX
(traceparent: abc123) - Note over NodeB: Links to trace_id: abc123
span: tx.relay -``` - -- **HTTP/RPC**: W3C Trace Context headers (`traceparent`) -- **P2P Messages**: Protocol Buffer extension fields - ---- - -## Slide 7: Implementation Plan - -### 5-Phase Rollout (9 Weeks) - -> **Note**: Dates shown are relative to project start, not calendar dates. - -```mermaid -gantt - title Implementation Timeline - dateFormat YYYY-MM-DD - axisFormat Week %W - - section Phase 1 - Core Infrastructure :p1, 2024-01-01, 2w - - section Phase 2 - RPC Tracing :p2, after p1, 2w - - section Phase 3 - Transaction Tracing :p3, after p2, 2w - - section Phase 4 - Consensus Tracing :p4, after p3, 2w - - section Phase 5 - Documentation :p5, after p4, 1w -``` - -### Phase Details - -| Phase | Focus | Key Deliverables | Effort | -| ----- | ------------------- | -------------------------------------------- | ------- | -| 1 | Core Infrastructure | SDK integration, Telemetry interface, Config | 10 days | -| 2 | RPC Tracing | HTTP context extraction, Handler spans | 10 days | -| 3 | Transaction Tracing | Protobuf context, P2P relay propagation | 10 days | -| 4 | Consensus Tracing | Round spans, Proposal/validation tracing | 10 days | -| 5 | Documentation | Runbook, Dashboards, Training | 7 days | - -**Total Effort**: ~47 developer-days (2 developers) - -> **Future Phases** (not in current scope): After traces are stable, OTel metrics can replace StatsD (~3 weeks), and OTel logs can replace Journal (~4 weeks, aligned with structured logging initiative). See Slides 3-4 for the full adoption roadmap. - ---- - -## Slide 8: Performance Overhead - -> **OTLP** = OpenTelemetry Protocol - -### Estimated System Impact - -| Metric | Overhead | Notes | -| ----------------- | ---------- | ------------------------------------------------ | -| **CPU** | 1-3% | Span creation and attribute setting | -| **Memory** | ~10 MB | SDK statics + batch buffer + worker thread stack | -| **Network** | 10-50 KB/s | Compressed OTLP export to collector | -| **Latency (p99)** | <2% | With proper sampling configuration | - -#### How We Arrived at These Numbers - -**Assumptions (XRPL mainnet baseline)**: - -| Parameter | Value | Source | -| ------------------------- | ---------------------- | --------------------------------------------------------------------------------------------------- | -| Transaction throughput | ~25 TPS (peaks to ~50) | Mainnet average | -| Default peers per node | 21 | `peerfinder/detail/Tuning.h` (`defaultMaxPeers`) | -| Consensus round frequency | ~1 round / 3-4 seconds | `ConsensusParms.h` (`ledgerMIN_CONSENSUS=1950ms`) | -| Proposers per round | ~20-35 | Mainnet UNL size | -| P2P message rate | ~160 msgs/sec | See message breakdown below | -| Avg TX processing time | ~200 μs | Profiled baseline | -| Single span creation cost | 500-1000 ns | OTel C++ SDK benchmarks (see [3.5.4](./03-implementation-strategy.md#354-performance-data-sources)) | - -**P2P message breakdown** (per node, mainnet): - -| Message Type | Rate | Derivation | -| ------------- | ------------ | --------------------------------------------------------------------- | -| TMTransaction | ~100/sec | ~25 TPS × ~4 relay hops per TX, deduplicated by HashRouter | -| TMValidation | ~50/sec | ~35 validators × ~1 validation/3s round ≈ ~12/sec, plus relay fan-out | -| TMProposeSet | ~10/sec | ~35 proposers / 3s round ≈ ~12/round, clustered in establish phase | -| **Total** | **~160/sec** | **Only traced message types counted** | - -**CPU (1-3%) — Calculation**: - -Per-transaction tracing cost breakdown: - -| Operation | Cost | Notes | -| ----------------------------------------------- | ----------- | ------------------------------------------ | -| `tx.receive` span (create + end + 4 attributes) | ~1400 ns | ~1000ns create + ~200ns end + 4×50ns attrs | -| `tx.validate` span | ~1200 ns | ~1000ns create + ~200ns for 2 attributes | -| `tx.relay` span | ~1200 ns | ~1000ns create + ~200ns for 2 attributes | -| Context injection into P2P message | ~200 ns | Serialize trace_id + span_id into protobuf | -| **Total per TX** | **~4.0 μs** | | - -> **CPU overhead**: 4.0 μs / 200 μs baseline = **~2.0% per transaction**. Under high load with consensus + RPC spans overlapping, reaches ~3%. Consensus itself adds only ~36 μs per 3-second round (~0.001%), so the TX path dominates. On production server hardware (3+ GHz Xeon), span creation drops to ~500-600 ns, bringing per-TX cost to ~2.6 μs (~1.3%). See [Section 3.5.4](./03-implementation-strategy.md#354-performance-data-sources) for benchmark sources. - -**Memory (~10 MB) — Calculation**: - -| Component | Size | Notes | -| --------------------------------------------- | ------------------ | ------------------------------------- | -| TracerProvider + Exporter (gRPC channel init) | ~320 KB | Allocated once at startup | -| BatchSpanProcessor (circular buffer) | ~16 KB | 2049 × 8-byte AtomicUniquePtr entries | -| BatchSpanProcessor (worker thread stack) | ~8 MB | Default Linux thread stack size | -| Active spans (in-flight, max ~1000) | ~500-800 KB | ~500-800 bytes/span × 1000 concurrent | -| Export queue (batch buffer, max 2048 spans) | ~1 MB | ~500 bytes/span × 2048 queue depth | -| Thread-local context storage (~100 threads) | ~6.4 KB | ~64 bytes/thread | -| **Total** | **~10 MB ceiling** | | - -> Memory plateaus once the export queue fills — the `max_queue_size=2048` config bounds growth. -> The worker thread stack (~8 MB) dominates the static footprint but is virtual memory; actual RSS -> depends on stack usage (typically much less). Active spans are larger than originally estimated -> (~500-800 bytes) because the OTel SDK `Span` object includes a mutex (~40 bytes), `SpanData` -> recordable (~250 bytes base), and `std::map`-based attribute storage (~200-500 bytes for 3-5 -> string attributes). See [Section 3.5.4](./03-implementation-strategy.md#354-performance-data-sources) for source references. - -> **Measured (perf-iac, telemetry on vs off, 9 nodes under payment load)**: the ~10 MB -> above is a theoretical SDK-footprint ceiling, dominated by virtual (not resident) thread-stack -> memory. In practice, per-node RSS showed **no measurable increase over the telemetry-off -> baseline** (~15 GiB mean / ~18–19 GiB peak on both sides), with no OOM, swap, or leak over the -> run. Treat memory overhead as negligible; the ceiling is a provisioning safety margin, not an -> expected increase. - -**Network (10-50 KB/s) — Calculation**: - -Two sources of network overhead: - -**(A) OTLP span export to Collector:** - -| Sampling Rate | Effective Spans/sec | Avg Span Size (compressed) | Bandwidth | -| -------------------------- | ------------------- | -------------------------- | ------------ | -| 100% (dev only) | ~500 | ~500 bytes | ~250 KB/s | -| **10% (recommended prod)** | **~50** | **~500 bytes** | **~25 KB/s** | -| 1% (minimal) | ~5 | ~500 bytes | ~2.5 KB/s | - -> The ~500 spans/sec at 100% comes from: ~100 TX spans + ~160 P2P context spans + ~23 consensus spans/round + ~50 RPC spans = ~500/sec. OTLP protobuf with gzip compression yields ~500 bytes/span average. - -**(B) P2P trace context overhead** (added to existing messages, always-on regardless of sampling): - -| Message Type | Rate | Context Size | Bandwidth | -| ------------- | -------- | ------------ | ------------- | -| TMTransaction | ~100/sec | 29 bytes | ~2.9 KB/s | -| TMValidation | ~50/sec | 29 bytes | ~1.5 KB/s | -| TMProposeSet | ~10/sec | 29 bytes | ~0.3 KB/s | -| **Total P2P** | | | **~4.7 KB/s** | - -> **Combined**: 25 KB/s (OTLP export at 10%) + 5 KB/s (P2P context) ≈ **~30 KB/s typical**. The 10-50 KB/s range covers 10-20% sampling under normal to peak mainnet load. - -**Latency (<2%) — Calculation**: - -| Path | Tracing Cost | Baseline | Overhead | -| ------------------------------ | ------------ | -------- | -------- | -| Fast RPC (e.g., `server_info`) | 2.75 μs | ~1 ms | 0.275% | -| Slow RPC (e.g., `path_find`) | 2.75 μs | ~100 ms | 0.003% | -| Transaction processing | 4.0 μs | ~200 μs | 2.0% | -| Consensus round | 36 μs | ~3 sec | 0.001% | - -> At p99, even the worst case (TX processing at 2.0%) is within the 1-3% range. RPC and consensus overhead are negligible. On production hardware, TX overhead drops to ~1.3%. - -### Per-Message Overhead (Context Propagation) - -Each P2P message carries trace context with the following overhead: - -| Field | Size | Description | -| ------------- | ------------- | ----------------------------------------- | -| `trace_id` | 16 bytes | Unique identifier for the entire trace | -| `span_id` | 8 bytes | Current span (becomes parent on receiver) | -| `trace_flags` | 1 byte | Sampling decision flags | -| `trace_state` | 0-4 bytes | Optional vendor-specific data | -| **Total** | **~29 bytes** | **Added per traced P2P message** | - -```mermaid -flowchart LR - subgraph msg["P2P Message with Trace Context"] - A["Original Message
(variable size)"] --> B["+ TraceContext
(~29 bytes)"] - end - - subgraph breakdown["Context Breakdown"] - C["trace_id
16 bytes"] - D["span_id
8 bytes"] - E["flags
1 byte"] - F["state
0-4 bytes"] - end - - B --> breakdown - - style A fill:#424242,stroke:#212121,color:#fff - style B fill:#2e7d32,stroke:#1b5e20,color:#fff - style C fill:#1565c0,stroke:#0d47a1,color:#fff - style D fill:#1565c0,stroke:#0d47a1,color:#fff - style E fill:#e65100,stroke:#bf360c,color:#fff - style F fill:#4a148c,stroke:#2e0d57,color:#fff -``` - -**Reading the diagram:** - -- **Original Message (gray, left)**: The existing P2P message payload of variable size — this is unchanged; trace context is appended, never modifying the original data. -- **+ TraceContext (green, right of message)**: The additional 29-byte context block attached to each traced message; the arrow from the original message shows it is a pure addition. -- **Context Breakdown (right subgraph)**: The four fields — `trace_id` (16 bytes), `span_id` (8 bytes), `flags` (1 byte), and `state` (0-4 bytes) — show exactly what is added and their individual sizes. -- **Color coding**: Blue fields (`trace_id`, `span_id`) are the core identifiers required for trace correlation; orange (`flags`) controls sampling decisions; purple (`state`) is optional vendor data typically omitted. - -> **Note**: 29 bytes represents ~1-6% overhead depending on message size (500B simple TX to 5KB proposal), which is acceptable for the observability benefits provided. - -### Mitigation Strategies - -```mermaid -flowchart LR - A["Head Sampling
fixed 1.0 (record all)"] --> B["Tail Sampling
Keep errors/slow"] --> C["Batch Export
Reduce I/O"] --> D["Conditional Compile
XRPL_ENABLE_TELEMETRY"] - - style A fill:#1565c0,stroke:#0d47a1,color:#fff - style B fill:#2e7d32,stroke:#1b5e20,color:#fff - style C fill:#e65100,stroke:#bf360c,color:#fff - style D fill:#4a148c,stroke:#2e0d57,color:#fff -``` - -> For a detailed explanation of head vs. tail sampling, see Slide 9. - -### Kill Switches (Rollback Options) - -1. **Config Disable**: Set `enabled=0` in config → instant disable, no restart needed for sampling -2. **Rebuild**: Compile with `XRPL_ENABLE_TELEMETRY=OFF` → zero overhead (no-op) -3. **Full Revert**: Clean separation allows easy commit reversion - ---- - -## Slide 9: Sampling Strategies — Head vs. Tail - -> Sampling controls **which traces are recorded and exported**. Without sampling, every operation generates a trace — at 500+ spans/sec, this overwhelms storage and network. Sampling lets you keep the signal, discard the noise. - -### Head Sampling (Decision at Start) - -The sampling decision is made **when a trace begins**, before any work is done. A random number is generated; if it falls within the configured ratio, the entire trace is recorded. Otherwise, the trace is silently dropped. - -```mermaid -flowchart LR - A["New Request
Arrives"] --> B{"Random < 10%?"} - B -->|"Yes (1 in 10)"| C["Record Entire Trace
(all spans)"] - B -->|"No (9 in 10)"| D["Drop Entire Trace
(zero overhead)"] - - style C fill:#2e7d32,stroke:#1b5e20,color:#fff - style D fill:#c62828,stroke:#8c2809,color:#fff - style B fill:#1565c0,stroke:#0d47a1,color:#fff -``` - -| Aspect | Details | -| ----------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **Where it runs** | Inside xrpld (SDK-level). In xrpld the ratio is fixed at 1.0 and not read from config (tail sampling in the collector needs every span) — the example below shows the general head-sampling mechanism. | -| **When the decision happens** | At trace creation time — before the first span is even populated. | -| **How it works** | `sampling_ratio=0.1` means each trace has a 10% probability of being recorded. Dropped traces incur near-zero overhead (no spans created, no attributes set, no export). | -| **Propagation** | Once a trace is sampled, the `trace_flags` field (1 byte in the context header) tells downstream nodes to also sample it. Unsampled traces propagate `trace_flags=0`, so downstream nodes skip them too. | -| **Pros** | Lowest overhead. Simple to configure. Predictable resource usage. | -| **Cons** | **Blind** — it doesn't know if the trace will be interesting. A rare error or slow consensus round has only a 10% chance of being captured. | -| **Best for** | High-volume, steady-state traffic where most traces look similar (e.g., routine RPC requests). | - -**xrpld configuration**: - -```ini -[telemetry] -# xrpld fixes head sampling at 1.0 (record every trace). This value is -# NOT read from config — the collector performs tail sampling instead, -# which needs all spans to arrive. See Slide 9 (tail sampling) and §7.4.2. -sampling_ratio=1.0 -``` - -### Tail Sampling (Decision at End) - -The sampling decision is made **after the trace completes**, based on its actual content — was it slow? Did it error? Was it a consensus round? This requires buffering complete traces before deciding. - -```mermaid -flowchart TB - A["All Traces
Buffered (100%)"] --> B["OTel Collector
Evaluates Rules"] - - B --> C{"Error?"} - C -->|Yes| K["KEEP"] - - C -->|No| D{"Slow?
(>5s consensus,
>1s RPC)"} - D -->|Yes| K - - D -->|No| E{"Random < 10%?"} - E -->|Yes| K - E -->|No| F["DROP"] - - style K fill:#2e7d32,stroke:#1b5e20,color:#fff - style F fill:#c62828,stroke:#8c2809,color:#fff - style B fill:#1565c0,stroke:#0d47a1,color:#fff - style C fill:#e65100,stroke:#bf360c,color:#fff - style D fill:#e65100,stroke:#bf360c,color:#fff - style E fill:#4a148c,stroke:#2e0d57,color:#fff -``` - -| Aspect | Details | -| ----------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **Where it runs** | In the **OTel Collector** (external process), not inside xrpld. xrpld exports 100% of traces; the Collector decides what to keep. | -| **When the decision happens** | After the Collector has received all spans for a trace (waits `decision_wait=10s` for stragglers). | -| **How it works** | Policy rules evaluate the completed trace: keep all errors, keep slow operations above a threshold, keep all consensus rounds, then probabilistically sample the rest at 10%. | -| **Pros** | **Never misses important traces**. Errors, slow requests, and consensus anomalies are always captured regardless of probability. | -| **Cons** | Higher resource usage — xrpld must export 100% of spans to the Collector, which buffers them in memory before deciding. The Collector needs more RAM (configured via `num_traces` and `decision_wait`). | -| **Best for** | Production troubleshooting where you can't afford to miss errors or anomalies. | - -**Collector configuration** (tail sampling rules for xrpld): - -```yaml -processors: - tail_sampling: - decision_wait: 10s # Wait for all spans in a trace - num_traces: 100000 # Buffer up to 100K concurrent traces - policies: - - name: errors # Always keep error traces - type: status_code - status_code: { status_codes: [ERROR] } - - - name: slow-consensus # Keep consensus rounds >5s - type: latency - latency: { threshold_ms: 5000 } - - - name: slow-rpc # Keep slow RPC requests >1s - type: latency - latency: { threshold_ms: 1000 } - - - name: probabilistic # Sample 10% of everything else - type: probabilistic - probabilistic: { sampling_percentage: 10 } -``` - -### Head vs. Tail — Side-by-Side - -| | Head Sampling | Tail Sampling | -| ----------------------------- | ----------------------------------------- | ------------------------------------------------ | -| **Decision point** | Trace start (inside xrpld) | Trace end (in OTel Collector) | -| **Knows trace content?** | No (random coin flip) | Yes (evaluates completed trace) | -| **Overhead on xrpld** | Lowest (dropped traces = no-op) | Higher (must export 100% to Collector) | -| **Collector resource usage** | Low (receives only sampled traces) | Higher (buffers all traces before deciding) | -| **Captures all errors?** | No (only if trace was randomly selected) | **Yes** (error policy catches them) | -| **Captures slow operations?** | No (random) | **Yes** (latency policy catches them) | -| **Configuration** | Fixed at `1.0` in xrpld (not config-read) | `otel-collector.yaml`: `tail_sampling` processor | -| **Best for** | High-throughput steady-state | Troubleshooting & anomaly detection | - -### Recommended Strategy for xrpld - -Use **both** in a layered approach: - -```mermaid -flowchart LR - subgraph xrpld["xrpld (Head Sampling)"] - HS["sampling_ratio=1.0
(export everything)"] - end - - subgraph collector["OTel Collector (Tail Sampling)"] - TS["Keep: errors + slow + 10% random
Drop: routine traces"] - end - - subgraph storage["Backend Storage"] - ST["Only interesting traces
stored long-term"] - end - - xrpld -->|"100% of spans"| collector -->|"~15-20% kept"| storage - - style xrpld fill:#424242,stroke:#212121,color:#fff - style collector fill:#1565c0,stroke:#0d47a1,color:#fff - style storage fill:#2e7d32,stroke:#1b5e20,color:#fff -``` - -> **Why this works**: xrpld exports everything (no blind drops), the Collector applies intelligent filtering (keep errors/slow/anomalies, sample the rest), and only ~15-20% of traces reach storage. xrpld's head sampling is fixed at 1.0 and not configurable, because tail sampling can only see traces that reach the Collector — any head drop would blind the error/slow policies. To reduce volume, tune the Collector's tail-sampling rules rather than adding head sampling. - ---- - -## Slide 10: Data Collection & Privacy - -### What Data is Collected - -| Category | Attributes Collected | Purpose | -| --------------- | -------------------------------------------------------------------------------------------------------------------- | --------------------------- | -| **Transaction** | `tx_hash`, `tx_type`, `tx_result`, `tx_fee`, `ledger_index` | Trace transaction lifecycle | -| **Consensus** | `consensus_round`, `consensus_phase`, `consensus_mode`, `proposers` (count of proposing validators), `round_time_ms` | Analyze consensus timing | -| **RPC** | `command`, `version`, `rpc_status`, `duration_ms` | Monitor RPC performance | -| **Peer** | `peer_id`(public key), `peer_latency_ms`, `message_type`, `message_size_bytes` | Network topology analysis | -| **Ledger** | `ledger_hash`, `ledger_index`, `close_time`, `ledger_tx_count` | Ledger progression tracking | -| **Job** | `job_type`, `job_queue_ms`, `job_worker` | JobQueue performance | - -### What is NOT Collected (Privacy Guarantees) - -```mermaid -flowchart LR - subgraph notCollected["❌ NOT Collected"] - direction LR - A["Private Keys"] ~~~ B["Account Balances"] ~~~ C["Transaction Amounts"] - end - - subgraph alsoNot["❌ Also Excluded"] - direction LR - D["IP Addresses
(configurable)"] ~~~ E["Personal Data"] ~~~ F["Raw TX Payloads"] - end - - style A fill:#c62828,stroke:#8c2809,color:#fff - style B fill:#c62828,stroke:#8c2809,color:#fff - style C fill:#c62828,stroke:#8c2809,color:#fff - style D fill:#c62828,stroke:#8c2809,color:#fff - style E fill:#c62828,stroke:#8c2809,color:#fff - style F fill:#c62828,stroke:#8c2809,color:#fff -``` - -**Reading the diagram:** - -- **NOT Collected (top row, red)**: Private Keys, Account Balances, and Transaction Amounts are explicitly excluded — these are financial/security-sensitive fields that telemetry never touches. -- **Also Excluded (bottom row, red)**: IP Addresses (configurable per deployment), Personal Data, and Raw TX Payloads are also excluded — these protect operator and user privacy. -- **All-red styling**: Every box is styled in red to visually reinforce that these are hard exclusions, not optional — the telemetry system has no code path to collect any of these fields. -- **Two-row layout**: The split between "NOT Collected" and "Also Excluded" distinguishes between financial data (top) and operational/personal data (bottom), making the privacy boundaries clear to auditors. - -### Privacy Protection Mechanisms - -| Mechanism | Description | -| -------------------------- | --------------------------------------------------------- | -| **Account Hashing** | `tx_account` is hashed at collector level before storage | -| **Configurable Redaction** | Sensitive fields can be excluded via config | -| **Sampling** | Only 10% of traces recorded by default (reduces exposure) | -| **Local Control** | Node operators control what gets exported | -| **No Raw Payloads** | Transaction content is never recorded, only metadata | - -> **Key Principle**: Telemetry collects **operational metadata** (timing, counts, hashes) — never **sensitive content** (keys, balances, amounts). - ---- - -_End of Presentation_