mirror of
https://github.com/XRPLF/rippled.git
synced 2026-07-24 07:30:30 +00:00
Merge branch 'pratik/otel-phase2-rpc-tracing' into pratik/otel-phase3-tx-tracing
Signed-off-by: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com>
This commit is contained in:
Pratik Mankawde
@@ -47,6 +47,7 @@ libxrpl.shamap > xrpl.protocol
|
||||
libxrpl.shamap > xrpl.shamap
|
||||
libxrpl.telemetry > xrpl.basics
|
||||
libxrpl.telemetry > xrpl.config
|
||||
libxrpl.telemetry > xrpl.protocol
|
||||
libxrpl.telemetry > xrpl.telemetry
|
||||
libxrpl.tx > xrpl.basics
|
||||
libxrpl.tx > xrpl.conditions
|
||||
|
||||
@@ -321,7 +321,7 @@ other layer must match them. A CI check enforces this end to end.
|
||||
3. Collision qualifier: `<domain>_<field>` — only when a bare name would collide
|
||||
with a DIFFERENT concept in the shared spanmetrics label space, or with the
|
||||
OTel-reserved `status` key (e.g. `rpc_status`, `grpc_status`,
|
||||
`consensus_state`, `consensus_round`). This disambiguates distinct concepts
|
||||
`consensus_phase`, `consensus_round`). This disambiguates distinct concepts
|
||||
that share a word; it is NOT used to tag the same concept with the workflow
|
||||
that emitted it — that is rule 2 (one shared name).
|
||||
4. Resource attribute: dotted `xrpl.<subsystem>.<field>` — reserved ONLY for
|
||||
|
||||
@@ -185,7 +185,7 @@ agree with the code. A CI check enforces this end to end.
|
||||
3. **Collision qualifier** → `<domain>_<field>`, only when a bare name would
|
||||
collide with a DIFFERENT concept in the shared spanmetrics label space or with
|
||||
the OTel-reserved `status` key (e.g. `rpc_status`, `grpc_status`,
|
||||
`consensus_state`, `consensus_round`, `consensus_mode`). This disambiguates
|
||||
`consensus_phase`, `consensus_round`, `consensus_mode`). This disambiguates
|
||||
distinct concepts that share a word; it is NOT used to tag the same concept
|
||||
with its emitting workflow — that is rule 2 (one shared name).
|
||||
4. **Resource attribute** → dotted `xrpl.<subsystem>.<field>`, reserved ONLY
|
||||
@@ -245,6 +245,8 @@ keys (the dotted form is reserved for resource scope per §2.3.3).
|
||||
| `tx_fee` | int64 | Fee in drops |
|
||||
| `tx_result` | string | `"tesSUCCESS"`, `"tecPATH_DRY"`, etc. |
|
||||
| `ledger_index` | int64 | Ledger containing transaction |
|
||||
| `relay_count` | int64 | Peers the transaction was relayed to |
|
||||
| `suppressed` | bool | `true` when HashRouter dropped a dup |
|
||||
|
||||
#### Consensus Attributes
|
||||
|
||||
@@ -261,12 +263,14 @@ keys (the dotted form is reserved for resource scope per §2.3.3).
|
||||
|
||||
#### RPC Attributes
|
||||
|
||||
| Key | Type | Description |
|
||||
| ---------- | ------ | ----------------------------------------------------- |
|
||||
| `command` | string | Command name (per-span unique on `rpc.command`) |
|
||||
| `version` | int64 | API version |
|
||||
| `rpc_role` | string | `"admin"` or `"user"` (qualified — `role` is generic) |
|
||||
| `params` | string | Sanitized parameters (optional) |
|
||||
| Key | Type | Description |
|
||||
| ------------- | ------- | ----------------------------------------------------------------------------- |
|
||||
| `command` | string | Command name (per-span unique on `rpc.command`) |
|
||||
| `version` | int64 | API version |
|
||||
| `rpc_role` | string | `"admin"` or `"user"` (qualified — `role` is generic) |
|
||||
| `params` | string | Sanitized parameters (optional) |
|
||||
| `rpc_status` | string | Response status: `success` \| `error` (qualified — `status` is OTel-reserved) |
|
||||
| `duration_ms` | float64 | Request duration in milliseconds |
|
||||
|
||||
#### Peer & Message Attributes
|
||||
|
||||
@@ -378,32 +382,48 @@ The following data is explicitly **excluded** from telemetry collection:
|
||||
|
||||
#### Privacy Protection Mechanisms
|
||||
|
||||
| Mechanism | Description |
|
||||
| ----------------------------- | ------------------------------------------------------------------------- |
|
||||
| **Account Hashing** | `tx_account` is hashed at collector level before storage |
|
||||
| **Configurable Redaction** | Sensitive fields can be excluded via `[telemetry]` config section |
|
||||
| **Sampling** | Only 10% of traces recorded by default, reducing data exposure |
|
||||
| **Local Control** | Node operators have full control over what gets exported |
|
||||
| **No Raw Payloads** | Transaction content is never recorded, only metadata (hash, type, result) |
|
||||
| **Collector-Level Filtering** | Additional redaction/hashing can be configured at OTel Collector |
|
||||
| Mechanism | Description |
|
||||
| ----------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
|
||||
| **Account Hashing** | Account addresses are hashed both SDK-side (`pathfind_source_account`, `pathfind_dest_account` — always hashed before emission) and again at the collector level, so raw addresses never reach storage |
|
||||
| **Configurable Redaction** | Sensitive fields can be excluded via `[telemetry]` config section |
|
||||
| **Collector Tail Sampling** | xrpld head sampling is fixed at 1.0 (every span emitted); the collector retains ~10% of non-error traces, reducing stored data exposure |
|
||||
| **Sampling** | Only 10% of traces recorded by default, reducing data exposure |
|
||||
| **Local Control** | Node operators have full control over what gets exported |
|
||||
| **No Raw Payloads** | Transaction content is never recorded, only metadata (hash, type, result) |
|
||||
| **Collector-Level Filtering** | Additional redaction/hashing can be configured at OTel Collector |
|
||||
|
||||
#### Account Address Hashing
|
||||
|
||||
Account addresses are **always** hashed before they reach the telemetry
|
||||
backend — there is no opt-out flag and therefore no insecure-by-default
|
||||
failure mode. Protection is applied in two independent layers:
|
||||
|
||||
1. **SDK-side** (this node): the path-finding RPC handlers call
|
||||
`redactAccount()` (`xrpl::telemetry`, `Redaction.h`) before setting the
|
||||
`pathfind_source_account` / `pathfind_dest_account` span attributes. The
|
||||
helper emits the first 16 characters of `sha512Half(address)` as
|
||||
lowercase hex — deterministic (spans for one account still correlate)
|
||||
but non-reversible.
|
||||
2. **Collector-side** (defense-in-depth): an `attributes/hash` processor in
|
||||
the OpenTelemetry Collector re-hashes those same attributes, so any node
|
||||
that emitted a raw value is still redacted before storage.
|
||||
|
||||
#### Collector-Level Data Protection
|
||||
|
||||
The OpenTelemetry Collector can be configured (via an `attributes` processor)
|
||||
to hash or redact sensitive attributes before export — for example, hashing
|
||||
`tx_account`, deleting `peer_address` to drop IP addresses, and deleting
|
||||
`params` to redact request parameters.
|
||||
`pathfind_source_account` / `pathfind_dest_account`, deleting `peer_address`
|
||||
to drop IP addresses, and deleting `params` to redact request parameters.
|
||||
|
||||
#### Configuration Options for Privacy
|
||||
|
||||
In `xrpld.cfg`, operators control data collection granularity through the
|
||||
`[telemetry]` section. Besides `enabled`, per-component toggles
|
||||
(`trace_transactions`, `trace_consensus`, `trace_rpc`, `trace_peer` — the last
|
||||
often disabled due to high volume) select which spans are emitted, and
|
||||
redaction flags (`redact_account` to hash account addresses, `redact_peer_address`
|
||||
to remove peer IP addresses) control SDK-level redaction before export.
|
||||
|
||||
> **Note**: The `redact_account` configuration in `xrpld.cfg` controls SDK-level redaction before export, while collector-level filtering (see [Collector-Level Data Protection](#collector-level-data-protection) above) provides an additional defense-in-depth layer. Both can operate independently.
|
||||
often disabled due to high volume) select which spans are emitted. Account
|
||||
address hashing is not configurable: addresses are hashed unconditionally by
|
||||
the SDK helper described above, with collector-level hashing as a second
|
||||
layer.
|
||||
|
||||
> **Key Principle**: Telemetry collects **operational metadata** (timing, counts, hashes) — never **sensitive content** (keys, balances, amounts, raw payloads).
|
||||
|
||||
@@ -627,7 +647,7 @@ flowchart TB
|
||||
- **xrpld Process (dark gray)**: The single xrpld node running all three observability frameworks side by side. Each framework operates independently with no interference.
|
||||
- **PerfLog to perf.log**: PerfLog writes JSON-formatted event logs to a local file. Grafana can ingest these via Loki or a file-based datasource.
|
||||
- **Beast Insight to StatsD Server**: Insight sends aggregated metrics (counters, gauges) over UDP to a StatsD server. Grafana reads from StatsD-compatible backends like Graphite or Prometheus (via StatsD exporter).
|
||||
- **OpenTelemetry to OTLP Collector**: OTel exports spans over OTLP/gRPC to a Collector, which then forwards to a trace backend (Tempo).
|
||||
- **OpenTelemetry to OTLP Collector**: OTel exports spans over OTLP/HTTP to a Collector, which then forwards to a trace backend (Tempo). (OTLP/gRPC is future work — §2.2.2.)
|
||||
- **Grafana (red, unified UI)**: All three data streams converge in Grafana, enabling operators to correlate logs, metrics, and traces in a single dashboard.
|
||||
|
||||
### 2.6.5 Correlation with PerfLog
|
||||
|
||||
@@ -180,14 +180,14 @@ pie showData
|
||||
| TracerProvider singleton | ~64 KB | At startup |
|
||||
| BatchSpanProcessor (circular buffer) | ~16 KB | At startup |
|
||||
| BatchSpanProcessor (worker thread) | ~8 MB | At startup |
|
||||
| OTLP exporter (gRPC channel init) | ~256 KB | At startup |
|
||||
| OTLP/HTTP exporter (client init) | ~64 KB | At startup |
|
||||
| Propagator registry | ~8 KB | At startup |
|
||||
| **Total static** | **~8.3 MB** | |
|
||||
| **Total static** | **~8.1 MB** | |
|
||||
|
||||
> **Why higher than earlier estimate**: The BatchSpanProcessor's circular buffer itself is only ~16 KB
|
||||
> (2049 x 8-byte `AtomicUniquePtr` entries), but it spawns a dedicated worker thread whose default
|
||||
> stack size on Linux is ~8 MB. The OTLP gRPC exporter allocates memory for channel stubs and TLS
|
||||
> initialization. The worker thread stack dominates the static footprint.
|
||||
> stack size on Linux is ~8 MB. The OTLP/HTTP exporter allocates a small client and TLS
|
||||
> initialization buffer. The worker thread stack dominates the static footprint.
|
||||
|
||||
### 3.5.2 Dynamic Memory
|
||||
|
||||
|
||||
@@ -166,10 +166,10 @@ flowchart TB
|
||||
|
||||
**Reading the diagram:**
|
||||
|
||||
- **Configuration Sources**: `xrpld.cfg` provides runtime settings (endpoint, sampling) while the CMake flag controls whether telemetry is compiled in at all.
|
||||
- **Configuration Sources**: `xrpld.cfg` provides runtime settings (endpoint, per-component trace toggles) while the CMake flag controls whether telemetry is compiled in at all. Head sampling is fixed at 1.0 and is not a config option; volume reduction happens via tail sampling in the collector.
|
||||
- **Initialization**: `setupTelemetry()` parses config values, then `makeTelemetry()` constructs the provider, processor, and exporter objects.
|
||||
- **Runtime Components**: The `TracerProvider` creates spans, the `BatchProcessor` buffers them, and the `OTLP Exporter` serializes and sends them over the wire.
|
||||
- **OTLP arrow to Collector**: Trace data leaves the xrpld process via OTLP (gRPC or HTTP) and enters the external Collector pipeline.
|
||||
- **OTLP arrow to Collector**: Trace data leaves the xrpld process via OTLP/HTTP and enters the external Collector pipeline. (OTLP/gRPC is future work — see design decisions §2.2.2.)
|
||||
- **Collector Pipeline**: `Receivers` ingest OTLP data, `Processors` apply sampling/filtering/enrichment, and `Exporters` forward traces to storage backends (Tempo, etc.).
|
||||
|
||||
---
|
||||
|
||||
@@ -237,7 +237,7 @@ A Tempo-backed dashboard (uid `xrpld-consensus-health`) with four panels, all dr
|
||||
- **Proposers per Round** (stat): average of the `span.proposers` attribute on `consensus.round` spans.
|
||||
- **Recent Slow Rounds (>5s)** (table): `consensus.round` spans filtered to `duration > 5s`.
|
||||
|
||||
The underlying TraceQL queries are listed in section 7.7.3 and used throughout this doc.
|
||||
Each panel's TraceQL query is described inline in its bullet above.
|
||||
|
||||
### 7.6.2 Node Overview Dashboard
|
||||
|
||||
@@ -261,9 +261,9 @@ Grafana provisions three TraceQL-based alert rules (group `xrpld-tracing-alerts`
|
||||
- **RPC Error Rate Spike** (critical, `for: 2m`): fires when the error rate across `rpc.command.*` spans exceeds 5%. Error _rate_ is a ratio, so it must divide the error-span rate by the total-span rate — a single TraceQL `rate()` returns spans/second, not a percentage, and would fire on traffic volume alone. This uses span metrics emitted by the collector's `spanmetrics` connector (Prometheus datasource), not a TraceQL query:
|
||||
|
||||
```
|
||||
sum(rate(calls_total{service_name="xrpld", span_name=~"rpc.command.*", status_code="STATUS_CODE_ERROR"}[5m]))
|
||||
sum(rate(traces_spanmetrics_calls_total{service_name="xrpld", span_name=~"rpc.command.*", status_code="STATUS_CODE_ERROR"}[5m]))
|
||||
/
|
||||
sum(rate(calls_total{service_name="xrpld", span_name=~"rpc.command.*"}[5m]))
|
||||
sum(rate(traces_spanmetrics_calls_total{service_name="xrpld", span_name=~"rpc.command.*"}[5m]))
|
||||
> 0.05
|
||||
```
|
||||
|
||||
|
||||
@@ -182,17 +182,15 @@ flowchart TB
|
||||
| [07-observability-backends.md](./07-observability-backends.md) | Backend selection and architecture |
|
||||
| [08-appendix.md](./08-appendix.md) | Glossary, references, version history |
|
||||
| [secure-OTel.md](./secure-OTel.md) | Threat model and hardening (mTLS, peer validation) |
|
||||
| [presentation.md](./presentation.md) | Slide deck for OTel plan overview |
|
||||
|
||||
### Task Lists
|
||||
|
||||
| Document | Description |
|
||||
| ------------------------------------------ | --------------------------------------------------- |
|
||||
| [Phase2_taskList.md](./Phase2_taskList.md) | RPC layer trace instrumentation |
|
||||
| [Phase3_taskList.md](./Phase3_taskList.md) | Peer overlay & consensus tracing |
|
||||
| [Phase4_taskList.md](./Phase4_taskList.md) | Transaction lifecycle tracing |
|
||||
| [Phase5_taskList.md](./Phase5_taskList.md) | Ledger processing & advanced tracing |
|
||||
| [presentation.md](./presentation.md) | Presentation slides for OpenTelemetry plan overview |
|
||||
| Document | Description |
|
||||
| ------------------------------------------ | ------------------------------------ |
|
||||
| [Phase2_taskList.md](./Phase2_taskList.md) | RPC layer trace instrumentation |
|
||||
| [Phase3_taskList.md](./Phase3_taskList.md) | Peer overlay & consensus tracing |
|
||||
| [Phase4_taskList.md](./Phase4_taskList.md) | Transaction lifecycle tracing |
|
||||
| [Phase5_taskList.md](./Phase5_taskList.md) | Ledger processing & advanced tracing |
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# [OpenTelemetry](00-tracing-fundamentals.md) Distributed Tracing Implementation Plan for xrpld (xrpld)
|
||||
# [OpenTelemetry](00-tracing-fundamentals.md) Distributed Tracing Implementation Plan for xrpld
|
||||
|
||||
## Executive Summary
|
||||
|
||||
@@ -130,7 +130,7 @@ Key trace points span across transaction submission via RPC, peer-to-peer messag
|
||||
|
||||
> **OTLP** = OpenTelemetry Protocol | **CNCF** = Cloud Native Computing Foundation
|
||||
|
||||
The OpenTelemetry C++ SDK is selected for its CNCF backing, active development, and native performance characteristics. Traces are exported via OTLP/gRPC (primary) or OTLP/HTTP (fallback) to an OpenTelemetry Collector, which provides flexible routing and sampling.
|
||||
The OpenTelemetry C++ SDK is selected for its CNCF backing, active development, and native performance characteristics. Traces are exported via OTLP/HTTP to an OpenTelemetry Collector, which provides flexible routing and sampling. OTLP/gRPC is planned future work (see design decisions §2.2.2).
|
||||
|
||||
Span naming follows a hierarchical `<component>.<operation>` convention (e.g., `rpc.submit`, `tx.relay`, `consensus.round`). Context propagation uses W3C Trace Context headers for HTTP and embedded Protocol Buffer fields for P2P messages. The implementation coexists with existing PerfLog and Insight observability systems through correlation IDs.
|
||||
|
||||
@@ -154,7 +154,7 @@ Performance optimization strategies include head sampling fixed at 100% (intenti
|
||||
|
||||
> **OTLP** = OpenTelemetry Protocol | **APM** = Application Performance Monitoring
|
||||
|
||||
Configuration is handled through the `[telemetry]` section in `xrpld.cfg` with options for enabling/disabling, exporter selection, endpoint configuration, sampling ratios, and component-level filtering. CMake integration includes a `XRPL_ENABLE_TELEMETRY` option for compile-time control.
|
||||
Configuration is handled through the `[telemetry]` section in `xrpld.cfg` with options for enabling/disabling, exporter selection, endpoint configuration, and component-level filtering. Head sampling is fixed at 1.0 (not operator-configurable); volume reduction is done by tail sampling in the collector. CMake integration includes a `XRPL_ENABLE_TELEMETRY` option for compile-time control.
|
||||
|
||||
OpenTelemetry Collector configurations are provided for development and production (with tail-based sampling, Tempo, and Elastic APM). Docker Compose examples enable quick local development environment setup.
|
||||
|
||||
|
||||
@@ -169,11 +169,12 @@ Node health (`amendment_blocked`, `server_state`) is not part of the telemetry s
|
||||
| `rpc.process` | `batch_size` | int64 | `params.size()` (only when batch) |
|
||||
| `rpc.ws_message` | `command` | string | `jv[command]` or `jv[method]` |
|
||||
| `rpc.command.*` | `load_type` | string | `context.loadType.label()` |
|
||||
| `pathfind.compute` | `pathfind_dest_amount` | string | `saDstAmount_.getFullText()` |
|
||||
| `pathfind.compute` | `pathfind_dest_currency` | string | `to_string(saDstAmount_.asset())` |
|
||||
| `pathfind.discover` | `pathfind_num_source_assets` | int64 | `sourceAssets.size()` |
|
||||
|
||||
**New attr keys**: `RpcSpanNames.h` (`isBatch`, `batchSize`, `loadType`), `PathFindSpanNames.h` (`destAmount`, `destCurrency`, `numSourceAssets`).
|
||||
_Note: `pathfind_dest_amount` was removed — the destination amount is a financial value excluded by the privacy policy (design §2.4.4)._
|
||||
|
||||
**New attr keys**: `RpcSpanNames.h` (`isBatch`, `batchSize`, `loadType`), `PathFindSpanNames.h` (`destCurrency`, `numSourceAssets`).
|
||||
|
||||
**Modified files**:
|
||||
|
||||
|
||||
@@ -1,682 +0,0 @@
|
||||
# OpenTelemetry Distributed Tracing for xrpld
|
||||
|
||||
---
|
||||
|
||||
## Slide 1: Introduction
|
||||
|
||||
> **CNCF** = Cloud Native Computing Foundation
|
||||
|
||||
### What is OpenTelemetry?
|
||||
|
||||
OpenTelemetry is an open-source, CNCF-backed observability framework for distributed tracing, metrics, and logs.
|
||||
|
||||
### Why OpenTelemetry for xrpld?
|
||||
|
||||
- **End-to-End Transaction Visibility**: Track transactions from submission → consensus → ledger inclusion
|
||||
- **Cross-Node Correlation**: Follow requests across multiple independent nodes using a unique `trace_id`
|
||||
- **Consensus Round Analysis**: Understand timing and behavior across validators
|
||||
- **Incident Debugging**: Correlate events across distributed nodes during issues
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
A["Node A<br/>tx.receive<br/>trace_id: abc123"] --> B["Node B<br/>tx.relay<br/>trace_id: abc123"] --> C["Node C<br/>tx.validate<br/>trace_id: abc123"] --> D["Node D<br/>ledger.apply<br/>trace_id: abc123"]
|
||||
|
||||
style A fill:#1565c0,stroke:#0d47a1,color:#fff
|
||||
style B fill:#2e7d32,stroke:#1b5e20,color:#fff
|
||||
style C fill:#2e7d32,stroke:#1b5e20,color:#fff
|
||||
style D fill:#e65100,stroke:#bf360c,color:#fff
|
||||
```
|
||||
|
||||
**Reading the diagram:**
|
||||
|
||||
- **Node A (blue, leftmost)**: The originating node that first receives the transaction and assigns a new `trace_id: abc123`; this ID becomes the correlation key for the entire distributed trace.
|
||||
- **Node B and Node C (green, middle)**: Relay and validation nodes — each creates its own span but carries the same `trace_id`, so their work is linked to the original submission without any central coordinator.
|
||||
- **Node D (orange, rightmost)**: The final node that applies the transaction to the ledger; the trace now spans the full lifecycle from submission to ledger inclusion.
|
||||
- **Left-to-right flow**: The horizontal progression shows the real-world message path — a transaction hops from node to node, and the shared `trace_id` stitches all hops into a single queryable trace.
|
||||
|
||||
> **Trace ID: abc123** — All nodes share the same trace, enabling cross-node correlation.
|
||||
|
||||
---
|
||||
|
||||
## Slide 2: OpenTelemetry vs Open Source Alternatives
|
||||
|
||||
> **CNCF** = Cloud Native Computing Foundation
|
||||
|
||||
| Feature | OpenTelemetry | Jaeger | Zipkin | SkyWalking | Pinpoint | Prometheus |
|
||||
| ------------------- | ---------------- | ---------------- | ------------------ | ---------- | ---------- | ---------- |
|
||||
| **Tracing** | YES | YES | YES | YES | YES | NO |
|
||||
| **Metrics** | YES | NO | NO | YES | YES | YES |
|
||||
| **Logs** | YES | NO | NO | YES | NO | NO |
|
||||
| **C++ SDK** | YES Official | YES (Deprecated) | YES (Unmaintained) | NO | NO | YES |
|
||||
| **Vendor Neutral** | YES Primary goal | NO | NO | NO | NO | NO |
|
||||
| **Instrumentation** | Manual + Auto | Manual | Manual | Auto-first | Auto-first | Manual |
|
||||
| **Backend** | Any (exporters) | Self | Self | Self | Self | Self |
|
||||
| **CNCF Status** | Incubating | Graduated | NO | Incubating | NO | Graduated |
|
||||
|
||||
> **Why OpenTelemetry?** It's the only actively maintained, full-featured C++ option with vendor neutrality — allowing export to Tempo, Prometheus, Grafana, or any commercial backend without changing instrumentation.
|
||||
|
||||
---
|
||||
|
||||
## Slide 3: Adoption Scope — Traces Only (Current Plan)
|
||||
|
||||
OpenTelemetry supports three signal types: **Traces**, **Metrics**, and **Logs**. xrpld already captures metrics (StatsD via Beast Insight) and logs (Journal/PerfLog). The question is: how much of OTel do we adopt?
|
||||
|
||||
> **Scenario A**: Add distributed tracing. Keep StatsD for metrics and Journal for logs.
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
subgraph xrpld["xrpld Process"]
|
||||
direction TB
|
||||
OTel["OTel SDK<br/>(Traces)"]
|
||||
Insight["Beast Insight<br/>(StatsD Metrics)"]
|
||||
Journal["Journal + PerfLog<br/>(Logging)"]
|
||||
end
|
||||
|
||||
OTel -->|"OTLP"| Collector["OTel Collector"]
|
||||
Insight -->|"UDP"| StatsD["StatsD Server"]
|
||||
Journal -->|"File I/O"| LogFile["perf.log / debug.log"]
|
||||
|
||||
Collector --> Tempo["Tempo"]
|
||||
StatsD --> Graphite["Graphite / Grafana"]
|
||||
LogFile --> Loki["Loki (optional)"]
|
||||
|
||||
style xrpld fill:#424242,stroke:#212121,color:#fff
|
||||
style OTel fill:#2e7d32,stroke:#1b5e20,color:#fff
|
||||
style Insight fill:#1565c0,stroke:#0d47a1,color:#fff
|
||||
style Journal fill:#e65100,stroke:#bf360c,color:#fff
|
||||
style Collector fill:#2e7d32,stroke:#1b5e20,color:#fff
|
||||
```
|
||||
|
||||
| Aspect | Details |
|
||||
| ------------------------------ | --------------------------------------------------------------------------------------------------------------- |
|
||||
| **What changes for operators** | Deploy OTel Collector + trace backend. Existing StatsD and log pipelines stay as-is. |
|
||||
| **Codebase impact** | New `Telemetry` module (~1500 LOC). Beast Insight and Journal untouched. |
|
||||
| **New capabilities** | Cross-node trace correlation, span-based debugging, request lifecycle visibility. |
|
||||
| **What we still can't do** | Correlate metrics with specific traces natively. StatsD metrics remain fire-and-forget with no trace exemplars. |
|
||||
| **Maintenance burden** | Three separate observability systems to maintain (OTel + StatsD + Journal). |
|
||||
| **Risk** | Lowest — additive change, no existing systems disturbed. |
|
||||
|
||||
---
|
||||
|
||||
## Slide 4: Future Adoption — Metrics & Logs via OTel
|
||||
|
||||
### Scenario B: + OTel Metrics (Replace StatsD)
|
||||
|
||||
> Migrate StatsD to OTel Metrics API, exposing Prometheus-compatible metrics. Remove Beast Insight.
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
subgraph xrpld["xrpld Process"]
|
||||
direction TB
|
||||
OTel["OTel SDK<br/>(Traces + Metrics)"]
|
||||
Journal["Journal + PerfLog<br/>(Logging)"]
|
||||
end
|
||||
|
||||
OTel -->|"OTLP"| Collector["OTel Collector"]
|
||||
Journal -->|"File I/O"| LogFile["perf.log / debug.log"]
|
||||
|
||||
Collector --> Tempo["Tempo<br/>(Traces)"]
|
||||
Collector --> Prom["Prometheus<br/>(Metrics)"]
|
||||
LogFile --> Loki["Loki (optional)"]
|
||||
|
||||
style xrpld fill:#424242,stroke:#212121,color:#fff
|
||||
style OTel fill:#2e7d32,stroke:#1b5e20,color:#fff
|
||||
style Journal fill:#e65100,stroke:#bf360c,color:#fff
|
||||
style Collector fill:#2e7d32,stroke:#1b5e20,color:#fff
|
||||
```
|
||||
|
||||
- **Better metrics?** Yes — Prometheus gives native histograms (p50/p95/p99), multi-dimensional labels, and exemplars linking metric spikes to traces.
|
||||
- **Codebase**: Remove `Beast::Insight` + `StatsDCollector` (~2000 LOC). Single SDK for traces and metrics.
|
||||
- **Operator effort**: Rewrite dashboards from StatsD/Graphite queries to PromQL. Run both in parallel during transition.
|
||||
- **Risk**: Medium — operators must migrate monitoring infrastructure.
|
||||
|
||||
### Scenario C: + OTel Logs (Full Stack)
|
||||
|
||||
> Also replace Journal logging with OTel Logs API. Single SDK for everything.
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
subgraph xrpld["xrpld Process"]
|
||||
OTel["OTel SDK<br/>(Traces + Metrics + Logs)"]
|
||||
end
|
||||
|
||||
OTel -->|"OTLP"| Collector["OTel Collector"]
|
||||
|
||||
Collector --> Tempo["Tempo<br/>(Traces)"]
|
||||
Collector --> Prom["Prometheus<br/>(Metrics)"]
|
||||
Collector --> Loki["Loki / Elastic<br/>(Logs)"]
|
||||
|
||||
style xrpld fill:#424242,stroke:#212121,color:#fff
|
||||
style OTel fill:#2e7d32,stroke:#1b5e20,color:#fff
|
||||
style Collector fill:#2e7d32,stroke:#1b5e20,color:#fff
|
||||
```
|
||||
|
||||
- **Structured logging**: OTel Logs API outputs structured records with `trace_id`, `span_id`, severity, and attributes by design.
|
||||
- **Full correlation**: Every log line carries `trace_id`. Click trace → see logs. Click metric spike → see trace → see logs.
|
||||
- **Codebase**: Remove Beast Insight (~2000 LOC) + simplify Journal/PerfLog (~3000 LOC). One dependency instead of three.
|
||||
- **Risk**: Highest — `beast::Journal` is deeply embedded in every component. Large refactor. OTel C++ Logs API is newer (stable since v1.11, less battle-tested).
|
||||
|
||||
### Recommendation
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
A["Phase 1<br/><b>Traces Only</b><br/>(Current Plan)"] --> B["Phase 2<br/><b>+ Metrics</b><br/>(Replace StatsD)"] --> C["Phase 3<br/><b>+ Logs</b><br/>(Full OTel)"]
|
||||
|
||||
style A fill:#2e7d32,stroke:#1b5e20,color:#fff
|
||||
style B fill:#1565c0,stroke:#0d47a1,color:#fff
|
||||
style C fill:#e65100,stroke:#bf360c,color:#fff
|
||||
```
|
||||
|
||||
| Phase | Signal | Strategy | Risk |
|
||||
| -------------------- | --------- | -------------------------------------------------------------- | ------ |
|
||||
| **Phase 1** (now) | Traces | Add OTel traces. Keep StatsD and Journal. Prove value. | Low |
|
||||
| **Phase 2** (future) | + Metrics | Migrate StatsD → Prometheus via OTel. Remove Beast Insight. | Medium |
|
||||
| **Phase 3** (future) | + Logs | Adopt OTel Logs API. Align with structured logging initiative. | High |
|
||||
|
||||
> **Key Takeaway**: Start with traces (unique value, lowest risk), then incrementally adopt metrics and logs as the OTel infrastructure proves itself.
|
||||
|
||||
---
|
||||
|
||||
## Slide 5: Comparison with xrpld's Existing Solutions
|
||||
|
||||
### Current Observability Stack
|
||||
|
||||
| Aspect | PerfLog (JSON) | StatsD (Metrics) | OpenTelemetry (NEW) |
|
||||
| --------------------- | --------------------- | --------------------- | --------------------------- |
|
||||
| **Type** | Logging | Metrics | Distributed Tracing |
|
||||
| **Scope** | Single node | Single node | **Cross-node** |
|
||||
| **Data** | JSON log entries | Counters, gauges | Spans with context |
|
||||
| **Correlation** | By timestamp | By metric name | By `trace_id` |
|
||||
| **Overhead** | Low (file I/O) | Low (UDP) | Low-Medium (configurable) |
|
||||
| **Question Answered** | "What happened here?" | "How many? How fast?" | **"What was the journey?"** |
|
||||
|
||||
### Use Case Matrix
|
||||
|
||||
| Scenario | PerfLog | StatsD | OpenTelemetry |
|
||||
| -------------------------------- | ------- | ------ | ------------- |
|
||||
| "How many TXs per second?" | ❌ | ✅ | ❌ |
|
||||
| "Why was this specific TX slow?" | ⚠️ | ❌ | ✅ |
|
||||
| "Which node delayed consensus?" | ❌ | ❌ | ✅ |
|
||||
| "Show TX journey across 5 nodes" | ❌ | ❌ | ✅ |
|
||||
|
||||
> **Key Insight**: In the **traces-only** approach (Phase 1), OpenTelemetry **complements** existing systems. In future phases, OTel metrics and logs could **replace** StatsD and Journal respectively — see Slides 3-4 for the full adoption roadmap.
|
||||
|
||||
---
|
||||
|
||||
## Slide 6: Architecture
|
||||
|
||||
> **OTLP** = OpenTelemetry Protocol | **WS** = WebSocket
|
||||
|
||||
### High-Level Integration Architecture
|
||||
|
||||
```mermaid
|
||||
flowchart TB
|
||||
subgraph xrpld["xrpld Node"]
|
||||
subgraph services["Core Services"]
|
||||
direction LR
|
||||
RPC["RPC Server<br/>(HTTP/WS)"] ~~~ Overlay["Overlay<br/>(P2P Network)"] ~~~ Consensus["Consensus<br/>(RCLConsensus)"]
|
||||
end
|
||||
|
||||
Telemetry["Telemetry Module<br/>(OpenTelemetry SDK)"]
|
||||
|
||||
services --> Telemetry
|
||||
end
|
||||
|
||||
Telemetry -->|OTLP/HTTP| Collector["OTel Collector"]
|
||||
|
||||
Collector --> Tempo["Grafana Tempo"]
|
||||
Collector --> Elastic["Elastic APM"]
|
||||
|
||||
style xrpld fill:#424242,stroke:#212121,color:#fff
|
||||
style services fill:#1565c0,stroke:#0d47a1,color:#fff
|
||||
style Telemetry fill:#2e7d32,stroke:#1b5e20,color:#fff
|
||||
style Collector fill:#e65100,stroke:#bf360c,color:#fff
|
||||
```
|
||||
|
||||
**Reading the diagram:**
|
||||
|
||||
- **Core Services (blue, top)**: RPC Server, Overlay, and Consensus are the three primary components that generate trace data — they represent the entry points for client requests, peer messages, and consensus rounds respectively.
|
||||
- **Telemetry Module (green, middle)**: The OpenTelemetry SDK sits below the core services and receives span data from all three; it acts as a single collection point within the xrpld process.
|
||||
- **OTel Collector (orange, center)**: An external process that receives spans over OTLP/HTTP from the Telemetry Module; it decouples xrpld from backend choices and handles batching, sampling, and routing.
|
||||
- **Backends (bottom row)**: Tempo and Elastic APM are interchangeable — the Collector fans out to any combination, so operators can switch backends without modifying xrpld code.
|
||||
- **Top-to-bottom flow**: Data flows from instrumented code down through the SDK, out over the network to the Collector, and finally into storage/visualization backends.
|
||||
|
||||
### Context Propagation
|
||||
|
||||
```mermaid
|
||||
sequenceDiagram
|
||||
participant Client
|
||||
participant NodeA as Node A
|
||||
participant NodeB as Node B
|
||||
|
||||
Client->>NodeA: Submit TX (no context)
|
||||
Note over NodeA: Creates trace_id: abc123<br/>span: tx.receive
|
||||
NodeA->>NodeB: Relay TX<br/>(traceparent: abc123)
|
||||
Note over NodeB: Links to trace_id: abc123<br/>span: tx.relay
|
||||
```
|
||||
|
||||
- **HTTP/RPC**: W3C Trace Context headers (`traceparent`)
|
||||
- **P2P Messages**: Protocol Buffer extension fields
|
||||
|
||||
---
|
||||
|
||||
## Slide 7: Implementation Plan
|
||||
|
||||
### 5-Phase Rollout (9 Weeks)
|
||||
|
||||
> **Note**: Dates shown are relative to project start, not calendar dates.
|
||||
|
||||
```mermaid
|
||||
gantt
|
||||
title Implementation Timeline
|
||||
dateFormat YYYY-MM-DD
|
||||
axisFormat Week %W
|
||||
|
||||
section Phase 1
|
||||
Core Infrastructure :p1, 2024-01-01, 2w
|
||||
|
||||
section Phase 2
|
||||
RPC Tracing :p2, after p1, 2w
|
||||
|
||||
section Phase 3
|
||||
Transaction Tracing :p3, after p2, 2w
|
||||
|
||||
section Phase 4
|
||||
Consensus Tracing :p4, after p3, 2w
|
||||
|
||||
section Phase 5
|
||||
Documentation :p5, after p4, 1w
|
||||
```
|
||||
|
||||
### Phase Details
|
||||
|
||||
| Phase | Focus | Key Deliverables | Effort |
|
||||
| ----- | ------------------- | -------------------------------------------- | ------- |
|
||||
| 1 | Core Infrastructure | SDK integration, Telemetry interface, Config | 10 days |
|
||||
| 2 | RPC Tracing | HTTP context extraction, Handler spans | 10 days |
|
||||
| 3 | Transaction Tracing | Protobuf context, P2P relay propagation | 10 days |
|
||||
| 4 | Consensus Tracing | Round spans, Proposal/validation tracing | 10 days |
|
||||
| 5 | Documentation | Runbook, Dashboards, Training | 7 days |
|
||||
|
||||
**Total Effort**: ~47 developer-days (2 developers)
|
||||
|
||||
> **Future Phases** (not in current scope): After traces are stable, OTel metrics can replace StatsD (~3 weeks), and OTel logs can replace Journal (~4 weeks, aligned with structured logging initiative). See Slides 3-4 for the full adoption roadmap.
|
||||
|
||||
---
|
||||
|
||||
## Slide 8: Performance Overhead
|
||||
|
||||
> **OTLP** = OpenTelemetry Protocol
|
||||
|
||||
### Estimated System Impact
|
||||
|
||||
| Metric | Overhead | Notes |
|
||||
| ----------------- | ---------- | ------------------------------------------------ |
|
||||
| **CPU** | 1-3% | Span creation and attribute setting |
|
||||
| **Memory** | ~10 MB | SDK statics + batch buffer + worker thread stack |
|
||||
| **Network** | 10-50 KB/s | Compressed OTLP export to collector |
|
||||
| **Latency (p99)** | <2% | With proper sampling configuration |
|
||||
|
||||
#### How We Arrived at These Numbers
|
||||
|
||||
**Assumptions (XRPL mainnet baseline)**:
|
||||
|
||||
| Parameter | Value | Source |
|
||||
| ------------------------- | ---------------------- | --------------------------------------------------------------------------------------------------- |
|
||||
| Transaction throughput | ~25 TPS (peaks to ~50) | Mainnet average |
|
||||
| Default peers per node | 21 | `peerfinder/detail/Tuning.h` (`defaultMaxPeers`) |
|
||||
| Consensus round frequency | ~1 round / 3-4 seconds | `ConsensusParms.h` (`ledgerMIN_CONSENSUS=1950ms`) |
|
||||
| Proposers per round | ~20-35 | Mainnet UNL size |
|
||||
| P2P message rate | ~160 msgs/sec | See message breakdown below |
|
||||
| Avg TX processing time | ~200 μs | Profiled baseline |
|
||||
| Single span creation cost | 500-1000 ns | OTel C++ SDK benchmarks (see [3.5.4](./03-implementation-strategy.md#354-performance-data-sources)) |
|
||||
|
||||
**P2P message breakdown** (per node, mainnet):
|
||||
|
||||
| Message Type | Rate | Derivation |
|
||||
| ------------- | ------------ | --------------------------------------------------------------------- |
|
||||
| TMTransaction | ~100/sec | ~25 TPS × ~4 relay hops per TX, deduplicated by HashRouter |
|
||||
| TMValidation | ~50/sec | ~35 validators × ~1 validation/3s round ≈ ~12/sec, plus relay fan-out |
|
||||
| TMProposeSet | ~10/sec | ~35 proposers / 3s round ≈ ~12/round, clustered in establish phase |
|
||||
| **Total** | **~160/sec** | **Only traced message types counted** |
|
||||
|
||||
**CPU (1-3%) — Calculation**:
|
||||
|
||||
Per-transaction tracing cost breakdown:
|
||||
|
||||
| Operation | Cost | Notes |
|
||||
| ----------------------------------------------- | ----------- | ------------------------------------------ |
|
||||
| `tx.receive` span (create + end + 4 attributes) | ~1400 ns | ~1000ns create + ~200ns end + 4×50ns attrs |
|
||||
| `tx.validate` span | ~1200 ns | ~1000ns create + ~200ns for 2 attributes |
|
||||
| `tx.relay` span | ~1200 ns | ~1000ns create + ~200ns for 2 attributes |
|
||||
| Context injection into P2P message | ~200 ns | Serialize trace_id + span_id into protobuf |
|
||||
| **Total per TX** | **~4.0 μs** | |
|
||||
|
||||
> **CPU overhead**: 4.0 μs / 200 μs baseline = **~2.0% per transaction**. Under high load with consensus + RPC spans overlapping, reaches ~3%. Consensus itself adds only ~36 μs per 3-second round (~0.001%), so the TX path dominates. On production server hardware (3+ GHz Xeon), span creation drops to ~500-600 ns, bringing per-TX cost to ~2.6 μs (~1.3%). See [Section 3.5.4](./03-implementation-strategy.md#354-performance-data-sources) for benchmark sources.
|
||||
|
||||
**Memory (~10 MB) — Calculation**:
|
||||
|
||||
| Component | Size | Notes |
|
||||
| --------------------------------------------- | ------------------ | ------------------------------------- |
|
||||
| TracerProvider + Exporter (gRPC channel init) | ~320 KB | Allocated once at startup |
|
||||
| BatchSpanProcessor (circular buffer) | ~16 KB | 2049 × 8-byte AtomicUniquePtr entries |
|
||||
| BatchSpanProcessor (worker thread stack) | ~8 MB | Default Linux thread stack size |
|
||||
| Active spans (in-flight, max ~1000) | ~500-800 KB | ~500-800 bytes/span × 1000 concurrent |
|
||||
| Export queue (batch buffer, max 2048 spans) | ~1 MB | ~500 bytes/span × 2048 queue depth |
|
||||
| Thread-local context storage (~100 threads) | ~6.4 KB | ~64 bytes/thread |
|
||||
| **Total** | **~10 MB ceiling** | |
|
||||
|
||||
> Memory plateaus once the export queue fills — the `max_queue_size=2048` config bounds growth.
|
||||
> The worker thread stack (~8 MB) dominates the static footprint but is virtual memory; actual RSS
|
||||
> depends on stack usage (typically much less). Active spans are larger than originally estimated
|
||||
> (~500-800 bytes) because the OTel SDK `Span` object includes a mutex (~40 bytes), `SpanData`
|
||||
> recordable (~250 bytes base), and `std::map`-based attribute storage (~200-500 bytes for 3-5
|
||||
> string attributes). See [Section 3.5.4](./03-implementation-strategy.md#354-performance-data-sources) for source references.
|
||||
|
||||
> **Measured (perf-iac, telemetry on vs off, 9 nodes under payment load)**: the ~10 MB
|
||||
> above is a theoretical SDK-footprint ceiling, dominated by virtual (not resident) thread-stack
|
||||
> memory. In practice, per-node RSS showed **no measurable increase over the telemetry-off
|
||||
> baseline** (~15 GiB mean / ~18–19 GiB peak on both sides), with no OOM, swap, or leak over the
|
||||
> run. Treat memory overhead as negligible; the ceiling is a provisioning safety margin, not an
|
||||
> expected increase.
|
||||
|
||||
**Network (10-50 KB/s) — Calculation**:
|
||||
|
||||
Two sources of network overhead:
|
||||
|
||||
**(A) OTLP span export to Collector:**
|
||||
|
||||
| Sampling Rate | Effective Spans/sec | Avg Span Size (compressed) | Bandwidth |
|
||||
| -------------------------- | ------------------- | -------------------------- | ------------ |
|
||||
| 100% (dev only) | ~500 | ~500 bytes | ~250 KB/s |
|
||||
| **10% (recommended prod)** | **~50** | **~500 bytes** | **~25 KB/s** |
|
||||
| 1% (minimal) | ~5 | ~500 bytes | ~2.5 KB/s |
|
||||
|
||||
> The ~500 spans/sec at 100% comes from: ~100 TX spans + ~160 P2P context spans + ~23 consensus spans/round + ~50 RPC spans = ~500/sec. OTLP protobuf with gzip compression yields ~500 bytes/span average.
|
||||
|
||||
**(B) P2P trace context overhead** (added to existing messages, always-on regardless of sampling):
|
||||
|
||||
| Message Type | Rate | Context Size | Bandwidth |
|
||||
| ------------- | -------- | ------------ | ------------- |
|
||||
| TMTransaction | ~100/sec | 29 bytes | ~2.9 KB/s |
|
||||
| TMValidation | ~50/sec | 29 bytes | ~1.5 KB/s |
|
||||
| TMProposeSet | ~10/sec | 29 bytes | ~0.3 KB/s |
|
||||
| **Total P2P** | | | **~4.7 KB/s** |
|
||||
|
||||
> **Combined**: 25 KB/s (OTLP export at 10%) + 5 KB/s (P2P context) ≈ **~30 KB/s typical**. The 10-50 KB/s range covers 10-20% sampling under normal to peak mainnet load.
|
||||
|
||||
**Latency (<2%) — Calculation**:
|
||||
|
||||
| Path | Tracing Cost | Baseline | Overhead |
|
||||
| ------------------------------ | ------------ | -------- | -------- |
|
||||
| Fast RPC (e.g., `server_info`) | 2.75 μs | ~1 ms | 0.275% |
|
||||
| Slow RPC (e.g., `path_find`) | 2.75 μs | ~100 ms | 0.003% |
|
||||
| Transaction processing | 4.0 μs | ~200 μs | 2.0% |
|
||||
| Consensus round | 36 μs | ~3 sec | 0.001% |
|
||||
|
||||
> At p99, even the worst case (TX processing at 2.0%) is within the 1-3% range. RPC and consensus overhead are negligible. On production hardware, TX overhead drops to ~1.3%.
|
||||
|
||||
### Per-Message Overhead (Context Propagation)
|
||||
|
||||
Each P2P message carries trace context with the following overhead:
|
||||
|
||||
| Field | Size | Description |
|
||||
| ------------- | ------------- | ----------------------------------------- |
|
||||
| `trace_id` | 16 bytes | Unique identifier for the entire trace |
|
||||
| `span_id` | 8 bytes | Current span (becomes parent on receiver) |
|
||||
| `trace_flags` | 1 byte | Sampling decision flags |
|
||||
| `trace_state` | 0-4 bytes | Optional vendor-specific data |
|
||||
| **Total** | **~29 bytes** | **Added per traced P2P message** |
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
subgraph msg["P2P Message with Trace Context"]
|
||||
A["Original Message<br/>(variable size)"] --> B["+ TraceContext<br/>(~29 bytes)"]
|
||||
end
|
||||
|
||||
subgraph breakdown["Context Breakdown"]
|
||||
C["trace_id<br/>16 bytes"]
|
||||
D["span_id<br/>8 bytes"]
|
||||
E["flags<br/>1 byte"]
|
||||
F["state<br/>0-4 bytes"]
|
||||
end
|
||||
|
||||
B --> breakdown
|
||||
|
||||
style A fill:#424242,stroke:#212121,color:#fff
|
||||
style B fill:#2e7d32,stroke:#1b5e20,color:#fff
|
||||
style C fill:#1565c0,stroke:#0d47a1,color:#fff
|
||||
style D fill:#1565c0,stroke:#0d47a1,color:#fff
|
||||
style E fill:#e65100,stroke:#bf360c,color:#fff
|
||||
style F fill:#4a148c,stroke:#2e0d57,color:#fff
|
||||
```
|
||||
|
||||
**Reading the diagram:**
|
||||
|
||||
- **Original Message (gray, left)**: The existing P2P message payload of variable size — this is unchanged; trace context is appended, never modifying the original data.
|
||||
- **+ TraceContext (green, right of message)**: The additional 29-byte context block attached to each traced message; the arrow from the original message shows it is a pure addition.
|
||||
- **Context Breakdown (right subgraph)**: The four fields — `trace_id` (16 bytes), `span_id` (8 bytes), `flags` (1 byte), and `state` (0-4 bytes) — show exactly what is added and their individual sizes.
|
||||
- **Color coding**: Blue fields (`trace_id`, `span_id`) are the core identifiers required for trace correlation; orange (`flags`) controls sampling decisions; purple (`state`) is optional vendor data typically omitted.
|
||||
|
||||
> **Note**: 29 bytes represents ~1-6% overhead depending on message size (500B simple TX to 5KB proposal), which is acceptable for the observability benefits provided.
|
||||
|
||||
### Mitigation Strategies
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
A["Head Sampling<br/>fixed 1.0 (record all)"] --> B["Tail Sampling<br/>Keep errors/slow"] --> C["Batch Export<br/>Reduce I/O"] --> D["Conditional Compile<br/>XRPL_ENABLE_TELEMETRY"]
|
||||
|
||||
style A fill:#1565c0,stroke:#0d47a1,color:#fff
|
||||
style B fill:#2e7d32,stroke:#1b5e20,color:#fff
|
||||
style C fill:#e65100,stroke:#bf360c,color:#fff
|
||||
style D fill:#4a148c,stroke:#2e0d57,color:#fff
|
||||
```
|
||||
|
||||
> For a detailed explanation of head vs. tail sampling, see Slide 9.
|
||||
|
||||
### Kill Switches (Rollback Options)
|
||||
|
||||
1. **Config Disable**: Set `enabled=0` in config → instant disable, no restart needed for sampling
|
||||
2. **Rebuild**: Compile with `XRPL_ENABLE_TELEMETRY=OFF` → zero overhead (no-op)
|
||||
3. **Full Revert**: Clean separation allows easy commit reversion
|
||||
|
||||
---
|
||||
|
||||
## Slide 9: Sampling Strategies — Head vs. Tail
|
||||
|
||||
> Sampling controls **which traces are recorded and exported**. Without sampling, every operation generates a trace — at 500+ spans/sec, this overwhelms storage and network. Sampling lets you keep the signal, discard the noise.
|
||||
|
||||
### Head Sampling (Decision at Start)
|
||||
|
||||
The sampling decision is made **when a trace begins**, before any work is done. A random number is generated; if it falls within the configured ratio, the entire trace is recorded. Otherwise, the trace is silently dropped.
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
A["New Request<br/>Arrives"] --> B{"Random < 10%?"}
|
||||
B -->|"Yes (1 in 10)"| C["Record Entire Trace<br/>(all spans)"]
|
||||
B -->|"No (9 in 10)"| D["Drop Entire Trace<br/>(zero overhead)"]
|
||||
|
||||
style C fill:#2e7d32,stroke:#1b5e20,color:#fff
|
||||
style D fill:#c62828,stroke:#8c2809,color:#fff
|
||||
style B fill:#1565c0,stroke:#0d47a1,color:#fff
|
||||
```
|
||||
|
||||
| Aspect | Details |
|
||||
| ----------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| **Where it runs** | Inside xrpld (SDK-level). In xrpld the ratio is fixed at 1.0 and not read from config (tail sampling in the collector needs every span) — the example below shows the general head-sampling mechanism. |
|
||||
| **When the decision happens** | At trace creation time — before the first span is even populated. |
|
||||
| **How it works** | `sampling_ratio=0.1` means each trace has a 10% probability of being recorded. Dropped traces incur near-zero overhead (no spans created, no attributes set, no export). |
|
||||
| **Propagation** | Once a trace is sampled, the `trace_flags` field (1 byte in the context header) tells downstream nodes to also sample it. Unsampled traces propagate `trace_flags=0`, so downstream nodes skip them too. |
|
||||
| **Pros** | Lowest overhead. Simple to configure. Predictable resource usage. |
|
||||
| **Cons** | **Blind** — it doesn't know if the trace will be interesting. A rare error or slow consensus round has only a 10% chance of being captured. |
|
||||
| **Best for** | High-volume, steady-state traffic where most traces look similar (e.g., routine RPC requests). |
|
||||
|
||||
**xrpld configuration**:
|
||||
|
||||
```ini
|
||||
[telemetry]
|
||||
# xrpld fixes head sampling at 1.0 (record every trace). This value is
|
||||
# NOT read from config — the collector performs tail sampling instead,
|
||||
# which needs all spans to arrive. See Slide 9 (tail sampling) and §7.4.2.
|
||||
sampling_ratio=1.0
|
||||
```
|
||||
|
||||
### Tail Sampling (Decision at End)
|
||||
|
||||
The sampling decision is made **after the trace completes**, based on its actual content — was it slow? Did it error? Was it a consensus round? This requires buffering complete traces before deciding.
|
||||
|
||||
```mermaid
|
||||
flowchart TB
|
||||
A["All Traces<br/>Buffered (100%)"] --> B["OTel Collector<br/>Evaluates Rules"]
|
||||
|
||||
B --> C{"Error?"}
|
||||
C -->|Yes| K["KEEP"]
|
||||
|
||||
C -->|No| D{"Slow?<br/>(>5s consensus,<br/>>1s RPC)"}
|
||||
D -->|Yes| K
|
||||
|
||||
D -->|No| E{"Random < 10%?"}
|
||||
E -->|Yes| K
|
||||
E -->|No| F["DROP"]
|
||||
|
||||
style K fill:#2e7d32,stroke:#1b5e20,color:#fff
|
||||
style F fill:#c62828,stroke:#8c2809,color:#fff
|
||||
style B fill:#1565c0,stroke:#0d47a1,color:#fff
|
||||
style C fill:#e65100,stroke:#bf360c,color:#fff
|
||||
style D fill:#e65100,stroke:#bf360c,color:#fff
|
||||
style E fill:#4a148c,stroke:#2e0d57,color:#fff
|
||||
```
|
||||
|
||||
| Aspect | Details |
|
||||
| ----------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| **Where it runs** | In the **OTel Collector** (external process), not inside xrpld. xrpld exports 100% of traces; the Collector decides what to keep. |
|
||||
| **When the decision happens** | After the Collector has received all spans for a trace (waits `decision_wait=10s` for stragglers). |
|
||||
| **How it works** | Policy rules evaluate the completed trace: keep all errors, keep slow operations above a threshold, keep all consensus rounds, then probabilistically sample the rest at 10%. |
|
||||
| **Pros** | **Never misses important traces**. Errors, slow requests, and consensus anomalies are always captured regardless of probability. |
|
||||
| **Cons** | Higher resource usage — xrpld must export 100% of spans to the Collector, which buffers them in memory before deciding. The Collector needs more RAM (configured via `num_traces` and `decision_wait`). |
|
||||
| **Best for** | Production troubleshooting where you can't afford to miss errors or anomalies. |
|
||||
|
||||
**Collector configuration** (tail sampling rules for xrpld):
|
||||
|
||||
```yaml
|
||||
processors:
|
||||
tail_sampling:
|
||||
decision_wait: 10s # Wait for all spans in a trace
|
||||
num_traces: 100000 # Buffer up to 100K concurrent traces
|
||||
policies:
|
||||
- name: errors # Always keep error traces
|
||||
type: status_code
|
||||
status_code: { status_codes: [ERROR] }
|
||||
|
||||
- name: slow-consensus # Keep consensus rounds >5s
|
||||
type: latency
|
||||
latency: { threshold_ms: 5000 }
|
||||
|
||||
- name: slow-rpc # Keep slow RPC requests >1s
|
||||
type: latency
|
||||
latency: { threshold_ms: 1000 }
|
||||
|
||||
- name: probabilistic # Sample 10% of everything else
|
||||
type: probabilistic
|
||||
probabilistic: { sampling_percentage: 10 }
|
||||
```
|
||||
|
||||
### Head vs. Tail — Side-by-Side
|
||||
|
||||
| | Head Sampling | Tail Sampling |
|
||||
| ----------------------------- | ----------------------------------------- | ------------------------------------------------ |
|
||||
| **Decision point** | Trace start (inside xrpld) | Trace end (in OTel Collector) |
|
||||
| **Knows trace content?** | No (random coin flip) | Yes (evaluates completed trace) |
|
||||
| **Overhead on xrpld** | Lowest (dropped traces = no-op) | Higher (must export 100% to Collector) |
|
||||
| **Collector resource usage** | Low (receives only sampled traces) | Higher (buffers all traces before deciding) |
|
||||
| **Captures all errors?** | No (only if trace was randomly selected) | **Yes** (error policy catches them) |
|
||||
| **Captures slow operations?** | No (random) | **Yes** (latency policy catches them) |
|
||||
| **Configuration** | Fixed at `1.0` in xrpld (not config-read) | `otel-collector.yaml`: `tail_sampling` processor |
|
||||
| **Best for** | High-throughput steady-state | Troubleshooting & anomaly detection |
|
||||
|
||||
### Recommended Strategy for xrpld
|
||||
|
||||
Use **both** in a layered approach:
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
subgraph xrpld["xrpld (Head Sampling)"]
|
||||
HS["sampling_ratio=1.0<br/>(export everything)"]
|
||||
end
|
||||
|
||||
subgraph collector["OTel Collector (Tail Sampling)"]
|
||||
TS["Keep: errors + slow + 10% random<br/>Drop: routine traces"]
|
||||
end
|
||||
|
||||
subgraph storage["Backend Storage"]
|
||||
ST["Only interesting traces<br/>stored long-term"]
|
||||
end
|
||||
|
||||
xrpld -->|"100% of spans"| collector -->|"~15-20% kept"| storage
|
||||
|
||||
style xrpld fill:#424242,stroke:#212121,color:#fff
|
||||
style collector fill:#1565c0,stroke:#0d47a1,color:#fff
|
||||
style storage fill:#2e7d32,stroke:#1b5e20,color:#fff
|
||||
```
|
||||
|
||||
> **Why this works**: xrpld exports everything (no blind drops), the Collector applies intelligent filtering (keep errors/slow/anomalies, sample the rest), and only ~15-20% of traces reach storage. xrpld's head sampling is fixed at 1.0 and not configurable, because tail sampling can only see traces that reach the Collector — any head drop would blind the error/slow policies. To reduce volume, tune the Collector's tail-sampling rules rather than adding head sampling.
|
||||
|
||||
---
|
||||
|
||||
## Slide 10: Data Collection & Privacy
|
||||
|
||||
### What Data is Collected
|
||||
|
||||
| Category | Attributes Collected | Purpose |
|
||||
| --------------- | -------------------------------------------------------------------------------------------------------------------- | --------------------------- |
|
||||
| **Transaction** | `tx_hash`, `tx_type`, `tx_result`, `tx_fee`, `ledger_index` | Trace transaction lifecycle |
|
||||
| **Consensus** | `consensus_round`, `consensus_phase`, `consensus_mode`, `proposers` (count of proposing validators), `round_time_ms` | Analyze consensus timing |
|
||||
| **RPC** | `command`, `version`, `rpc_status`, `duration_ms` | Monitor RPC performance |
|
||||
| **Peer** | `peer_id`(public key), `peer_latency_ms`, `message_type`, `message_size_bytes` | Network topology analysis |
|
||||
| **Ledger** | `ledger_hash`, `ledger_index`, `close_time`, `ledger_tx_count` | Ledger progression tracking |
|
||||
| **Job** | `job_type`, `job_queue_ms`, `job_worker` | JobQueue performance |
|
||||
|
||||
### What is NOT Collected (Privacy Guarantees)
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
subgraph notCollected["❌ NOT Collected"]
|
||||
direction LR
|
||||
A["Private Keys"] ~~~ B["Account Balances"] ~~~ C["Transaction Amounts"]
|
||||
end
|
||||
|
||||
subgraph alsoNot["❌ Also Excluded"]
|
||||
direction LR
|
||||
D["IP Addresses<br/>(configurable)"] ~~~ E["Personal Data"] ~~~ F["Raw TX Payloads"]
|
||||
end
|
||||
|
||||
style A fill:#c62828,stroke:#8c2809,color:#fff
|
||||
style B fill:#c62828,stroke:#8c2809,color:#fff
|
||||
style C fill:#c62828,stroke:#8c2809,color:#fff
|
||||
style D fill:#c62828,stroke:#8c2809,color:#fff
|
||||
style E fill:#c62828,stroke:#8c2809,color:#fff
|
||||
style F fill:#c62828,stroke:#8c2809,color:#fff
|
||||
```
|
||||
|
||||
**Reading the diagram:**
|
||||
|
||||
- **NOT Collected (top row, red)**: Private Keys, Account Balances, and Transaction Amounts are explicitly excluded — these are financial/security-sensitive fields that telemetry never touches.
|
||||
- **Also Excluded (bottom row, red)**: IP Addresses (configurable per deployment), Personal Data, and Raw TX Payloads are also excluded — these protect operator and user privacy.
|
||||
- **All-red styling**: Every box is styled in red to visually reinforce that these are hard exclusions, not optional — the telemetry system has no code path to collect any of these fields.
|
||||
- **Two-row layout**: The split between "NOT Collected" and "Also Excluded" distinguishes between financial data (top) and operational/personal data (bottom), making the privacy boundaries clear to auditors.
|
||||
|
||||
### Privacy Protection Mechanisms
|
||||
|
||||
| Mechanism | Description |
|
||||
| -------------------------- | --------------------------------------------------------- |
|
||||
| **Account Hashing** | `tx_account` is hashed at collector level before storage |
|
||||
| **Configurable Redaction** | Sensitive fields can be excluded via config |
|
||||
| **Sampling** | Only 10% of traces recorded by default (reduces exposure) |
|
||||
| **Local Control** | Node operators control what gets exported |
|
||||
| **No Raw Payloads** | Transaction content is never recorded, only metadata |
|
||||
|
||||
> **Key Principle**: Telemetry collects **operational metadata** (timing, counts, hashes) — never **sensitive content** (keys, balances, amounts).
|
||||
|
||||
---
|
||||
|
||||
_End of Presentation_
|
||||
@@ -203,10 +203,13 @@ target_link_libraries(
|
||||
# When telemetry=ON, links the Conan-provided umbrella target
|
||||
# opentelemetry-cpp::opentelemetry-cpp (individual component targets like
|
||||
# ::api, ::sdk are not available in the Conan package).
|
||||
#
|
||||
# Links xrpl.libxrpl.protocol PRIVATELY for sha512Half (digest.h)
|
||||
add_module(xrpl telemetry)
|
||||
target_link_libraries(
|
||||
xrpl.libxrpl.telemetry
|
||||
PUBLIC xrpl.libxrpl.basics xrpl.libxrpl.beast xrpl.libxrpl.config
|
||||
PRIVATE xrpl.libxrpl.protocol
|
||||
)
|
||||
if(telemetry)
|
||||
target_link_libraries(
|
||||
|
||||
@@ -38,7 +38,7 @@ datasources:
|
||||
tag: service.name
|
||||
operator: "="
|
||||
scope: resource
|
||||
type: static
|
||||
type: dynamic
|
||||
# service.instance.id: unique node identifier — defaults to the
|
||||
# node's public key (e.g., nHB1X37...). Distinguishes individual
|
||||
# nodes in a multi-node cluster or network.
|
||||
@@ -46,7 +46,7 @@ datasources:
|
||||
tag: service.instance.id
|
||||
operator: "="
|
||||
scope: resource
|
||||
type: static
|
||||
type: dynamic
|
||||
# service.version: xrpld build version (e.g., "2.4.0-b1").
|
||||
# Filter traces from specific software releases.
|
||||
- id: node-version
|
||||
@@ -68,7 +68,7 @@ datasources:
|
||||
tag: xrpl.network.type
|
||||
operator: "="
|
||||
scope: resource
|
||||
type: static
|
||||
type: dynamic
|
||||
# --- Span intrinsic filters ---
|
||||
# name: the span operation name (e.g., "rpc.command.server_info").
|
||||
# Use to find traces for a specific RPC command or subsystem.
|
||||
@@ -76,21 +76,21 @@ datasources:
|
||||
tag: name
|
||||
operator: "="
|
||||
scope: intrinsic
|
||||
type: static
|
||||
type: dynamic
|
||||
# status: span completion status ("ok", "error", "unset").
|
||||
# Filter for failed operations to diagnose errors.
|
||||
- id: span-status
|
||||
tag: status
|
||||
operator: "="
|
||||
scope: intrinsic
|
||||
type: static
|
||||
type: dynamic
|
||||
# duration: span wall-clock duration. Use with ">" operator
|
||||
# to find slow operations (e.g., duration > 500ms).
|
||||
- id: span-duration
|
||||
tag: duration
|
||||
operator: ">"
|
||||
scope: intrinsic
|
||||
type: static
|
||||
type: dynamic
|
||||
# Phase 2: RPC tracing filters
|
||||
- id: rpc-command
|
||||
tag: command
|
||||
|
||||
@@ -48,6 +48,15 @@ processors:
|
||||
action: delete
|
||||
- key: telemetry.sdk.version
|
||||
action: delete
|
||||
# Defense-in-depth: hash path-finding account attributes. The xrpld SDK
|
||||
# already hashes these before export, but a node that emitted raw values
|
||||
# is caught here so raw addresses never reach the backend.
|
||||
attributes/hash:
|
||||
actions:
|
||||
- key: pathfind_source_account
|
||||
action: hash
|
||||
- key: pathfind_dest_account
|
||||
action: hash
|
||||
|
||||
exporters:
|
||||
debug:
|
||||
@@ -66,5 +75,5 @@ service:
|
||||
pipelines:
|
||||
traces:
|
||||
receivers: [otlp]
|
||||
processors: [resource/tier, resource/stripsdk, batch]
|
||||
processors: [resource/tier, resource/stripsdk, attributes/hash, batch]
|
||||
exporters: [debug, otlp/tempo]
|
||||
|
||||
56
include/xrpl/telemetry/Redaction.h
Normal file
56
include/xrpl/telemetry/Redaction.h
Normal file
@@ -0,0 +1,56 @@
|
||||
#pragma once
|
||||
|
||||
/** Account-address redaction for telemetry span attributes.
|
||||
|
||||
Path-finding RPC handlers would otherwise emit the caller's raw
|
||||
account addresses as span attributes. To keep addresses out of the
|
||||
telemetry backend, they are hashed at the point of emission. This
|
||||
header exposes a single pure helper that turns an address into a
|
||||
short, stable, non-reversible token.
|
||||
|
||||
Data flow:
|
||||
|
||||
handler -> redactAccount(addr) -> span attribute -> OTLP export
|
||||
|
||||
The returned token is the first 16 hex characters (lowercase) of the
|
||||
SHA-512Half digest of the address. It is deterministic (same address
|
||||
always maps to the same token) so operators can still correlate spans
|
||||
for a given account, but the original address cannot be recovered.
|
||||
|
||||
A second, independent hashing layer runs in the OpenTelemetry
|
||||
Collector (an `attributes/hash` processor) as defense-in-depth for
|
||||
any node that emits a raw value.
|
||||
|
||||
@note This function is pure and reentrant: it holds no global state,
|
||||
performs no I/O, and is safe to call concurrently from any thread.
|
||||
|
||||
Usage example:
|
||||
@code
|
||||
#include <xrpl/telemetry/Redaction.h>
|
||||
using namespace xrpl::telemetry;
|
||||
|
||||
span.setAttribute(
|
||||
pathfind_span::attr::sourceAccount, redactAccount(src.asString()));
|
||||
@endcode
|
||||
|
||||
Edge case (empty input yields empty output):
|
||||
@code
|
||||
assert(redactAccount("") == "");
|
||||
@endcode
|
||||
*/
|
||||
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
|
||||
namespace xrpl::telemetry {
|
||||
|
||||
/** Hash an account address into a short, stable, non-reversible token.
|
||||
|
||||
@param addr The account address to redact (e.g. an r-address).
|
||||
@return The first 16 lowercase hex characters of sha512Half(addr),
|
||||
or an empty string when @p addr is empty.
|
||||
*/
|
||||
[[nodiscard]] std::string
|
||||
redactAccount(std::string_view addr);
|
||||
|
||||
} // namespace xrpl::telemetry
|
||||
32
src/libxrpl/telemetry/Redaction.cpp
Normal file
32
src/libxrpl/telemetry/Redaction.cpp
Normal file
@@ -0,0 +1,32 @@
|
||||
#include <xrpl/telemetry/Redaction.h>
|
||||
|
||||
#include <xrpl/basics/Slice.h>
|
||||
#include <xrpl/basics/base_uint.h>
|
||||
#include <xrpl/protocol/digest.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <cctype>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
|
||||
namespace xrpl::telemetry {
|
||||
|
||||
std::string
|
||||
redactAccount(std::string_view addr)
|
||||
{
|
||||
// Empty in, empty out: nothing to hash and no token to emit.
|
||||
if (addr.empty())
|
||||
return {};
|
||||
|
||||
// sha512Half yields a uint256; to_string renders it as uppercase hex.
|
||||
// Keep the first 16 chars (64 bits) — enough to correlate spans while
|
||||
// staying non-reversible — and lowercase them for a stable token.
|
||||
auto const digest = sha512Half(Slice(addr.data(), addr.size()));
|
||||
std::string token = to_string(digest).substr(0, 16);
|
||||
std::transform(token.begin(), token.end(), token.begin(), [](unsigned char c) {
|
||||
return static_cast<char>(std::tolower(c));
|
||||
});
|
||||
return token;
|
||||
}
|
||||
|
||||
} // namespace xrpl::telemetry
|
||||
@@ -235,6 +235,7 @@ SpanGuard::linkedSpan(std::string_view name) const
|
||||
return SpanGuard(
|
||||
std::make_unique<Impl>(tracer->StartSpan(
|
||||
std::string(name), {}, {{spanCtx, {{kLinkTypeKey, kLinkTypeFollowsFrom}}}}, opts)));
|
||||
// LCOV_EXCL_STOP
|
||||
}
|
||||
|
||||
SpanGuard
|
||||
|
||||
44
src/tests/libxrpl/telemetry/Redaction.cpp
Normal file
44
src/tests/libxrpl/telemetry/Redaction.cpp
Normal file
@@ -0,0 +1,44 @@
|
||||
#include <xrpl/telemetry/Redaction.h>
|
||||
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <cctype>
|
||||
#include <string>
|
||||
|
||||
using namespace xrpl;
|
||||
using namespace xrpl::telemetry;
|
||||
|
||||
// Empty input must produce empty output (edge / negative path).
|
||||
TEST(Redaction, empty_input_returns_empty)
|
||||
{
|
||||
EXPECT_EQ(redactAccount(""), "");
|
||||
}
|
||||
|
||||
// Success path: assert the EXACT expected token. The value is the first
|
||||
// 16 chars of sha512Half(addr) rendered as lowercase hex, computed once
|
||||
// and hard-coded here so any change to the hashing recipe is caught.
|
||||
TEST(Redaction, known_account_exact_hash)
|
||||
{
|
||||
EXPECT_EQ(redactAccount("rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh"), "a513895f49311f54");
|
||||
}
|
||||
|
||||
// Structural invariants: length, lowercase-hex alphabet, and stability.
|
||||
TEST(Redaction, token_is_16_lowercase_hex_and_stable)
|
||||
{
|
||||
auto const token = redactAccount("rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh");
|
||||
EXPECT_EQ(token.size(), 16u);
|
||||
EXPECT_TRUE(std::all_of(token.begin(), token.end(), [](unsigned char c) {
|
||||
return std::isdigit(c) || (c >= 'a' && c <= 'f');
|
||||
}));
|
||||
// Deterministic: hashing the same input twice yields the same token.
|
||||
EXPECT_EQ(token, redactAccount("rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh"));
|
||||
}
|
||||
|
||||
// Distinct inputs must produce distinct tokens (no accidental constant).
|
||||
TEST(Redaction, distinct_inputs_distinct_tokens)
|
||||
{
|
||||
EXPECT_NE(
|
||||
redactAccount("rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh"),
|
||||
redactAccount("rDifferentAccount1234567890abcdefgh"));
|
||||
}
|
||||
@@ -84,8 +84,6 @@ inline constexpr auto numPaths = makeStr("pathfind_num_paths");
|
||||
inline constexpr auto numRequests = makeStr("pathfind_num_requests");
|
||||
/// "pathfind_ledger_index" — pathfind target ledger index.
|
||||
inline constexpr auto ledgerIndex = makeStr("pathfind_ledger_index");
|
||||
/// "pathfind_dest_amount" — requested destination amount as string.
|
||||
inline constexpr auto destAmount = makeStr("pathfind_dest_amount");
|
||||
/// "pathfind_dest_currency" — destination currency code.
|
||||
inline constexpr auto destCurrency = makeStr("pathfind_dest_currency");
|
||||
/// "pathfind_num_source_assets" — candidate source assets count.
|
||||
|
||||
@@ -742,7 +742,6 @@ PathRequest::doUpdate(
|
||||
auto span = SpanGuard::span(
|
||||
TraceCategory::Rpc, pathfind_span::prefix::pathfind, pathfind_span::op::compute);
|
||||
span.setAttribute(pathfind_span::attr::fast, fast);
|
||||
span.setAttribute(pathfind_span::attr::destAmount, saDstAmount_.getFullText().c_str());
|
||||
span.setAttribute(pathfind_span::attr::destCurrency, to_string(saDstAmount_.asset()).c_str());
|
||||
|
||||
JLOG(journal_.debug()) << iIdentifier_ << " update " << (fast ? "fast" : "normal");
|
||||
|
||||
@@ -73,15 +73,16 @@ PathRequestManager::updateAll(std::shared_ptr<ReadView const> const& inLedger)
|
||||
cache = getAssetCache(inLedger, true);
|
||||
}
|
||||
|
||||
// updateAll runs on every ledger close; skip span emission entirely when
|
||||
// there are no active path subscriptions to avoid a steady stream of empty
|
||||
// spans at mainnet close cadence.
|
||||
if (requests.empty())
|
||||
return;
|
||||
|
||||
using namespace telemetry;
|
||||
auto span = SpanGuard::span(
|
||||
TraceCategory::Rpc, pathfind_span::prefix::pathfind, pathfind_span::op::updateAll);
|
||||
// updateAll runs on every ledger close. Skip span emission when there are
|
||||
// no active path subscriptions, to avoid a steady stream of empty spans at
|
||||
// mainnet close cadence. A null guard is used in that case; all other work
|
||||
// still runs unchanged (notably the isNewPathRequest() flag reset below),
|
||||
// so behaviour matches the pre-span code path.
|
||||
auto span = requests.empty()
|
||||
? SpanGuard{}
|
||||
: SpanGuard::span(
|
||||
TraceCategory::Rpc, pathfind_span::prefix::pathfind, pathfind_span::op::updateAll);
|
||||
span.setAttribute(pathfind_span::attr::ledgerIndex, static_cast<int64_t>(inLedger->seq()));
|
||||
span.setAttribute(pathfind_span::attr::numRequests, static_cast<int64_t>(requests.size()));
|
||||
|
||||
|
||||
@@ -221,6 +221,31 @@ callMethod(JsonContext& context, Method method, std::string const& name, Object&
|
||||
}
|
||||
}
|
||||
|
||||
// Resolve the span suffix / command attribute for a request that failed in
|
||||
// fillHandler. Returns the canonical handler name for a recognized command
|
||||
// (a finite, bounded set) or the literal "unknown" for a request that omits
|
||||
// both fields or names an unregistered command. The raw request value is
|
||||
// deliberately NOT used: the command attribute is promoted to a Prometheus
|
||||
// label by the spanmetrics connector, so an attacker-controlled string would
|
||||
// let arbitrary request input drive unbounded span-name / label cardinality.
|
||||
// Resolving against the registry keeps per-command error attribution for real
|
||||
// commands (e.g. a submit rejected with rpcTOO_BUSY stays rpc.command.submit)
|
||||
// while collapsing garbage input to a single series.
|
||||
std::string_view
|
||||
resolveCommandSpanName(JsonContext const& context)
|
||||
{
|
||||
if (!context.params.isMember(jss::command) && !context.params.isMember(jss::method))
|
||||
return rpc_span::val::unknownCommand;
|
||||
|
||||
std::string const cmd = context.params.isMember(jss::command)
|
||||
? context.params[jss::command].asString()
|
||||
: context.params[jss::method].asString();
|
||||
|
||||
auto const* handler = getHandler(context.apiVersion, context.app.config().betaRpcApi, cmd);
|
||||
return (handler != nullptr) ? std::string_view{handler->name}
|
||||
: std::string_view{rpc_span::val::unknownCommand};
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
Status
|
||||
@@ -229,25 +254,12 @@ doCommand(RPC::JsonContext& context, json::Value& result)
|
||||
Handler const* handler = nullptr;
|
||||
if (auto error = fillHandler(context, handler))
|
||||
{
|
||||
std::string cmdName;
|
||||
if (context.params.isMember(jss::command))
|
||||
{
|
||||
cmdName = context.params[jss::command].asString();
|
||||
}
|
||||
else if (context.params.isMember(jss::method))
|
||||
{
|
||||
cmdName = context.params[jss::method].asString();
|
||||
}
|
||||
else
|
||||
{
|
||||
cmdName = "unknown"; // LCOV_EXCL_LINE
|
||||
}
|
||||
// Use the resolved command name as the span suffix so dashboards
|
||||
// can break out per-command error rates (e.g. rpc.command.submit
|
||||
// for a submit that hit rpcTOO_BUSY). Falling back to a single
|
||||
// "unknown" name only when the request truly omits both fields.
|
||||
// Bound the span name and command attribute to the finite set of
|
||||
// registered handler names (plus "unknown") — see the helper for why
|
||||
// raw request input must not reach the telemetry pipeline.
|
||||
auto const cmdName = resolveCommandSpanName(context);
|
||||
auto span = SpanGuard::span(TraceCategory::Rpc, rpc_span::prefix::command, cmdName);
|
||||
span.setAttribute(rpc_span::attr::command, cmdName.c_str());
|
||||
span.setAttribute(rpc_span::attr::command, cmdName);
|
||||
span.setAttribute(rpc_span::attr::rpcStatus, rpc_span::val::error);
|
||||
span.setError(getErrorInfo(error).token.cStr());
|
||||
|
||||
|
||||
@@ -10,6 +10,7 @@
|
||||
#include <xrpl/protocol/jss.h>
|
||||
#include <xrpl/resource/Fees.h>
|
||||
#include <xrpl/server/InfoSub.h>
|
||||
#include <xrpl/telemetry/Redaction.h>
|
||||
#include <xrpl/telemetry/SpanGuard.h>
|
||||
|
||||
namespace xrpl {
|
||||
@@ -20,10 +21,11 @@ doPathFind(RPC::JsonContext& context)
|
||||
using namespace telemetry;
|
||||
auto span = SpanGuard::span(
|
||||
TraceCategory::Rpc, pathfind_span::prefix::pathfind, pathfind_span::op::request);
|
||||
// Addresses are hashed before emission for privacy.
|
||||
if (auto const& src = context.params[jss::source_account]; src.isString())
|
||||
span.setAttribute(pathfind_span::attr::sourceAccount, src.asString());
|
||||
span.setAttribute(pathfind_span::attr::sourceAccount, redactAccount(src.asString()));
|
||||
if (auto const& dst = context.params[jss::destination_account]; dst.isString())
|
||||
span.setAttribute(pathfind_span::attr::destAccount, dst.asString());
|
||||
span.setAttribute(pathfind_span::attr::destAccount, redactAccount(dst.asString()));
|
||||
|
||||
if (context.app.config().pathSearchMax == 0)
|
||||
return rpcError(RpcNotSupported);
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
#include <xrpl/protocol/RPCErr.h>
|
||||
#include <xrpl/protocol/jss.h>
|
||||
#include <xrpl/resource/Fees.h>
|
||||
#include <xrpl/telemetry/Redaction.h>
|
||||
#include <xrpl/telemetry/SpanGuard.h>
|
||||
|
||||
#include <memory>
|
||||
@@ -28,10 +29,11 @@ doRipplePathFind(RPC::JsonContext& context)
|
||||
using namespace telemetry;
|
||||
auto span = SpanGuard::span(
|
||||
TraceCategory::Rpc, pathfind_span::prefix::pathfind, pathfind_span::op::request);
|
||||
// Addresses are hashed before emission for privacy.
|
||||
if (auto const& src = context.params[jss::source_account]; src.isString())
|
||||
span.setAttribute(pathfind_span::attr::sourceAccount, src.asString());
|
||||
span.setAttribute(pathfind_span::attr::sourceAccount, redactAccount(src.asString()));
|
||||
if (auto const& dst = context.params[jss::destination_account]; dst.isString())
|
||||
span.setAttribute(pathfind_span::attr::destAccount, dst.asString());
|
||||
span.setAttribute(pathfind_span::attr::destAccount, redactAccount(dst.asString()));
|
||||
|
||||
if (context.app.config().pathSearchMax == 0)
|
||||
return rpcError(RpcNotSupported);
|
||||
|
||||
Reference in New Issue
Block a user