From dcf7d5550395aac3df6e238a9cc942407f177937 Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Wed, 8 Jul 2026 17:52:11 +0100 Subject: [PATCH 1/2] fix(telemetry): remove stale xrpld_ references from code comments Co-Authored-By: Claude Opus 4.6 --- src/xrpld/telemetry/MetricsRegistry.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/xrpld/telemetry/MetricsRegistry.cpp b/src/xrpld/telemetry/MetricsRegistry.cpp index e0e5855a05..73c40ffbef 100644 --- a/src/xrpld/telemetry/MetricsRegistry.cpp +++ b/src/xrpld/telemetry/MetricsRegistry.cpp @@ -262,9 +262,8 @@ MetricsRegistry::initSyncInstruments() "txq_expired_total", "Total transactions expired out of the transaction queue"); txqDroppedCounter_ = meter_->CreateUInt64Counter( "txq_dropped_total", "Total transactions refused admission to the queue by reason"); - // Note: xrpld_validation_agreements_total / xrpld_validation_missed_total - // are monotonic ObservableCounters created in registerValidationTotalsCounters() - // (below), observed from ValidationTracker's gross lifetime tallies. + // Note: validation_agreements_total / validation_missed_total are monotonic + // ObservableCounters created in registerValidationTotalsCounters() (below). } #endif // XRPL_ENABLE_TELEMETRY @@ -1364,7 +1363,7 @@ MetricsRegistry::registerValidationTotalsCounters() // count). We therefore observe the tracker's GROSS lifetime tallies, which // count each ledger once at first classification and are never adjusted on // repair (initial-classification semantics — see ValidationTracker). The - // repaired/agreement view remains available from xrpld_validation_agreement. + // repaired/agreement view remains available from validation_agreement. // // reconcile() is called first so pending events are resolved before the // tallies are read; the callback fires every ~10 s from the From 829cad81af1a798acb2acbe534a76b21e8e613ee Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Wed, 8 Jul 2026 18:01:53 +0100 Subject: [PATCH 2/2] docs(telemetry): update metric names across docs, tests, alert rules, and plan docs Strip xrpld_ prefix, lowercase beast::insight names, and replace traces_span_metrics_ with span_ in all remaining tracked files: alert rules, integration tests, workload validation, TESTING.md, OpenTelemetryPlan docs, code comments, and config templates. Co-Authored-By: Claude Opus 4.6 --- .cspell.config.yaml | 1 + .../05-configuration-reference.md | 2 +- OpenTelemetryPlan/06-implementation-phases.md | 238 +++++----- .../09-data-collection-reference.md | 436 +++++++++--------- docker/telemetry/TESTING.md | 10 +- .../grafana/provisioning/alerting/rules.yaml | 12 +- docker/telemetry/integration-test.sh | 54 +-- .../telemetry/workload/expected_metrics.json | 70 +-- .../workload/regression-metrics.json | 6 +- .../telemetry/workload/run-full-validation.sh | 2 +- .../telemetry/workload/validate_telemetry.py | 30 +- .../workload/xrpld-validator.cfg.template | 2 +- src/libxrpl/beast/insight/OTelCollector.cpp | 10 +- src/libxrpl/telemetry/Telemetry.cpp | 2 +- src/xrpld/telemetry/MetricsRegistry.h | 54 +-- src/xrpld/telemetry/ValidationTracker.h | 10 +- 16 files changed, 475 insertions(+), 464 deletions(-) diff --git a/.cspell.config.yaml b/.cspell.config.yaml index 0e8021c9a2..7ccd223863 100644 --- a/.cspell.config.yaml +++ b/.cspell.config.yaml @@ -303,6 +303,7 @@ words: - statsd - STATSDCOLLECTOR - stissue + - stledgerentry - stnum - stobj - stobject diff --git a/OpenTelemetryPlan/05-configuration-reference.md b/OpenTelemetryPlan/05-configuration-reference.md index 04b51d45a6..7e47023f51 100644 --- a/OpenTelemetryPlan/05-configuration-reference.md +++ b/OpenTelemetryPlan/05-configuration-reference.md @@ -266,7 +266,7 @@ In the Prometheus datasource, set `exemplarTraceIdDestinations` to map the `trac **Step 4: Dashboard panel with exemplars** -Add a timeseries panel over Prometheus (e.g. `histogram_quantile(0.99, rate(xrpld_rpc_duration_seconds_bucket[5m]))`) with `exemplar: true` enabled. +Add a timeseries panel over Prometheus (e.g. `histogram_quantile(0.99, rate(rpc_duration_seconds_bucket[5m]))`) with `exemplar: true` enabled. This allows clicking on metric data points to jump directly to the related trace. diff --git a/OpenTelemetryPlan/06-implementation-phases.md b/OpenTelemetryPlan/06-implementation-phases.md index a2546d9c11..dbdb204627 100644 --- a/OpenTelemetryPlan/06-implementation-phases.md +++ b/OpenTelemetryPlan/06-implementation-phases.md @@ -390,7 +390,7 @@ The `StatsDMeterImpl` in `StatsDCollector.cpp` sends metrics with `|m` suffix, w ### Exit Criteria -- [ ] StatsD metrics visible in Prometheus (`curl localhost:9090/api/v1/query?query=xrpld_LedgerMaster_Validated_Ledger_Age`) +- [ ] StatsD metrics visible in Prometheus (`curl localhost:9090/api/v1/query?query=ledgermaster_validated_ledger_age`) - [ ] All 3 new Grafana dashboards load without errors - [ ] Integration test verifies at least core StatsD metrics (ledger age, peer counts, RPC requests) - [ ] ~~Meter metrics (`warn`, `drop`) flow correctly after `|m` → `|c` fix~~ — DEFERRED (breaking change, tracked separately; resolved by Phase 7's OTel Counter mapping) @@ -500,7 +500,7 @@ graph LR BP -->|"OTLP/gRPC"| D SM -->|"RED metrics"| E - R1 -->|"xrpld_* metrics
(native OTLP)"| E + R1 -->|"system metrics
(native OTLP)"| E E --> F D --> F @@ -680,7 +680,7 @@ graph LR BP -->|"OTLP/gRPC"| D SM -->|"RED metrics"| E - R1 -->|"xrpld_* metrics
(native OTLP)"| E + R1 -->|"system metrics
(native OTLP)"| E E --> F D --> F @@ -1028,7 +1028,7 @@ flowchart LR ### Key Implementation Details - **Transaction submitter and RPC load generator** both use xrpld's native WebSocket command format (`{"command": ...}`) — not JSON-RPC format. Response data lives inside `"result"` with `"status"` at the top level. -- **Node config** requires `[signing_support] true` for server-side signing, and `[ips]` (not `[ips_fixed]`) to ensure peer connections count in `Peer_Finder_Active_*` metrics. +- **Node config** requires `[signing_support] true` for server-side signing, and `[ips]` (not `[ips_fixed]`) to ensure peer connections count in `peer_finder_active_*` metrics. - **Metric validation** uses the Prometheus `/api/v1/series` endpoint (not instant queries) to avoid false negatives from stale StatsD gauges. Every metric in `expected_metrics.json` must have > 0 series. - **StatsD gauge fix**: `StatsDGaugeImpl` initializes `m_dirty = true` so all gauges emit their initial value on first flush. Without this, gauges starting at 0 that never change (e.g. `jobq_job_count`) would be invisible in Prometheus. - **I/O latency fix**: `io_latency_sampler` emits unconditionally on first sample, then applies the 10 ms threshold. This ensures `ios_latency` is registered in Prometheus even in low-load CI environments. @@ -1057,7 +1057,7 @@ The validation suite (`validate_telemetry.py`) runs exactly 71 checks, broken do - **14 span attribute** — required attributes on the 14 spans that define them (22 unique attributes total) - **2 span hierarchies** — `rpc.process` -> `rpc.command.*`, `ledger.build` -> `tx.apply` (1 skipped: `rpc.request` -> `rpc.process`, cross-thread) - **1 span duration bounds** — all spans > 0 and < 60 s -- **26 metric existence** — 4 SpanMetrics (`traces_span_metrics_calls_total`, `..._duration_milliseconds_{bucket,count,sum}`), 6 StatsD gauges (`LedgerMaster_Validated_Ledger_Age`, `Published_Ledger_Age`, `State_Accounting_Full_duration`, `Peer_Finder_Active_{Inbound,Outbound}_Peers`, `jobq_job_count`), 2 StatsD counters (`rpc_requests_total`, `ledger_fetches_total`), 3 StatsD histograms (`rpc_time`, `rpc_size`, `ios_latency`), 4 overlay traffic (`total_Bytes_{In,Out}`, `total_Messages_{In,Out}`), 7 Phase 9 OTLP (`nodestore_state`, `cache_metrics`, `txq_metrics`, `rpc_method_{started,finished}_total`, `object_count`, `load_factor_metrics`) +- **26 metric existence** — 4 SpanMetrics (`span_calls_total`, `span_duration_milliseconds_{bucket,count,sum}`), 6 StatsD gauges (`ledgermaster_validated_ledger_age`, `published_ledger_age`, `state_accounting_full_duration`, `peer_finder_active_{inbound,outbound}_peers`, `jobq_job_count`), 2 StatsD counters (`rpc_requests_total`, `ledger_fetches_total`), 3 StatsD histograms (`rpc_time`, `rpc_size`, `ios_latency`), 4 overlay traffic (`total_bytes_{in,out}`, `total_messages_{in,out}`), 7 Phase 9 OTLP (`nodestore_state`, `cache_metrics`, `txq_metrics`, `rpc_method_{started,finished}_total`, `object_count`, `load_factor_metrics`) - **10 dashboard loads** — `rpc-performance`, `transaction-overview`, `consensus-health`, `ledger-operations`, `peer-network`, `node-health`, `network-traffic`, `rpc-pathfinding`, `overlay-traffic-detail`, `ledger-data-sync` See [Phase10_taskList.md](./Phase10_taskList.md) for the full numbered check-by-check enumeration. @@ -1701,12 +1701,12 @@ The overlay already tracks resource-limit disconnects via `OverlayImpl::Stats::p **What to do**: -- Ensure `xrpld_Overlay_Peer_Disconnects_Charges` appears in the StatsD-to-Prometheus metric name mapping +- Ensure `overlay_peer_disconnects_charges` appears in the StatsD-to-Prometheus metric name mapping - Verify the metric appears in Prometheus after StatsD bridge is active **File**: `src/xrpld/overlay/detail/OverlayImpl.cpp` -**Prometheus name**: `xrpld_Overlay_Peer_Disconnects_Charges` +**Prometheus name**: `overlay_peer_disconnects_charges` --- @@ -1813,12 +1813,12 @@ class ValidationTracker New MetricsRegistry observable gauge for amendment, UNL, and quorum health. -| Gauge Name | Label `metric=` | Type | Source | -| ------------------------ | ------------------- | ------ | ------------------------------------------------- | -| `xrpld_validator_health` | `amendment_blocked` | int64 | `app_.getOPs().isAmendmentBlocked()` → 0/1 | -| | `unl_blocked` | int64 | `app_.getOPs().isUNLBlocked()` → 0/1 | -| | `unl_expiry_days` | double | `app_.validators().expires()` → days until expiry | -| | `validation_quorum` | int64 | `app_.validators().quorum()` | +| Gauge Name | Label `metric=` | Type | Source | +| ------------------ | ------------------- | ------ | ------------------------------------------------- | +| `validator_health` | `amendment_blocked` | int64 | `app_.getOPs().isAmendmentBlocked()` → 0/1 | +| | `unl_blocked` | int64 | `app_.getOPs().isUNLBlocked()` → 0/1 | +| | `unl_expiry_days` | double | `app_.validators().expires()` → days until expiry | +| | `validation_quorum` | int64 | `app_.validators().quorum()` | **File**: `src/xrpld/telemetry/MetricsRegistry.cpp` (new gauge callback in `registerAsyncGauges()`) @@ -1834,12 +1834,12 @@ New MetricsRegistry observable gauge for amendment, UNL, and quorum health. New MetricsRegistry observable gauge for peer health aggregates. -| Gauge Name | Label `metric=` | Type | Source | -| -------------------- | -------------------------- | ------ | ------------------------------------------ | -| `xrpld_peer_quality` | `peer_latency_p90_ms` | double | Iterate peers, compute P90 from `latency_` | -| | `peers_insane_count` | int64 | Count peers with `tracking_ == diverged` | -| | `peers_higher_version_pct` | double | Compare `getVersion()` to own version | -| | `upgrade_recommended` | int64 | 1 if `peers_higher_version_pct > 60%` | +| Gauge Name | Label `metric=` | Type | Source | +| -------------- | -------------------------- | ------ | ------------------------------------------ | +| `peer_quality` | `peer_latency_p90_ms` | double | Iterate peers, compute P90 from `latency_` | +| | `peers_insane_count` | int64 | Count peers with `tracking_ == diverged` | +| | `peers_higher_version_pct` | double | Compare `getVersion()` to own version | +| | `upgrade_recommended` | int64 | 1 if `peers_higher_version_pct > 60%` | **Implementation note**: The callback iterates `app_.overlay().foreach(...)` to collect per-peer latency and version data. This runs every 10s on the metrics reader thread — acceptable overhead for ~50-200 peers. @@ -1858,13 +1858,13 @@ New MetricsRegistry observable gauge for peer health aggregates. New MetricsRegistry observable gauge for fee and ledger metrics. -| Gauge Name | Label `metric=` | Type | Source | -| ---------------------- | -------------------- | ------ | ----------------------------------------- | -| `xrpld_ledger_economy` | `base_fee_xrp` | double | `app_.getFeeTrack().getBaseFee()` → drops | -| | `reserve_base_xrp` | double | From validated ledger fee settings | -| | `reserve_inc_xrp` | double | From validated ledger fee settings | -| | `ledger_age_seconds` | double | `now - lastValidatedCloseTime` | -| | `transaction_rate` | double | Derived: tx count delta / time delta | +| Gauge Name | Label `metric=` | Type | Source | +| ---------------- | -------------------- | ------ | ----------------------------------------- | +| `ledger_economy` | `base_fee_xrp` | double | `app_.getFeeTrack().getBaseFee()` → drops | +| | `reserve_base_xrp` | double | From validated ledger fee settings | +| | `reserve_inc_xrp` | double | From validated ledger fee settings | +| | `ledger_age_seconds` | double | `now - lastValidatedCloseTime` | +| | `transaction_rate` | double | Derived: tx count delta / time delta | **File**: `src/xrpld/telemetry/MetricsRegistry.cpp` @@ -1880,10 +1880,10 @@ New MetricsRegistry observable gauge for fee and ledger metrics. New MetricsRegistry observable gauge for node state duration. -| Gauge Name | Label `metric=` | Type | Source | -| ---------------------- | ------------------------------- | ------ | ------------------------------------------------ | -| `xrpld_state_tracking` | `state_value` | int64 | 0-7 numeric encoding matching external dashboard | -| | `time_in_current_state_seconds` | double | `now - lastModeChangeTime` | +| Gauge Name | Label `metric=` | Type | Source | +| ---------------- | ------------------------------- | ------ | ------------------------------------------------ | +| `state_tracking` | `state_value` | int64 | 0-7 numeric encoding matching external dashboard | +| | `time_in_current_state_seconds` | double | `now - lastModeChangeTime` | **State value encoding**: @@ -1912,9 +1912,9 @@ xrpld's `OperatingMode` enum maps 0-4 (DISCONNECTED through FULL). The external **Task 7.13: Storage Detail Observable Gauge** -| Gauge Name | Label `metric=` | Type | Source | -| ---------------------- | --------------- | ----- | ---------------------------------------- | -| `xrpld_storage_detail` | `nudb_bytes` | int64 | NuDB backend file size (filesystem stat) | +| Gauge Name | Label `metric=` | Type | Source | +| ---------------- | --------------- | ----- | ---------------------------------------- | +| `storage_detail` | `nudb_bytes` | int64 | NuDB backend file size (filesystem stat) | **File**: `src/xrpld/telemetry/MetricsRegistry.cpp` @@ -1929,15 +1929,15 @@ xrpld's `OperatingMode` enum maps 0-4 (DISCONNECTED through FULL). The external New counters incremented at event sites. Declared in MetricsRegistry, recording sites added in consensus/overlay/network code. -| Counter Name | Increment Site | Source File | -| ----------------------------------- | -------------------------------- | --------------------- | -| `xrpld_ledgers_closed_total` | `onAccept()` in consensus | RCLConsensus.cpp | -| `xrpld_validations_sent_total` | `validate()` in consensus | RCLConsensus.cpp | -| `xrpld_validations_checked_total` | Network validation received | LedgerMaster.cpp | -| `xrpld_validation_agreements_total` | ValidationTracker reconciliation | ValidationTracker.cpp | -| `xrpld_validation_missed_total` | ValidationTracker reconciliation | ValidationTracker.cpp | -| `xrpld_state_changes_total` | `setMode()` in NetworkOPs | NetworkOPs.cpp | -| `xrpld_jq_trans_overflow_total` | Job queue overflow path | JobQueue.cpp | +| Counter Name | Increment Site | Source File | +| ----------------------------- | -------------------------------- | --------------------- | +| `ledgers_closed_total` | `onAccept()` in consensus | RCLConsensus.cpp | +| `validations_sent_total` | `validate()` in consensus | RCLConsensus.cpp | +| `validations_checked_total` | Network validation received | LedgerMaster.cpp | +| `validation_agreements_total` | ValidationTracker reconciliation | ValidationTracker.cpp | +| `validation_missed_total` | ValidationTracker reconciliation | ValidationTracker.cpp | +| `state_changes_total` | `setMode()` in NetworkOPs | NetworkOPs.cpp | +| `jq_trans_overflow_total` | Job queue overflow path | JobQueue.cpp | **Key modified files**: @@ -1958,14 +1958,14 @@ New counters incremented at event sites. Declared in MetricsRegistry, recording Reads from the `ValidationTracker` (Task 7.8) to export rolling window stats. -| Gauge Name | Label `metric=` | Type | Source | -| ---------------------------- | ------------------- | ------ | --------------------------- | -| `xrpld_validation_agreement` | `agreement_pct_1h` | double | `tracker.agreementPct1h()` | -| | `agreements_1h` | int64 | `tracker.agreements1h()` | -| | `missed_1h` | int64 | `tracker.missed1h()` | -| | `agreement_pct_24h` | double | `tracker.agreementPct24h()` | -| | `agreements_24h` | int64 | `tracker.agreements24h()` | -| | `missed_24h` | int64 | `tracker.missed24h()` | +| Gauge Name | Label `metric=` | Type | Source | +| ---------------------- | ------------------- | ------ | --------------------------- | +| `validation_agreement` | `agreement_pct_1h` | double | `tracker.agreementPct1h()` | +| | `agreements_1h` | int64 | `tracker.agreements1h()` | +| | `missed_1h` | int64 | `tracker.missed1h()` | +| | `agreement_pct_24h` | double | `tracker.agreementPct24h()` | +| | `agreements_24h` | int64 | `tracker.agreements24h()` | +| | `missed_24h` | int64 | `tracker.missed24h()` | **File**: `src/xrpld/telemetry/MetricsRegistry.cpp` @@ -1985,21 +1985,21 @@ Reads from the `ValidationTracker` (Task 7.8) to export rolling window stats. New Grafana dashboard: `validator-health.json` -| Panel | Type | PromQL | -| -------------------------- | ---------- | -------------------------------------------------------------- | -| Agreement % (1h) | stat | `xrpld_validation_agreement{metric="agreement_pct_1h"}` | -| Agreement % (24h) | stat | `xrpld_validation_agreement{metric="agreement_pct_24h"}` | -| Agreements vs Missed (1h) | bargauge | `agreements_1h` and `missed_1h` side by side | -| Agreements vs Missed (24h) | bargauge | `agreements_24h` and `missed_24h` side by side | -| Validation Rate | stat | `rate(xrpld_validations_sent_total[5m]) * 60` | -| Validations Checked Rate | stat | `rate(xrpld_validations_checked_total[5m]) * 60` | -| Amendment Blocked | stat | `xrpld_validator_health{metric="amendment_blocked"}` | -| UNL Expiry (days) | stat | `xrpld_validator_health{metric="unl_expiry_days"}` | -| Validation Quorum | stat | `xrpld_validator_health{metric="validation_quorum"}` | -| State Value Timeline | timeseries | `xrpld_state_tracking{metric="state_value"}` | -| Time in Current State | stat | `xrpld_state_tracking{metric="time_in_current_state_seconds"}` | -| State Changes Rate | stat | `rate(xrpld_state_changes_total[1h])` | -| Ledgers Closed Rate | stat | `rate(xrpld_ledgers_closed_total[5m]) * 60` | +| Panel | Type | PromQL | +| -------------------------- | ---------- | -------------------------------------------------------- | +| Agreement % (1h) | stat | `validation_agreement{metric="agreement_pct_1h"}` | +| Agreement % (24h) | stat | `validation_agreement{metric="agreement_pct_24h"}` | +| Agreements vs Missed (1h) | bargauge | `agreements_1h` and `missed_1h` side by side | +| Agreements vs Missed (24h) | bargauge | `agreements_24h` and `missed_24h` side by side | +| Validation Rate | stat | `rate(validations_sent_total[5m]) * 60` | +| Validations Checked Rate | stat | `rate(validations_checked_total[5m]) * 60` | +| Amendment Blocked | stat | `validator_health{metric="amendment_blocked"}` | +| UNL Expiry (days) | stat | `validator_health{metric="unl_expiry_days"}` | +| Validation Quorum | stat | `validator_health{metric="validation_quorum"}` | +| State Value Timeline | timeseries | `state_tracking{metric="state_value"}` | +| Time in Current State | stat | `state_tracking{metric="time_in_current_state_seconds"}` | +| State Changes Rate | stat | `rate(state_changes_total[1h])` | +| Ledgers Closed Rate | stat | `rate(ledgers_closed_total[5m]) * 60` | **Dashboard conventions**: `$node` template variable for `service_instance_id` filtering, dark theme, matching existing panel sizes and color schemes. @@ -2009,14 +2009,14 @@ New Grafana dashboard: `validator-health.json` New Grafana dashboard: `peer-quality.json` -| Panel | Type | PromQL | -| ---------------------- | ---------- | -------------------------------------------------------------- | -| P90 Peer Latency | timeseries | `xrpld_peer_quality{metric="peer_latency_p90_ms"}` | -| Insane/Diverged Peers | stat | `xrpld_peer_quality{metric="peers_insane_count"}` | -| Higher Version Peers % | stat | `xrpld_peer_quality{metric="peers_higher_version_pct"}` | -| Upgrade Recommended | stat | `xrpld_peer_quality{metric="upgrade_recommended"}` | -| Resource Disconnects | timeseries | `xrpld_Overlay_Peer_Disconnects_Charges` | -| Inbound vs Outbound | bargauge | `xrpld_Peer_Finder_Active_Inbound_Peers`, `..._Outbound_Peers` | +| Panel | Type | PromQL | +| ---------------------- | ---------- | -------------------------------------------------------- | +| P90 Peer Latency | timeseries | `peer_quality{metric="peer_latency_p90_ms"}` | +| Insane/Diverged Peers | stat | `peer_quality{metric="peers_insane_count"}` | +| Higher Version Peers % | stat | `peer_quality{metric="peers_higher_version_pct"}` | +| Upgrade Recommended | stat | `peer_quality{metric="upgrade_recommended"}` | +| Resource Disconnects | timeseries | `overlay_peer_disconnects_charges` | +| Inbound vs Outbound | bargauge | `peer_finder_active_inbound_peers`, `..._outbound_peers` | --- @@ -2024,13 +2024,13 @@ New Grafana dashboard: `peer-quality.json` Add a "Ledger Economy" row to the existing `node-health.json` dashboard: -| Panel | Type | PromQL | -| -------------------- | ---------- | --------------------------------------------------- | -| Base Fee (drops) | stat | `xrpld_ledger_economy{metric="base_fee_xrp"}` | -| Reserve Base (drops) | stat | `xrpld_ledger_economy{metric="reserve_base_xrp"}` | -| Reserve Inc (drops) | stat | `xrpld_ledger_economy{metric="reserve_inc_xrp"}` | -| Ledger Age | stat | `xrpld_ledger_economy{metric="ledger_age_seconds"}` | -| Transaction Rate | timeseries | `xrpld_ledger_economy{metric="transaction_rate"}` | +| Panel | Type | PromQL | +| -------------------- | ---------- | --------------------------------------------- | +| Base Fee (drops) | stat | `ledger_economy{metric="base_fee_xrp"}` | +| Reserve Base (drops) | stat | `ledger_economy{metric="reserve_base_xrp"}` | +| Reserve Inc (drops) | stat | `ledger_economy{metric="reserve_inc_xrp"}` | +| Ledger Age | stat | `ledger_economy{metric="ledger_age_seconds"}` | +| Transaction Rate | timeseries | `ledger_economy{metric="transaction_rate"}` | --- @@ -2057,21 +2057,21 @@ Add checks to `validate_telemetry.py` for all new span attributes and metrics. **New metric existence checks (~13)**: -| Metric Name | -| -------------------------------------------------------- | -| `xrpld_validation_agreement{metric="agreement_pct_1h"}` | -| `xrpld_validation_agreement{metric="agreement_pct_24h"}` | -| `xrpld_validator_health{metric="amendment_blocked"}` | -| `xrpld_validator_health{metric="unl_expiry_days"}` | -| `xrpld_peer_quality{metric="peer_latency_p90_ms"}` | -| `xrpld_peer_quality{metric="peers_insane_count"}` | -| `xrpld_ledger_economy{metric="base_fee_xrp"}` | -| `xrpld_ledger_economy{metric="transaction_rate"}` | -| `xrpld_state_tracking{metric="state_value"}` | -| `xrpld_ledgers_closed_total` | -| `xrpld_validations_sent_total` | -| `xrpld_state_changes_total` | -| `xrpld_storage_detail{metric="nudb_bytes"}` | +| Metric Name | +| -------------------------------------------------- | +| `validation_agreement{metric="agreement_pct_1h"}` | +| `validation_agreement{metric="agreement_pct_24h"}` | +| `validator_health{metric="amendment_blocked"}` | +| `validator_health{metric="unl_expiry_days"}` | +| `peer_quality{metric="peer_latency_p90_ms"}` | +| `peer_quality{metric="peers_insane_count"}` | +| `ledger_economy{metric="base_fee_xrp"}` | +| `ledger_economy{metric="transaction_rate"}` | +| `state_tracking{metric="state_value"}` | +| `ledgers_closed_total` | +| `validations_sent_total` | +| `state_changes_total` | +| `storage_detail{metric="nudb_bytes"}` | **New dashboard load checks (~3)**: @@ -2104,36 +2104,36 @@ Port 18 alert rules from the external `xrpl-validator-dashboard` to Grafana aler **Critical Group** (8 rules, eval interval 10s): -| Rule | Condition | For | -| ------------------- | ------------------------------------------------------------- | --- | -| Agreement Below 90% | `xrpld_validation_agreement{metric="agreement_pct_24h"} < 90` | 30s | -| Not Proposing | `xrpld_state_tracking{metric="state_value"} < 6` | 10s | -| Unhealthy State | `xrpld_state_tracking{metric="state_value"} < 4` | 10s | -| Amendment Blocked | `xrpld_validator_health{metric="amendment_blocked"} == 1` | 1m | -| UNL Expiring | `xrpld_validator_health{metric="unl_expiry_days"} < 14` | 1h | -| High IO Latency | `histogram_quantile(0.95, xrpld_ios_latency_bucket) > 50` | 1m | -| High Load Factor | `xrpld_load_factor_metrics{metric="load_factor"} > 1000` | 1m | -| Peer Count Critical | `xrpld_server_info{metric="peers"} < 5` | 1m | +| Rule | Condition | For | +| ------------------- | ------------------------------------------------------- | --- | +| Agreement Below 90% | `validation_agreement{metric="agreement_pct_24h"} < 90` | 30s | +| Not Proposing | `state_tracking{metric="state_value"} < 6` | 10s | +| Unhealthy State | `state_tracking{metric="state_value"} < 4` | 10s | +| Amendment Blocked | `validator_health{metric="amendment_blocked"} == 1` | 1m | +| UNL Expiring | `validator_health{metric="unl_expiry_days"} < 14` | 1h | +| High IO Latency | `histogram_quantile(0.95, ios_latency_bucket) > 50` | 1m | +| High Load Factor | `load_factor_metrics{metric="load_factor"} > 1000` | 1m | +| Peer Count Critical | `server_info{metric="peers"} < 5` | 1m | **Network Group** (3 rules, eval interval 10s): -| Rule | Condition | For | -| ------------------------- | ----------------------------------------------------------------- | --- | -| Peer Drop >10% | `delta(xrpld_server_info{metric="peers"}[30s]) / ... * 100 < -10` | 30s | -| Peer Drop >30% | Same formula, threshold -30 | 30s | -| P90 Latency + Disconnects | `peer_latency_p90_ms > 500 AND rate(disconnects) > 0` | 2m | +| Rule | Condition | For | +| ------------------------- | ----------------------------------------------------------- | --- | +| Peer Drop >10% | `delta(server_info{metric="peers"}[30s]) / ... * 100 < -10` | 30s | +| Peer Drop >30% | Same formula, threshold -30 | 30s | +| P90 Latency + Disconnects | `peer_latency_p90_ms > 500 AND rate(disconnects) > 0` | 2m | **Performance Group** (7 rules, eval interval 10s): -| Rule | Condition | For | -| ------------------- | ------------------------------------------------------------ | --- | -| CPU High | Per-core CPU > 80% | 2m | -| Memory Critical | Memory usage > 90% | 1m | -| Disk Warning | Disk usage > 85% | 2m | -| Job Queue Overflow | `rate(xrpld_jq_trans_overflow_total[5m]) > 0` | 1m | -| Upgrade Recommended | `xrpld_peer_quality{metric="peers_higher_version_pct"} > 60` | 1m | -| TX Rate Drop | Transaction rate dropped > 50% in 5m window | 5m | -| Stale Ledger | `xrpld_ledger_economy{metric="ledger_age_seconds"} > 30` | 1m | +| Rule | Condition | For | +| ------------------- | ------------------------------------------------------ | --- | +| CPU High | Per-core CPU > 80% | 2m | +| Memory Critical | Memory usage > 90% | 1m | +| Disk Warning | Disk usage > 85% | 2m | +| Job Queue Overflow | `rate(jq_trans_overflow_total[5m]) > 0` | 1m | +| Upgrade Recommended | `peer_quality{metric="peers_higher_version_pct"} > 60` | 1m | +| TX Rate Drop | Transaction rate dropped > 50% in 5m window | 5m | +| Stale Ledger | `ledger_economy{metric="ledger_age_seconds"} > 30` | 1m | **Notification channels**: Template configs for Email/SMTP, Discord, Slack, PagerDuty. diff --git a/OpenTelemetryPlan/09-data-collection-reference.md b/OpenTelemetryPlan/09-data-collection-reference.md index 756abb96fc..6e5b700e0e 100644 --- a/OpenTelemetryPlan/09-data-collection-reference.md +++ b/OpenTelemetryPlan/09-data-collection-reference.md @@ -43,7 +43,7 @@ graph LR BP -->|"OTLP/gRPC :4317"| D SM -->|"span_calls_total
span_duration_ms
(6 dimension labels)"| E - R1 -->|"xrpld_* gauges
xrpld_* counters
xrpld_* histograms"| E + R1 -->|"gauges, counters,
histograms (OTLP)"| E E -->|"Prometheus
data source"| F D -->|"Tempo
data source"| F @@ -426,12 +426,12 @@ the parent `ledger.build` carries `ledger_seq` and the close-time attributes. The OTel Collector's SpanMetrics connector automatically generates RED (Rate, Errors, Duration) metrics from every span. No custom metrics code in xrpld is needed. -| Prometheus Metric | Type | Description | -| -------------------------------------------------- | --------- | ------------------------------------------------------------------------------ | -| `traces_span_metrics_calls_total` | Counter | Total span invocations | -| `traces_span_metrics_duration_milliseconds_bucket` | Histogram | Latency distribution (buckets: 1, 5, 10, 25, 50, 100, 250, 500, 1000, 5000 ms) | -| `traces_span_metrics_duration_milliseconds_count` | Histogram | Observation count | -| `traces_span_metrics_duration_milliseconds_sum` | Histogram | Cumulative latency | +| Prometheus Metric | Type | Description | +| ----------------------------------- | --------- | ------------------------------------------------------------------------------ | +| `span_calls_total` | Counter | Total span invocations | +| `span_duration_milliseconds_bucket` | Histogram | Latency distribution (buckets: 1, 5, 10, 25, 50, 100, 250, 500, 1000, 5000 ms) | +| `span_duration_milliseconds_count` | Histogram | Observation count | +| `span_duration_milliseconds_sum` | Histogram | Cumulative latency | **Standard labels on every metric**: `span_name`, `status_code`, `service_name`, `span_kind` @@ -474,7 +474,7 @@ _Transaction Overview_ dashboard charts rate, p95 latency, and failure rate by s > the retained traces, whereas native StatsD/meter metrics do not sample. > Account for any collector-side tail sampling when reading absolute stage rates. -**Where to query**: Prometheus → `traces_span_metrics_calls_total{span_name="rpc.command.server_info"}` +**Where to query**: Prometheus → `span_calls_total{span_name="rpc.command.server_info"}` --- @@ -507,51 +507,51 @@ prefix=xrpld ### 2.1 Gauges -| Prometheus Metric | Source File | Description | Typical Range | -| ------------------------------------------------- | --------------------- | ----------------------------------------- | ------------------------------- | -| `xrpld_LedgerMaster_Validated_Ledger_Age` | LedgerMaster.h | Seconds since last validated ledger | 0–10 (healthy), >30 (stale) | -| `xrpld_LedgerMaster_Published_Ledger_Age` | LedgerMaster.h | Seconds since last published ledger | 0–10 (healthy) | -| `xrpld_State_Accounting_Disconnected_duration` | NetworkOPs.cpp | Cumulative seconds in Disconnected state | Monotonic | -| `xrpld_State_Accounting_Connected_duration` | NetworkOPs.cpp | Cumulative seconds in Connected state | Monotonic | -| `xrpld_State_Accounting_Syncing_duration` | NetworkOPs.cpp | Cumulative seconds in Syncing state | Monotonic | -| `xrpld_State_Accounting_Tracking_duration` | NetworkOPs.cpp | Cumulative seconds in Tracking state | Monotonic | -| `xrpld_State_Accounting_Full_duration` | NetworkOPs.cpp | Cumulative seconds in Full state | Monotonic (should dominate) | -| `xrpld_State_Accounting_Disconnected_transitions` | NetworkOPs.cpp | Count of transitions to Disconnected | Low | -| `xrpld_State_Accounting_Connected_transitions` | NetworkOPs.cpp | Count of transitions to Connected | Low | -| `xrpld_State_Accounting_Syncing_transitions` | NetworkOPs.cpp | Count of transitions to Syncing | Low | -| `xrpld_State_Accounting_Tracking_transitions` | NetworkOPs.cpp | Count of transitions to Tracking | Low | -| `xrpld_State_Accounting_Full_transitions` | NetworkOPs.cpp | Count of transitions to Full | Low (should be 1 after startup) | -| `xrpld_Peer_Finder_Active_Inbound_Peers` | PeerfinderManager.cpp | Active inbound peer connections | 0–85 | -| `xrpld_Peer_Finder_Active_Outbound_Peers` | PeerfinderManager.cpp | Active outbound peer connections | 10–21 | -| `xrpld_Overlay_Peer_Disconnects` | OverlayImpl.cpp | Cumulative peer disconnection count | Low growth | -| `xrpld_Overlay_Peer_Disconnects_Charges` | OverlayImpl.cpp | Disconnects due to resource limit charges | Low growth (subset of above) | -| `xrpld_jobq_job_count` | JobQueue.cpp | Current job queue depth (group `jobq`) | 0–100 (healthy) | +| Prometheus Metric | Source File | Description | Typical Range | +| ------------------------------------------- | --------------------- | ----------------------------------------- | ------------------------------- | +| `ledgermaster_validated_ledger_age` | LedgerMaster.h | Seconds since last validated ledger | 0–10 (healthy), >30 (stale) | +| `ledgermaster_published_ledger_age` | LedgerMaster.h | Seconds since last published ledger | 0–10 (healthy) | +| `state_accounting_disconnected_duration` | NetworkOPs.cpp | Cumulative seconds in Disconnected state | Monotonic | +| `state_accounting_connected_duration` | NetworkOPs.cpp | Cumulative seconds in Connected state | Monotonic | +| `state_accounting_syncing_duration` | NetworkOPs.cpp | Cumulative seconds in Syncing state | Monotonic | +| `state_accounting_tracking_duration` | NetworkOPs.cpp | Cumulative seconds in Tracking state | Monotonic | +| `state_accounting_full_duration` | NetworkOPs.cpp | Cumulative seconds in Full state | Monotonic (should dominate) | +| `state_accounting_disconnected_transitions` | NetworkOPs.cpp | Count of transitions to Disconnected | Low | +| `state_accounting_connected_transitions` | NetworkOPs.cpp | Count of transitions to Connected | Low | +| `state_accounting_syncing_transitions` | NetworkOPs.cpp | Count of transitions to Syncing | Low | +| `state_accounting_tracking_transitions` | NetworkOPs.cpp | Count of transitions to Tracking | Low | +| `state_accounting_full_transitions` | NetworkOPs.cpp | Count of transitions to Full | Low (should be 1 after startup) | +| `peer_finder_active_inbound_peers` | PeerfinderManager.cpp | Active inbound peer connections | 0–85 | +| `peer_finder_active_outbound_peers` | PeerfinderManager.cpp | Active outbound peer connections | 10–21 | +| `overlay_peer_disconnects` | OverlayImpl.cpp | Cumulative peer disconnection count | Low growth | +| `overlay_peer_disconnects_charges` | OverlayImpl.cpp | Disconnects due to resource limit charges | Low growth (subset of above) | +| `jobq_job_count` | JobQueue.cpp | Current job queue depth (group `jobq`) | 0–100 (healthy) | **Grafana dashboard**: _Node Health_ (`node-health`) ### 2.2 Counters -| Prometheus Metric | Source File | Description | -| ------------------------------- | ------------------ | --------------------------------------------- | -| `xrpld_rpc_requests` | ServerHandler.cpp | Total RPC requests received | -| `xrpld_ledger_fetches` | InboundLedgers.cpp | Inbound ledger fetch attempts | -| `xrpld_ledger_history_mismatch` | LedgerHistory.cpp | Ledger hash mismatches detected | -| `xrpld_warn` | Logic.h | Resource manager warnings issued | -| `xrpld_drop` | Logic.h | Resource manager drops (connections rejected) | +| Prometheus Metric | Source File | Description | +| ------------------------- | ------------------ | --------------------------------------------- | +| `rpc_requests` | ServerHandler.cpp | Total RPC requests received | +| `ledger_fetches` | InboundLedgers.cpp | Inbound ledger fetch attempts | +| `ledger_history_mismatch` | LedgerHistory.cpp | Ledger hash mismatches detected | +| `warn` | Logic.h | Resource manager warnings issued | +| `drop` | Logic.h | Resource manager drops (connections rejected) | -**Note**: With `server=otel`, `xrpld_warn` and `xrpld_drop` are properly exported as OTel Counter instruments. The previous StatsD `|m` type limitation no longer applies. +**Note**: With `server=otel`, `warn` and `drop` are properly exported as OTel Counter instruments. The previous StatsD `|m` type limitation no longer applies. **Grafana dashboard**: _RPC & Pathfinding_ (`rpc-pathfinding`) ### 2.3 Histograms (Event timers) -| Prometheus Metric | Source File | Unit | Description | -| --------------------- | ----------------- | ----- | ------------------------------ | -| `xrpld_rpc_time` | ServerHandler.cpp | ms | RPC response time distribution | -| `xrpld_rpc_size` | ServerHandler.cpp | bytes | RPC response size distribution | -| `xrpld_ios_latency` | Application.cpp | ms | I/O service loop latency | -| `xrpld_pathfind_fast` | PathRequests.h | ms | Fast pathfinding duration | -| `xrpld_pathfind_full` | PathRequests.h | ms | Full pathfinding duration | +| Prometheus Metric | Source File | Unit | Description | +| ----------------- | ----------------- | ----- | ------------------------------ | +| `rpc_time` | ServerHandler.cpp | ms | RPC response time distribution | +| `rpc_size` | ServerHandler.cpp | bytes | RPC response size distribution | +| `ios_latency` | Application.cpp | ms | I/O service loop latency | +| `pathfind_fast` | PathRequests.h | ms | Fast pathfinding duration | +| `pathfind_full` | PathRequests.h | ms | Full pathfinding duration | Quantiles collected: 0th, 50th, 90th, 95th, 99th, 100th percentile. @@ -561,10 +561,10 @@ Quantiles collected: 0th, 50th, 90th, 95th, 99th, 100th percentile. For each of the 45+ overlay traffic categories (defined in `TrafficCount.h`), four gauges are emitted: -- `xrpld_{category}_Bytes_In` -- `xrpld_{category}_Bytes_Out` -- `xrpld_{category}_Messages_In` -- `xrpld_{category}_Messages_Out` +- `{category}_bytes_in` +- `{category}_bytes_out` +- `{category}_messages_in` +- `{category}_messages_out` **Key categories**: @@ -695,38 +695,38 @@ ledger.store (persist to DB) ```promql # RPC request rate by command (last 5 minutes) -sum by (command) (rate(traces_span_metrics_calls_total{span_name=~"rpc.command.*"}[5m])) +sum by (command) (rate(span_calls_total{span_name=~"rpc.command.*"}[5m])) # RPC p95 latency by command -histogram_quantile(0.95, sum by (le, command) (rate(traces_span_metrics_duration_milliseconds_bucket{span_name=~"rpc.command.*"}[5m]))) +histogram_quantile(0.95, sum by (le, command) (rate(span_duration_milliseconds_bucket{span_name=~"rpc.command.*"}[5m]))) # Consensus round duration p95 -histogram_quantile(0.95, sum by (le) (rate(traces_span_metrics_duration_milliseconds_bucket{span_name="consensus.round"}[5m]))) +histogram_quantile(0.95, sum by (le) (rate(span_duration_milliseconds_bucket{span_name="consensus.round"}[5m]))) # Transaction processing rate (local vs relay) -sum by (local) (rate(traces_span_metrics_calls_total{span_name="tx.process"}[5m])) +sum by (local) (rate(span_calls_total{span_name="tx.process"}[5m])) # Trusted vs untrusted proposal rate -sum by (proposal_trusted) (rate(traces_span_metrics_calls_total{span_name="peer.proposal.receive"}[5m])) +sum by (proposal_trusted) (rate(span_calls_total{span_name="peer.proposal.receive"}[5m])) ``` ### StatsD Metrics ```promql # Validated ledger age (should be < 10s) -xrpld_LedgerMaster_Validated_Ledger_Age +ledgermaster_validated_ledger_age # Active peer count -xrpld_Peer_Finder_Active_Inbound_Peers + xrpld_Peer_Finder_Active_Outbound_Peers +peer_finder_active_inbound_peers + peer_finder_active_outbound_peers # RPC response time p95 -histogram_quantile(0.95, xrpld_rpc_time_bucket) +histogram_quantile(0.95, rpc_time_bucket) # Total network bytes in (rate) -rate(xrpld_total_Bytes_In[5m]) +rate(total_bytes_in[5m]) # Operating mode (should be "Full" after startup) -xrpld_State_Accounting_Full_duration +state_accounting_full_duration ``` --- @@ -816,60 +816,60 @@ async callbacks for new categories. > **Authoritative metric names live in [§ Phase 9: OTel SDK-Exported Metrics](#phase-9-otel-sdk-exported-metrics-metricsregistry) below.** > Most internal metrics are emitted as **labeled** gauges — one instrument carrying many logical -> values via a `metric` label (e.g. `xrpld_cache_metrics{metric="SLE_hit_rate"}`, -> `xrpld_txq_metrics{metric="txq_count"}`, `xrpld_load_factor_metrics{metric="load_factor"}`, -> `xrpld_nodestore_state{metric="node_reads_total"}`) — not the flat per-name form. Query the -> labeled names; the flat names (`xrpld_cache_SLE_hit_rate`, `xrpld_txq_count`, …) are **not** emitted. +> values via a `metric` label (e.g. `cache_metrics{metric="sle_hit_rate"}`, +> `txq_metrics{metric="txq_count"}`, `load_factor_metrics{metric="load_factor"}`, +> `nodestore_state{metric="node_reads_total"}`) — not the flat per-name form. Query the +> labeled names; the flat names (`cache_sle_hit_rate`, `txq_count`, …) are **not** emitted. #### Server Info (via OTel MetricsRegistry) -| Prometheus Metric | Type | Labels | Description | -| --------------------------------------------------------- | ----- | -------- | -------------------------------------------- | -| `xrpld_server_info{metric="server_state"}` | Gauge | `metric` | Operating mode (0=DISCONNECTED .. 4=FULL) | -| `xrpld_server_info{metric="uptime"}` | Gauge | `metric` | Seconds since server start | -| `xrpld_server_info{metric="peers"}` | Gauge | `metric` | Total connected peers | -| `xrpld_server_info{metric="validated_ledger_seq"}` | Gauge | `metric` | Validated ledger sequence number | -| `xrpld_server_info{metric="ledger_current_index"}` | Gauge | `metric` | Current open ledger sequence | -| `xrpld_server_info{metric="peer_disconnects_resources"}` | Gauge | `metric` | Cumulative resource-related peer disconnects | -| `xrpld_server_info{metric="last_close_proposers"}` | Gauge | `metric` | Proposers in last closed round | -| `xrpld_server_info{metric="last_close_converge_time_ms"}` | Gauge | `metric` | Last close convergence time (milliseconds) | +| Prometheus Metric | Type | Labels | Description | +| --------------------------------------------------- | ----- | -------- | -------------------------------------------- | +| `server_info{metric="server_state"}` | Gauge | `metric` | Operating mode (0=DISCONNECTED .. 4=FULL) | +| `server_info{metric="uptime"}` | Gauge | `metric` | Seconds since server start | +| `server_info{metric="peers"}` | Gauge | `metric` | Total connected peers | +| `server_info{metric="validated_ledger_seq"}` | Gauge | `metric` | Validated ledger sequence number | +| `server_info{metric="ledger_current_index"}` | Gauge | `metric` | Current open ledger sequence | +| `server_info{metric="peer_disconnects_resources"}` | Gauge | `metric` | Cumulative resource-related peer disconnects | +| `server_info{metric="last_close_proposers"}` | Gauge | `metric` | Proposers in last closed round | +| `server_info{metric="last_close_converge_time_ms"}` | Gauge | `metric` | Last close convergence time (milliseconds) | #### Build Info (via OTel MetricsRegistry) -| Prometheus Metric | Type | Labels | Description | -| ----------------------------------- | ----- | --------- | --------------------------------- | -| `xrpld_build_info{version=""}` | Gauge | `version` | Info-style metric, always value 1 | +| Prometheus Metric | Type | Labels | Description | +| ----------------------------- | ----- | --------- | --------------------------------- | +| `build_info{version=""}` | Gauge | `version` | Info-style metric, always value 1 | #### Complete Ledger Ranges (via OTel MetricsRegistry) -| Prometheus Metric | Type | Labels | Description | -| --------------------------------------------------- | ----- | --------------- | --------------------------- | -| `xrpld_complete_ledgers{bound="start",index=""}` | Gauge | `bound`,`index` | Start of contiguous range N | -| `xrpld_complete_ledgers{bound="end",index=""}` | Gauge | `bound`,`index` | End of contiguous range N | +| Prometheus Metric | Type | Labels | Description | +| --------------------------------------------- | ----- | --------------- | --------------------------- | +| `complete_ledgers{bound="start",index=""}` | Gauge | `bound`,`index` | Start of contiguous range N | +| `complete_ledgers{bound="end",index=""}` | Gauge | `bound`,`index` | End of contiguous range N | #### Database Metrics (via OTel MetricsRegistry) -| Prometheus Metric | Type | Labels | Description | -| ------------------------------------------------- | ----- | -------- | --------------------------------- | -| `xrpld_db_metrics{metric="db_kb_total"}` | Gauge | `metric` | Total database size (KB) | -| `xrpld_db_metrics{metric="db_kb_ledger"}` | Gauge | `metric` | Ledger database size (KB) | -| `xrpld_db_metrics{metric="db_kb_transaction"}` | Gauge | `metric` | Transaction database size (KB) | -| `xrpld_db_metrics{metric="historical_perminute"}` | Gauge | `metric` | Historical ledger fetches per min | +| Prometheus Metric | Type | Labels | Description | +| ------------------------------------------- | ----- | -------- | --------------------------------- | +| `db_metrics{metric="db_kb_total"}` | Gauge | `metric` | Total database size (KB) | +| `db_metrics{metric="db_kb_ledger"}` | Gauge | `metric` | Ledger database size (KB) | +| `db_metrics{metric="db_kb_transaction"}` | Gauge | `metric` | Transaction database size (KB) | +| `db_metrics{metric="historical_perminute"}` | Gauge | `metric` | Historical ledger fetches per min | -#### Extended Cache Metrics (additions to existing xrpld_cache_metrics) +#### Extended Cache Metrics (additions to existing cache_metrics) -| Prometheus Metric | Type | Labels | Description | -| --------------------------------------- | ----- | -------- | ------------------------- | -| `xrpld_cache_metrics{metric="AL_size"}` | Gauge | `metric` | AcceptedLedger cache size | +| Prometheus Metric | Type | Labels | Description | +| --------------------------------- | ----- | -------- | ------------------------- | +| `cache_metrics{metric="al_size"}` | Gauge | `metric` | AcceptedLedger cache size | -#### Extended NodeStore Metrics (additions to existing xrpld_nodestore_state) +#### Extended NodeStore Metrics (additions to existing nodestore_state) -| Prometheus Metric | Type | Labels | Description | -| -------------------------------------------------------- | ----- | -------- | ----------------------------------- | -| `xrpld_nodestore_state{metric="node_reads_duration_us"}` | Gauge | `metric` | Cumulative read time (microseconds) | -| `xrpld_nodestore_state{metric="read_request_bundle"}` | Gauge | `metric` | Read request bundle count | -| `xrpld_nodestore_state{metric="read_threads_running"}` | Gauge | `metric` | Active read threads | -| `xrpld_nodestore_state{metric="read_threads_total"}` | Gauge | `metric` | Total read threads configured | +| Prometheus Metric | Type | Labels | Description | +| -------------------------------------------------- | ----- | -------- | ----------------------------------- | +| `nodestore_state{metric="node_reads_duration_us"}` | Gauge | `metric` | Cumulative read time (microseconds) | +| `nodestore_state{metric="read_request_bundle"}` | Gauge | `metric` | Read request bundle count | +| `nodestore_state{metric="read_threads_running"}` | Gauge | `metric` | Active read threads | +| `nodestore_state{metric="read_threads_total"}` | Gauge | `metric` | Total read threads configured | ### New Grafana Dashboards (Phase 9) @@ -906,20 +906,20 @@ docker/telemetry/workload/benchmark.sh --xrpld .build/xrpld --duration 300 > **Counting note — families vs series.** A _metric family_ is one distinct Prometheus `__name__` > (histogram `_bucket`/`_count`/`_sum` collapsed to one). A _series_ is a family × its label > combinations. The legacy overlay-traffic block is the bulk of the count: ~56 message categories × -> 4 (`_Bytes_In/_Out`, `_Messages_In/_Out`) ≈ 224 families on its own. The labeled gauges -> (`xrpld_cache_metrics{metric}`, …) are few families but many series. Validate against the figures +> 4 (`_bytes_in/_out`, `_messages_in/_out`) ≈ 224 families on its own. The labeled gauges +> (`cache_metrics{metric}`, …) are few families but many series. Validate against the figures > below as **families currently emitting** (idle nodes under-report — workload-gated metrics such as > per-RPC/error counters appear only once exercised, which is Phase 10's purpose). -| Category | Expected Count | Validation Method | Config File | -| ------------------------- | ------------------------- | -------------------------------- | ----------------------- | -| Trace spans | ~37 (required + optional) | Tempo API query | `expected_spans.json` | -| Span attributes | per-span assertion | Per-span attribute assertion | `expected_spans.json` | -| Legacy `xrpld_*` families | ~270 (≈224 traffic) | Prometheus `__name__` query | `expected_metrics.json` | -| Native MetricsRegistry | 35 instruments | Prometheus query | `expected_metrics.json` | -| SpanMetrics RED | 4 per span | Prometheus query | `expected_metrics.json` | -| Grafana dashboards | 15 | Dashboard API "no data" check | `expected_metrics.json` | -| Log-trace links | Present | Loki query + Tempo reverse check | — | +| Category | Expected Count | Validation Method | Config File | +| ------------------------------ | ------------------------- | -------------------------------- | ----------------------- | +| Trace spans | ~37 (required + optional) | Tempo API query | `expected_spans.json` | +| Span attributes | per-span assertion | Per-span attribute assertion | `expected_spans.json` | +| Legacy beast::insight families | ~270 (≈224 traffic) | Prometheus `__name__` query | `expected_metrics.json` | +| Native MetricsRegistry | 35 instruments | Prometheus query | `expected_metrics.json` | +| SpanMetrics RED | 4 per span | Prometheus query | `expected_metrics.json` | +| Grafana dashboards | 15 | Dashboard API "no data" check | `expected_metrics.json` | +| Log-trace links | Present | Loki query + Tempo reverse check | — | ### Performance Overhead Targets @@ -1009,102 +1009,102 @@ via OTLP/HTTP to the OTel Collector and scraped by Prometheus. #### NodeStore I/O (Observable Gauge — `nodestore_state`) -| Prometheus Metric | Type | Labels | Description | -| ---------------------------------------------------- | ----- | -------- | ------------------------------------ | -| `xrpld_nodestore_state{metric="node_reads_total"}` | Gauge | `metric` | Cumulative NodeStore read operations | -| `xrpld_nodestore_state{metric="node_reads_hit"}` | Gauge | `metric` | Reads served from cache | -| `xrpld_nodestore_state{metric="node_writes"}` | Gauge | `metric` | Cumulative write operations | -| `xrpld_nodestore_state{metric="node_written_bytes"}` | Gauge | `metric` | Cumulative bytes written | -| `xrpld_nodestore_state{metric="node_read_bytes"}` | Gauge | `metric` | Cumulative bytes read | -| `xrpld_nodestore_state{metric="write_load"}` | Gauge | `metric` | Current write load score | -| `xrpld_nodestore_state{metric="read_queue"}` | Gauge | `metric` | Items in read prefetch queue | +| Prometheus Metric | Type | Labels | Description | +| ---------------------------------------------- | ----- | -------- | ------------------------------------ | +| `nodestore_state{metric="node_reads_total"}` | Gauge | `metric` | Cumulative NodeStore read operations | +| `nodestore_state{metric="node_reads_hit"}` | Gauge | `metric` | Reads served from cache | +| `nodestore_state{metric="node_writes"}` | Gauge | `metric` | Cumulative write operations | +| `nodestore_state{metric="node_written_bytes"}` | Gauge | `metric` | Cumulative bytes written | +| `nodestore_state{metric="node_read_bytes"}` | Gauge | `metric` | Cumulative bytes read | +| `nodestore_state{metric="write_load"}` | Gauge | `metric` | Current write load score | +| `nodestore_state{metric="read_queue"}` | Gauge | `metric` | Items in read prefetch queue | #### Cache Hit Rates & Sizes (Observable Gauge — `cache_metrics`) -| Prometheus Metric | Type | Labels | Description | -| --------------------------------------------------- | ----- | -------- | ----------------------------- | -| `xrpld_cache_metrics{metric="SLE_hit_rate"}` | Gauge | `metric` | SLE cache hit rate (0.0-1.0) | -| `xrpld_cache_metrics{metric="ledger_hit_rate"}` | Gauge | `metric` | Ledger cache hit rate | -| `xrpld_cache_metrics{metric="AL_hit_rate"}` | Gauge | `metric` | AcceptedLedger cache hit rate | -| `xrpld_cache_metrics{metric="treenode_cache_size"}` | Gauge | `metric` | SHAMap TreeNode cache entries | -| `xrpld_cache_metrics{metric="treenode_track_size"}` | Gauge | `metric` | Tracked tree nodes | -| `xrpld_cache_metrics{metric="fullbelow_size"}` | Gauge | `metric` | FullBelow cache entries | +| Prometheus Metric | Type | Labels | Description | +| --------------------------------------------- | ----- | -------- | ----------------------------- | +| `cache_metrics{metric="sle_hit_rate"}` | Gauge | `metric` | SLE cache hit rate (0.0-1.0) | +| `cache_metrics{metric="ledger_hit_rate"}` | Gauge | `metric` | Ledger cache hit rate | +| `cache_metrics{metric="al_hit_rate"}` | Gauge | `metric` | AcceptedLedger cache hit rate | +| `cache_metrics{metric="treenode_cache_size"}` | Gauge | `metric` | SHAMap TreeNode cache entries | +| `cache_metrics{metric="treenode_track_size"}` | Gauge | `metric` | Tracked tree nodes | +| `cache_metrics{metric="fullbelow_size"}` | Gauge | `metric` | FullBelow cache entries | #### Transaction Queue (Observable Gauge — `txq_metrics`) -| Prometheus Metric | Type | Labels | Description | -| ---------------------------------------------------------- | ----- | -------- | -------------------------------- | -| `xrpld_txq_metrics{metric="txq_count"}` | Gauge | `metric` | Transactions currently in queue | -| `xrpld_txq_metrics{metric="txq_max_size"}` | Gauge | `metric` | Maximum queue capacity | -| `xrpld_txq_metrics{metric="txq_in_ledger"}` | Gauge | `metric` | Transactions in open ledger | -| `xrpld_txq_metrics{metric="txq_per_ledger"}` | Gauge | `metric` | Expected transactions per ledger | -| `xrpld_txq_metrics{metric="txq_reference_fee_level"}` | Gauge | `metric` | Reference fee level | -| `xrpld_txq_metrics{metric="txq_min_processing_fee_level"}` | Gauge | `metric` | Minimum fee to get processed | -| `xrpld_txq_metrics{metric="txq_med_fee_level"}` | Gauge | `metric` | Median fee level in queue | -| `xrpld_txq_metrics{metric="txq_open_ledger_fee_level"}` | Gauge | `metric` | Open ledger fee escalation level | +| Prometheus Metric | Type | Labels | Description | +| ---------------------------------------------------- | ----- | -------- | -------------------------------- | +| `txq_metrics{metric="txq_count"}` | Gauge | `metric` | Transactions currently in queue | +| `txq_metrics{metric="txq_max_size"}` | Gauge | `metric` | Maximum queue capacity | +| `txq_metrics{metric="txq_in_ledger"}` | Gauge | `metric` | Transactions in open ledger | +| `txq_metrics{metric="txq_per_ledger"}` | Gauge | `metric` | Expected transactions per ledger | +| `txq_metrics{metric="txq_reference_fee_level"}` | Gauge | `metric` | Reference fee level | +| `txq_metrics{metric="txq_min_processing_fee_level"}` | Gauge | `metric` | Minimum fee to get processed | +| `txq_metrics{metric="txq_med_fee_level"}` | Gauge | `metric` | Median fee level in queue | +| `txq_metrics{metric="txq_open_ledger_fee_level"}` | Gauge | `metric` | Open ledger fee escalation level | #### Per-RPC Method Metrics (Synchronous Counters/Histogram) -| Prometheus Metric | Type | Labels | Description | -| --------------------------------- | --------- | ----------------- | -------------------------------- | -| `xrpld_rpc_method_started_total` | Counter | `method=""` | RPC calls started | -| `xrpld_rpc_method_finished_total` | Counter | `method=""` | RPC calls completed successfully | -| `xrpld_rpc_method_errored_total` | Counter | `method=""` | RPC calls that errored | -| `xrpld_rpc_method_duration_us` | Histogram | `method=""` | Execution time distribution (us) | +| Prometheus Metric | Type | Labels | Description | +| --------------------------- | --------- | ----------------- | -------------------------------- | +| `rpc_method_started_total` | Counter | `method=""` | RPC calls started | +| `rpc_method_finished_total` | Counter | `method=""` | RPC calls completed successfully | +| `rpc_method_errored_total` | Counter | `method=""` | RPC calls that errored | +| `rpc_method_us` | Histogram | `method=""` | Execution time distribution (us) | #### Per-Job-Type Metrics (Synchronous Counters/Histogram) -| Prometheus Metric | Type | Labels | Description | -| ------------------------------- | --------- | ------------------- | --------------------------------- | -| `xrpld_job_queued_total` | Counter | `job_type=""` | Jobs enqueued | -| `xrpld_job_started_total` | Counter | `job_type=""` | Jobs started | -| `xrpld_job_finished_total` | Counter | `job_type=""` | Jobs completed | -| `xrpld_job_queued_duration_us` | Histogram | `job_type=""` | Queue wait time distribution (us) | -| `xrpld_job_running_duration_us` | Histogram | `job_type=""` | Execution time distribution (us) | +| Prometheus Metric | Type | Labels | Description | +| -------------------- | --------- | ------------------- | --------------------------------- | +| `job_queued_total` | Counter | `job_type=""` | Jobs enqueued | +| `job_started_total` | Counter | `job_type=""` | Jobs started | +| `job_finished_total` | Counter | `job_type=""` | Jobs completed | +| `job_queued_us` | Histogram | `job_type=""` | Queue wait time distribution (us) | +| `job_running_us` | Histogram | `job_type=""` | Execution time distribution (us) | #### Counted Object Instances (Observable Gauge — `object_count`) -| Prometheus Metric | Type | Labels | Description | -| -------------------------------------------- | ----- | --------------- | ------------------------------ | -| `xrpld_object_count{type="Transaction"}` | Gauge | `type=""` | Live Transaction objects | -| `xrpld_object_count{type="Ledger"}` | Gauge | `type=""` | Live Ledger objects | -| `xrpld_object_count{type="NodeObject"}` | Gauge | `type=""` | Live NodeObject instances | -| `xrpld_object_count{type="STTx"}` | Gauge | `type=""` | Serialized transaction objects | -| `xrpld_object_count{type="STLedgerEntry"}` | Gauge | `type=""` | Serialized ledger entries | -| `xrpld_object_count{type="InboundLedger"}` | Gauge | `type=""` | Ledgers being fetched | -| `xrpld_object_count{type="Pathfinder"}` | Gauge | `type=""` | Active pathfinding operations | -| `xrpld_object_count{type="PathRequest"}` | Gauge | `type=""` | Active path requests | -| `xrpld_object_count{type="HashRouterEntry"}` | Gauge | `type=""` | Hash router entries | +| Prometheus Metric | Type | Labels | Description | +| -------------------------------------- | ----- | --------------- | ------------------------------ | +| `object_count{type="transaction"}` | Gauge | `type=""` | Live Transaction objects | +| `object_count{type="ledger"}` | Gauge | `type=""` | Live Ledger objects | +| `object_count{type="nodeobject"}` | Gauge | `type=""` | Live NodeObject instances | +| `object_count{type="sttx"}` | Gauge | `type=""` | Serialized transaction objects | +| `object_count{type="stledgerentry"}` | Gauge | `type=""` | Serialized ledger entries | +| `object_count{type="inboundledger"}` | Gauge | `type=""` | Ledgers being fetched | +| `object_count{type="pathfinder"}` | Gauge | `type=""` | Active pathfinding operations | +| `object_count{type="pathrequest"}` | Gauge | `type=""` | Active path requests | +| `object_count{type="hashrouterentry"}` | Gauge | `type=""` | Hash router entries | #### Load Factor Breakdown (Observable Gauge — `load_factor_metrics`) -| Prometheus Metric | Type | Labels | Description | -| ---------------------------------------------------------------- | ----- | -------- | --------------------------------------- | -| `xrpld_load_factor_metrics{metric="load_factor"}` | Gauge | `metric` | Combined transaction cost multiplier | -| `xrpld_load_factor_metrics{metric="load_factor_server"}` | Gauge | `metric` | Server + cluster + network contribution | -| `xrpld_load_factor_metrics{metric="load_factor_local"}` | Gauge | `metric` | Local server load only | -| `xrpld_load_factor_metrics{metric="load_factor_net"}` | Gauge | `metric` | Network-wide load estimate | -| `xrpld_load_factor_metrics{metric="load_factor_cluster"}` | Gauge | `metric` | Cluster peer load | -| `xrpld_load_factor_metrics{metric="load_factor_fee_escalation"}` | Gauge | `metric` | Open ledger fee escalation | -| `xrpld_load_factor_metrics{metric="load_factor_fee_queue"}` | Gauge | `metric` | Queue entry fee level | +| Prometheus Metric | Type | Labels | Description | +| ---------------------------------------------------------- | ----- | -------- | --------------------------------------- | +| `load_factor_metrics{metric="load_factor"}` | Gauge | `metric` | Combined transaction cost multiplier | +| `load_factor_metrics{metric="load_factor_server"}` | Gauge | `metric` | Server + cluster + network contribution | +| `load_factor_metrics{metric="load_factor_local"}` | Gauge | `metric` | Local server load only | +| `load_factor_metrics{metric="load_factor_net"}` | Gauge | `metric` | Network-wide load estimate | +| `load_factor_metrics{metric="load_factor_cluster"}` | Gauge | `metric` | Cluster peer load | +| `load_factor_metrics{metric="load_factor_fee_escalation"}` | Gauge | `metric` | Open ledger fee escalation | +| `load_factor_metrics{metric="load_factor_fee_queue"}` | Gauge | `metric` | Queue entry fee level | #### Prometheus Query Examples (Phase 9) ```promql # NodeStore cache hit ratio -xrpld_nodestore_state{metric="node_reads_hit"} / xrpld_nodestore_state{metric="node_reads_total"} +nodestore_state{metric="node_reads_hit"} / nodestore_state{metric="node_reads_total"} # RPC error rate for server_info -rate(xrpld_rpc_method_errored_total{method="server_info"}[5m]) +rate(rpc_method_errored_total{method="server_info"}[5m]) # Job queue wait time p95 -histogram_quantile(0.95, sum by (le) (rate(xrpld_job_queued_duration_us_bucket[5m]))) +histogram_quantile(0.95, sum by (le) (rate(job_queued_us_bucket[5m]))) # TxQ utilization percentage -xrpld_txq_metrics{metric="txq_count"} / xrpld_txq_metrics{metric="txq_max_size"} +txq_metrics{metric="txq_count"} / txq_metrics{metric="txq_max_size"} # High load factor alert candidate -xrpld_load_factor_metrics{metric="load_factor"} > 5 +load_factor_metrics{metric="load_factor"} > 5 ``` ### Phase 7+: External Dashboard Parity Metrics @@ -1117,83 +1117,83 @@ These metrics fill gaps identified by comparing xrpld's internal observability w #### Validation Agreement (Observable Gauge — `validation_agreement`) -| Prometheus Metric | Type | Labels | Description | -| -------------------------------------------------------- | ------ | -------- | --------------------------------------- | -| `xrpld_validation_agreement{metric="agreement_pct_1h"}` | Double | `metric` | Rolling 1h agreement percentage (0-100) | -| `xrpld_validation_agreement{metric="agreement_pct_24h"}` | Double | `metric` | Rolling 24h agreement percentage | -| `xrpld_validation_agreement{metric="agreements_1h"}` | Int64 | `metric` | Agreed validations in 1h window | -| `xrpld_validation_agreement{metric="missed_1h"}` | Int64 | `metric` | Missed validations in 1h window | -| `xrpld_validation_agreement{metric="agreements_24h"}` | Int64 | `metric` | Agreed validations in 24h window | -| `xrpld_validation_agreement{metric="missed_24h"}` | Int64 | `metric` | Missed validations in 24h window | +| Prometheus Metric | Type | Labels | Description | +| -------------------------------------------------- | ------ | -------- | --------------------------------------- | +| `validation_agreement{metric="agreement_pct_1h"}` | Double | `metric` | Rolling 1h agreement percentage (0-100) | +| `validation_agreement{metric="agreement_pct_24h"}` | Double | `metric` | Rolling 24h agreement percentage | +| `validation_agreement{metric="agreements_1h"}` | Int64 | `metric` | Agreed validations in 1h window | +| `validation_agreement{metric="missed_1h"}` | Int64 | `metric` | Missed validations in 1h window | +| `validation_agreement{metric="agreements_24h"}` | Int64 | `metric` | Agreed validations in 24h window | +| `validation_agreement{metric="missed_24h"}` | Int64 | `metric` | Missed validations in 24h window | Data source: `ValidationTracker` class with 8s grace period and 5m late repair window. #### Validator Health (Observable Gauge — `validator_health`) -| Prometheus Metric | Type | Labels | Description | -| ---------------------------------------------------- | ------ | -------- | ------------------------------ | -| `xrpld_validator_health{metric="amendment_blocked"}` | Int64 | `metric` | 1 if amendment-blocked, else 0 | -| `xrpld_validator_health{metric="unl_blocked"}` | Int64 | `metric` | 1 if UNL-blocked, else 0 | -| `xrpld_validator_health{metric="unl_expiry_days"}` | Double | `metric` | Days until UNL list expires | -| `xrpld_validator_health{metric="validation_quorum"}` | Int64 | `metric` | Validation quorum threshold | +| Prometheus Metric | Type | Labels | Description | +| ---------------------------------------------- | ------ | -------- | ------------------------------ | +| `validator_health{metric="amendment_blocked"}` | Int64 | `metric` | 1 if amendment-blocked, else 0 | +| `validator_health{metric="unl_blocked"}` | Int64 | `metric` | 1 if UNL-blocked, else 0 | +| `validator_health{metric="unl_expiry_days"}` | Double | `metric` | Days until UNL list expires | +| `validator_health{metric="validation_quorum"}` | Int64 | `metric` | Validation quorum threshold | #### Peer Quality (Observable Gauge — `peer_quality`) -| Prometheus Metric | Type | Labels | Description | -| ------------------------------------------------------- | ------ | -------- | ------------------------------------ | -| `xrpld_peer_quality{metric="peer_latency_p90_ms"}` | Double | `metric` | P90 peer latency in milliseconds | -| `xrpld_peer_quality{metric="peers_insane_count"}` | Int64 | `metric` | Peers with diverged tracking status | -| `xrpld_peer_quality{metric="peers_higher_version_pct"}` | Double | `metric` | % of peers on newer xrpld version | -| `xrpld_peer_quality{metric="upgrade_recommended"}` | Int64 | `metric` | 1 if >60% of peers are newer version | +| Prometheus Metric | Type | Labels | Description | +| ------------------------------------------------- | ------ | -------- | ------------------------------------ | +| `peer_quality{metric="peer_latency_p90_ms"}` | Double | `metric` | P90 peer latency in milliseconds | +| `peer_quality{metric="peers_insane_count"}` | Int64 | `metric` | Peers with diverged tracking status | +| `peer_quality{metric="peers_higher_version_pct"}` | Double | `metric` | % of peers on newer xrpld version | +| `peer_quality{metric="upgrade_recommended"}` | Int64 | `metric` | 1 if >60% of peers are newer version | #### Ledger Economy (Observable Gauge — `ledger_economy`) -| Prometheus Metric | Type | Labels | Description | -| --------------------------------------------------- | ------ | -------- | ---------------------------------- | -| `xrpld_ledger_economy{metric="base_fee_xrp"}` | Double | `metric` | Base transaction fee in drops | -| `xrpld_ledger_economy{metric="reserve_base_xrp"}` | Double | `metric` | Account reserve in drops | -| `xrpld_ledger_economy{metric="reserve_inc_xrp"}` | Double | `metric` | Owner reserve increment in drops | -| `xrpld_ledger_economy{metric="ledger_age_seconds"}` | Double | `metric` | Seconds since last validated close | -| `xrpld_ledger_economy{metric="transaction_rate"}` | Double | `metric` | Smoothed transaction rate (tx/s) | +| Prometheus Metric | Type | Labels | Description | +| --------------------------------------------- | ------ | -------- | ---------------------------------- | +| `ledger_economy{metric="base_fee_xrp"}` | Double | `metric` | Base transaction fee in drops | +| `ledger_economy{metric="reserve_base_xrp"}` | Double | `metric` | Account reserve in drops | +| `ledger_economy{metric="reserve_inc_xrp"}` | Double | `metric` | Owner reserve increment in drops | +| `ledger_economy{metric="ledger_age_seconds"}` | Double | `metric` | Seconds since last validated close | +| `ledger_economy{metric="transaction_rate"}` | Double | `metric` | Smoothed transaction rate (tx/s) | #### State Tracking (Observable Gauge — `state_tracking`) -| Prometheus Metric | Type | Labels | Description | -| -------------------------------------------------------------- | ------ | -------- | -------------------------------------- | -| `xrpld_state_tracking{metric="state_value"}` | Int64 | `metric` | Numeric state 0-6 (see encoding below) | -| `xrpld_state_tracking{metric="time_in_current_state_seconds"}` | Double | `metric` | Duration in current state | +| Prometheus Metric | Type | Labels | Description | +| -------------------------------------------------------- | ------ | -------- | -------------------------------------- | +| `state_tracking{metric="state_value"}` | Int64 | `metric` | Numeric state 0-6 (see encoding below) | +| `state_tracking{metric="time_in_current_state_seconds"}` | Double | `metric` | Duration in current state | State value encoding: 0=disconnected, 1=connected, 2=syncing, 3=tracking, 4=full, 5=validating (FULL + validating), 6=proposing (FULL + proposing). #### Storage Detail (Observable Gauge — `storage_detail`) -| Prometheus Metric | Type | Labels | Description | -| ------------------------------------------- | ----- | -------- | ---------------------- | -| `xrpld_storage_detail{metric="nudb_bytes"}` | Int64 | `metric` | NuDB backend file size | +| Prometheus Metric | Type | Labels | Description | +| ------------------------------------- | ----- | -------- | ---------------------- | +| `storage_detail{metric="nudb_bytes"}` | Int64 | `metric` | NuDB backend file size | #### Synchronous Counters (Phase 7+) -| Prometheus Metric | Type | Description | Increment Site | -| --------------------------------- | ------- | ------------------------------- | ---------------- | -| `xrpld_ledgers_closed_total` | Counter | Ledgers closed by consensus | RCLConsensus.cpp | -| `xrpld_validations_sent_total` | Counter | Validations sent | RCLConsensus.cpp | -| `xrpld_validations_checked_total` | Counter | Network validations observed | LedgerMaster.cpp | -| `xrpld_state_changes_total` | Counter | Operating mode transitions | NetworkOPs.cpp | -| `xrpld_jq_trans_overflow_total` | Counter | Job queue transaction overflows | JobQueue.cpp | +| Prometheus Metric | Type | Description | Increment Site | +| --------------------------- | ------- | ------------------------------- | ---------------- | +| `ledgers_closed_total` | Counter | Ledgers closed by consensus | RCLConsensus.cpp | +| `validations_sent_total` | Counter | Validations sent | RCLConsensus.cpp | +| `validations_checked_total` | Counter | Network validations observed | LedgerMaster.cpp | +| `state_changes_total` | Counter | Operating mode transitions | NetworkOPs.cpp | +| `jq_trans_overflow_total` | Counter | Job queue transaction overflows | JobQueue.cpp | Lifetime validation agreement/miss tallies are exported as monotonic **ObservableCounters** (not synchronous counters) observed from `ValidationTracker`'s gross lifetime totals: -| Prometheus Metric | Type | Description | Source | -| ----------------------------------- | ----------------- | ------------------------------------------ | --------------------- | -| `xrpld_validation_agreements_total` | ObservableCounter | Lifetime validations that initially agreed | ValidationTracker.cpp | -| `xrpld_validation_missed_total` | ObservableCounter | Lifetime validations that initially missed | ValidationTracker.cpp | +| Prometheus Metric | Type | Description | Source | +| ----------------------------- | ----------------- | ------------------------------------------ | --------------------- | +| `validation_agreements_total` | ObservableCounter | Lifetime validations that initially agreed | ValidationTracker.cpp | +| `validation_missed_total` | ObservableCounter | Lifetime validations that initially missed | ValidationTracker.cpp | > **Counting semantics (initial-classification only):** each reconciled ledger increments exactly > one of these two counters, at first classification. A later late-repair (miss → agreement) does > **not** move either counter — keeping both strictly monotonic (a Prometheus `_total` must never > decrease) and additive (`agreements_total + missed_total` = ledgers reconciled). The -> repair-aware, windowed view remains on `xrpld_validation_agreement{metric="…"}`. +> repair-aware, windowed view remains on `validation_agreement{metric="…"}`. #### Span Attribute Enrichments (Phases 2-4) @@ -1258,8 +1258,8 @@ Lifetime validation agreement/miss tallies are exported as monotonic **Observabl | Issue | Impact | Status | | ------------------------------------------------------------------ | ------------------------------------------------ | -------------------------------------------------------------------- | | `warn` and `drop` metrics use non-standard StatsD `\|m` meter type | Metrics silently dropped by OTel StatsD receiver | Phase 6 Task 6.1 — needs `\|m` → `\|c` change in StatsDCollector.cpp | -| `xrpld_jobq_job_count` may not emit in standalone mode | Missing from Prometheus in some test configs | Requires active job queue activity | -| `xrpld_rpc_requests` depends on `[insight]` config | Zero series if StatsD not configured | Requires `[insight] server=statsd` in xrpld.cfg | +| `jobq_job_count` may not emit in standalone mode | Missing from Prometheus in some test configs | Requires active job queue activity | +| `rpc_requests` depends on `[insight]` config | Zero series if StatsD not configured | Requires `[insight] server=statsd` in xrpld.cfg | | Peer tracing enabled by default | `peer.*` spans emit unless `trace_peer=0` | High volume — set `trace_peer=0` to opt out on busy mainnet nodes | --- diff --git a/docker/telemetry/TESTING.md b/docker/telemetry/TESTING.md index ceccc10b5c..7cf2b19fa0 100644 --- a/docker/telemetry/TESTING.md +++ b/docker/telemetry/TESTING.md @@ -434,21 +434,21 @@ Base URL: `http://localhost:9090` PROM="http://localhost:9090" # Span call counts (from spanmetrics connector) -curl -s "$PROM/api/v1/query?query=traces_span_metrics_calls_total" | +curl -s "$PROM/api/v1/query?query=span_calls_total" | jq '.data.result[] | {span: .metric.span_name, count: .value[1]}' # Latency histogram -curl -s "$PROM/api/v1/query?query=traces_span_metrics_duration_milliseconds_count" | +curl -s "$PROM/api/v1/query?query=span_duration_milliseconds_count" | jq '.data.result[] | {span: .metric.span_name, count: .value[1]}' # RPC calls by command -curl -s "$PROM/api/v1/query?query=traces_span_metrics_calls_total{span_name=~\"rpc.command.*\"}" | +curl -s "$PROM/api/v1/query?query=span_calls_total{span_name=~\"rpc.command.*\"}" | jq '.data.result[] | {command: .metric["command"], count: .value[1]}' # Deployment-tier labels present on metrics (set by the collector's # resource/tier processor and promoted via resource_to_telemetry_conversion). # Expect deployment_environment and xrpl_network_type on each series. -curl -s "$PROM/api/v1/query?query=traces_span_metrics_calls_total" | +curl -s "$PROM/api/v1/query?query=span_calls_total" | jq '.data.result[0].metric | {deployment_environment, xrpl_network_type, service_name}' ``` @@ -515,7 +515,7 @@ After exercising RPC/transaction workflows (Tests 1 or 2), open your Grafana Cloud instance and confirm: - **Traces**: Explore → hosted Tempo datasource → search `{resource.service.name="xrpld"}` -- **Metrics**: Explore → hosted Prometheus/Mimir → query `traces_span_metrics_calls_total` +- **Metrics**: Explore → hosted Prometheus/Mimir → query `span_calls_total` - **Logs**: Explore → hosted Loki → query `{job="xrpld"}` (requires `warning`+ file logging) If nothing appears, check the collector logs for auth/export errors: diff --git a/docker/telemetry/grafana/provisioning/alerting/rules.yaml b/docker/telemetry/grafana/provisioning/alerting/rules.yaml index 64ac7fd9df..cf396da0c5 100644 --- a/docker/telemetry/grafana/provisioning/alerting/rules.yaml +++ b/docker/telemetry/grafana/provisioning/alerting/rules.yaml @@ -54,7 +54,7 @@ groups: datasourceUid: prometheus model: refId: A - expr: sum by (service_instance_id) (rate(xrpld_ledger_history_mismatch_total[5m])) + expr: sum by (service_instance_id) (rate(ledger_history_mismatch_total[5m])) instant: true range: false intervalMs: 1000 @@ -115,7 +115,7 @@ groups: datasourceUid: prometheus model: refId: A - expr: sum by (service_instance_id) (rate(xrpld_ledgers_closed_total[5m])) + expr: sum by (service_instance_id) (rate(ledgers_closed_total[5m])) instant: true range: false intervalMs: 1000 @@ -184,7 +184,7 @@ groups: datasourceUid: prometheus model: refId: A - expr: sum by (service_instance_id) (rate(xrpld_validation_missed_total[5m])) + expr: sum by (service_instance_id) (rate(validation_missed_total[5m])) instant: true range: false intervalMs: 1000 @@ -245,7 +245,7 @@ groups: datasourceUid: prometheus model: refId: A - expr: sum by (service_instance_id) (rate(xrpld_validations_checked_total[5m])) + expr: sum by (service_instance_id) (rate(validations_checked_total[5m])) instant: true range: false intervalMs: 1000 @@ -313,7 +313,7 @@ groups: datasourceUid: prometheus model: refId: A - expr: sum by (service_instance_id) (rate(xrpld_jq_trans_overflow_total[5m])) + expr: sum by (service_instance_id) (rate(jq_trans_overflow_total[5m])) instant: true range: false intervalMs: 1000 @@ -374,7 +374,7 @@ groups: datasourceUid: prometheus model: refId: A - expr: histogram_quantile(0.99, sum by (le, service_instance_id) (rate(xrpld_job_queued_duration_us_bucket[5m]))) + expr: histogram_quantile(0.99, sum by (le, service_instance_id) (rate(job_queued_us_bucket[5m]))) instant: true range: false intervalMs: 1000 diff --git a/docker/telemetry/integration-test.sh b/docker/telemetry/integration-test.sh index 2442cdff35..92fdc374bd 100755 --- a/docker/telemetry/integration-test.sh +++ b/docker/telemetry/integration-test.sh @@ -584,15 +584,15 @@ log "--- Phase 5: Spanmetrics ---" log "Waiting 20s for Prometheus scrape cycle..." sleep 20 -calls_count=$(curl -sf "$PROM/api/v1/query?query=traces_span_metrics_calls_total" | +calls_count=$(curl -sf "$PROM/api/v1/query?query=span_calls_total" | jq '.data.result | length' 2>/dev/null || echo 0) if [ "$calls_count" -gt 0 ]; then - ok "Prometheus: traces_span_metrics_calls_total ($calls_count series)" + ok "Prometheus: span_calls_total ($calls_count series)" else - fail "Prometheus: traces_span_metrics_calls_total (0 series)" + fail "Prometheus: span_calls_total (0 series)" fi -duration_count=$(curl -sf "$PROM/api/v1/query?query=traces_span_metrics_duration_milliseconds_count" | +duration_count=$(curl -sf "$PROM/api/v1/query?query=span_duration_milliseconds_count" | jq '.data.result | length' 2>/dev/null || echo 0) if [ "$duration_count" -gt 0 ]; then ok "Prometheus: duration histogram ($duration_count series)" @@ -680,46 +680,46 @@ check_otel_metric() { } # Task 9.1: NodeStore I/O -check_otel_metric 'xrpld_nodestore_state{metric="node_reads_total"}' -check_otel_metric 'xrpld_nodestore_state{metric="write_load"}' +check_otel_metric 'nodestore_state{metric="node_reads_total"}' +check_otel_metric 'nodestore_state{metric="write_load"}' # Task 9.2: Cache hit rates -check_otel_metric 'xrpld_cache_metrics{metric="SLE_hit_rate"}' -check_otel_metric 'xrpld_cache_metrics{metric="treenode_cache_size"}' +check_otel_metric 'cache_metrics{metric="SLE_hit_rate"}' +check_otel_metric 'cache_metrics{metric="treenode_cache_size"}' # Task 9.3: TxQ metrics -check_otel_metric 'xrpld_txq_metrics{metric="txq_count"}' -check_otel_metric 'xrpld_txq_metrics{metric="txq_reference_fee_level"}' +check_otel_metric 'txq_metrics{metric="txq_count"}' +check_otel_metric 'txq_metrics{metric="txq_reference_fee_level"}' # Task 9.4: Per-RPC metrics -check_otel_metric "xrpld_rpc_method_started_total" -check_otel_metric "xrpld_rpc_method_finished_total" +check_otel_metric "rpc_method_started_total" +check_otel_metric "rpc_method_finished_total" # Task 9.5: Per-job metrics -check_otel_metric "xrpld_job_queued_total" -check_otel_metric "xrpld_job_finished_total" +check_otel_metric "job_queued_total" +check_otel_metric "job_finished_total" # Task 9.6: Counted object instances -check_otel_metric "xrpld_object_count" +check_otel_metric "object_count" # Task 9.7: Load factor breakdown -check_otel_metric 'xrpld_load_factor_metrics{metric="load_factor"}' -check_otel_metric 'xrpld_load_factor_metrics{metric="load_factor_server"}' +check_otel_metric 'load_factor_metrics{metric="load_factor"}' +check_otel_metric 'load_factor_metrics{metric="load_factor_server"}' # Task 7.15 / Phase 9: ValidationTracker rolling-window agreement gauge. # MetricsRegistry::registerValidationAgreementGauge() publishes -# xrpld_validation_agreement with a `metric` label for each window +# validation_agreement with a `metric` label for each window # (1h / 24h / 7d) plus the matching agreement/miss counts. The 7-day # window matches the external xrpl-validator-dashboard parity target. -check_otel_metric 'xrpld_validation_agreement{metric="agreement_pct_1h"}' -check_otel_metric 'xrpld_validation_agreement{metric="agreement_pct_24h"}' -check_otel_metric 'xrpld_validation_agreement{metric="agreement_pct_7d"}' -check_otel_metric 'xrpld_validation_agreement{metric="agreements_1h"}' -check_otel_metric 'xrpld_validation_agreement{metric="missed_1h"}' -check_otel_metric 'xrpld_validation_agreement{metric="agreements_24h"}' -check_otel_metric 'xrpld_validation_agreement{metric="missed_24h"}' -check_otel_metric 'xrpld_validation_agreement{metric="agreements_7d"}' -check_otel_metric 'xrpld_validation_agreement{metric="missed_7d"}' +check_otel_metric 'validation_agreement{metric="agreement_pct_1h"}' +check_otel_metric 'validation_agreement{metric="agreement_pct_24h"}' +check_otel_metric 'validation_agreement{metric="agreement_pct_7d"}' +check_otel_metric 'validation_agreement{metric="agreements_1h"}' +check_otel_metric 'validation_agreement{metric="missed_1h"}' +check_otel_metric 'validation_agreement{metric="agreements_24h"}' +check_otel_metric 'validation_agreement{metric="missed_24h"}' +check_otel_metric 'validation_agreement{metric="agreements_7d"}' +check_otel_metric 'validation_agreement{metric="missed_7d"}' # --------------------------------------------------------------------------- # Step 11: Summary diff --git a/docker/telemetry/workload/expected_metrics.json b/docker/telemetry/workload/expected_metrics.json index 6365c11c77..b305dff39b 100644 --- a/docker/telemetry/workload/expected_metrics.json +++ b/docker/telemetry/workload/expected_metrics.json @@ -1,12 +1,12 @@ { - "description": "Expected metric inventory for xrpld telemetry validation. Metric names use the xrpld_ prefix (the [insight] prefix and OTel resource service name). Sourced from the live Grafana dashboards and MetricsRegistry.cpp.", + "description": "Expected metric inventory for xrpld telemetry validation. Metric names have no prefix (the xrpld_ prefix was removed). beast::insight metrics are lowercased by formatName. Sourced from the live Grafana dashboards and MetricsRegistry.cpp.", "spanmetrics": { "description": "SpanMetrics-derived RED metrics from the OTel Collector spanmetrics connector.", "metrics": [ - "traces_span_metrics_calls_total", - "traces_span_metrics_duration_milliseconds_bucket", - "traces_span_metrics_duration_milliseconds_count", - "traces_span_metrics_duration_milliseconds_sum" + "span_calls_total", + "span_duration_milliseconds_bucket", + "span_duration_milliseconds_count", + "span_duration_milliseconds_sum" ], "required_labels": [ "span_name", @@ -34,94 +34,94 @@ "statsd_gauges": { "description": "beast::insight gauges exported via OTLP/HTTP to the collector (server=otel).", "metrics": [ - "xrpld_LedgerMaster_Validated_Ledger_Age", - "xrpld_LedgerMaster_Published_Ledger_Age", - "xrpld_State_Accounting_Full_duration", - "xrpld_Peer_Finder_Active_Inbound_Peers", - "xrpld_Peer_Finder_Active_Outbound_Peers", - "xrpld_jobq_job_count" + "ledgermaster_validated_ledger_age", + "ledgermaster_published_ledger_age", + "state_accounting_full_duration", + "peer_finder_active_inbound_peers", + "peer_finder_active_outbound_peers", + "jobq_job_count" ] }, "statsd_counters": { "description": "beast::insight counters exported via OTLP/HTTP. The OTel Prometheus exporter appends _total to monotonic counters.", - "metrics": ["xrpld_rpc_requests_total", "xrpld_ledger_fetches_total"] + "metrics": ["rpc_requests_total", "ledger_fetches_total"] }, "overlay_traffic": { "description": "Overlay traffic metrics (subset — full list has 45+ categories).", "metrics": [ - "xrpld_total_Bytes_In", - "xrpld_total_Bytes_Out", - "xrpld_total_Messages_In", - "xrpld_total_Messages_Out" + "total_bytes_in", + "total_bytes_out", + "total_messages_in", + "total_messages_out" ] }, "phase9_nodestore": { "description": "Phase 9 NodeStore I/O observable gauge (MetricsRegistry via OTLP). Single metric with 'metric' label distinguishing sub-metrics.", - "metrics": ["xrpld_nodestore_state"] + "metrics": ["nodestore_state"] }, "phase9_cache": { "description": "Phase 9 cache hit rate observable gauge (MetricsRegistry via OTLP). Single metric with 'metric' label.", - "metrics": ["xrpld_cache_metrics"] + "metrics": ["cache_metrics"] }, "phase9_txq": { "description": "Phase 9 transaction queue observable gauge (MetricsRegistry via OTLP). Single metric with 'metric' label.", - "metrics": ["xrpld_txq_metrics"] + "metrics": ["txq_metrics"] }, "phase9_rpc_method": { "description": "Phase 9 per-RPC-method counters (MetricsRegistry via OTLP).", - "metrics": ["xrpld_rpc_method_started_total"] + "metrics": ["rpc_method_started_total"] }, "phase9_objects": { "description": "Phase 9 counted object instances observable gauge (MetricsRegistry via OTLP).", - "metrics": ["xrpld_object_count"] + "metrics": ["object_count"] }, "phase9_load": { "description": "Phase 9 fee escalation and load factor observable gauge (MetricsRegistry via OTLP).", - "metrics": ["xrpld_load_factor_metrics"] + "metrics": ["load_factor_metrics"] }, "parity_validation_agreement": { "description": "External dashboard parity: validation agreement percentages (MetricsRegistry).", "metrics": [ - "xrpld_validation_agreement{metric=\"agreement_pct_1h\"}", - "xrpld_validation_agreement{metric=\"agreement_pct_24h\"}" + "validation_agreement{metric=\"agreement_pct_1h\"}", + "validation_agreement{metric=\"agreement_pct_24h\"}" ] }, "parity_validator_health": { "description": "External dashboard parity: validator health indicators (MetricsRegistry).", "metrics": [ - "xrpld_validator_health{metric=\"amendment_blocked\"}", - "xrpld_validator_health{metric=\"unl_expiry_days\"}" + "validator_health{metric=\"amendment_blocked\"}", + "validator_health{metric=\"unl_expiry_days\"}" ] }, "parity_peer_quality": { "description": "External dashboard parity: peer quality metrics (MetricsRegistry).", "metrics": [ - "xrpld_peer_quality{metric=\"peer_latency_p90_ms\"}", - "xrpld_peer_quality{metric=\"peers_insane_count\"}" + "peer_quality{metric=\"peer_latency_p90_ms\"}", + "peer_quality{metric=\"peers_insane_count\"}" ] }, "parity_ledger_economy": { "description": "External dashboard parity: ledger economy metrics (MetricsRegistry).", "metrics": [ - "xrpld_ledger_economy{metric=\"base_fee_xrp\"}", - "xrpld_ledger_economy{metric=\"transaction_rate\"}" + "ledger_economy{metric=\"base_fee_xrp\"}", + "ledger_economy{metric=\"transaction_rate\"}" ] }, "parity_state_tracking": { "description": "External dashboard parity: server state tracking (MetricsRegistry).", - "metrics": ["xrpld_state_tracking{metric=\"state_value\"}"] + "metrics": ["state_tracking{metric=\"state_value\"}"] }, "parity_counters": { "description": "External dashboard parity: monotonic counters (MetricsRegistry).", "metrics": [ - "xrpld_ledgers_closed_total", - "xrpld_validations_sent_total", - "xrpld_state_changes_total" + "ledgers_closed_total", + "validations_sent_total", + "state_changes_total" ] }, "parity_storage": { "description": "External dashboard parity: storage detail metrics (MetricsRegistry).", - "metrics": ["xrpld_storage_detail{metric=\"nudb_bytes\"}"] + "metrics": ["storage_detail{metric=\"nudb_bytes\"}"] }, "grafana_dashboards": { "description": "All Grafana dashboards that must render data (UIDs as provisioned on disk under docker/telemetry/grafana/dashboards/).", diff --git a/docker/telemetry/workload/regression-metrics.json b/docker/telemetry/workload/regression-metrics.json index 0fd4e29060..f475d86fd2 100644 --- a/docker/telemetry/workload/regression-metrics.json +++ b/docker/telemetry/workload/regression-metrics.json @@ -2,7 +2,7 @@ "_description": "Metric surface for the OTel-driven regression gate. Each entry names a metric, the quantiles to capture, and how to query Prometheus. The comparator compares current run against baseline-timings.json under these exact keys.", "_key_format": "{category}.{name}.p{quantile} (e.g. span.tx.process.p99, rpc.server_info.p95, job.transaction.queued.p95)", "spans": { - "_query_template": "histogram_quantile({quantile}, sum by (le) (rate(traces_span_metrics_duration_milliseconds_bucket{span_name=\"{name}\"}[{window}])))", + "_query_template": "histogram_quantile({quantile}, sum by (le) (rate(span_duration_milliseconds_bucket{span_name=\"{name}\"}[{window}])))", "_unit": "ms", "_quantiles": [0.5, 0.95, 0.99], "names": [ @@ -18,8 +18,8 @@ ] }, "job_queue": { - "_queued_template": "histogram_quantile({quantile}, sum by (le) (rate(xrpld_job_queued_duration_us_bucket{job_type=\"{name}\"}[{window}])))", - "_running_template": "histogram_quantile({quantile}, sum by (le) (rate(xrpld_job_running_duration_us_bucket{job_type=\"{name}\"}[{window}])))", + "_queued_template": "histogram_quantile({quantile}, sum by (le) (rate(job_queued_us_bucket{job_type=\"{name}\"}[{window}])))", + "_running_template": "histogram_quantile({quantile}, sum by (le) (rate(job_running_us_bucket{job_type=\"{name}\"}[{window}])))", "_unit": "us", "_quantiles": [0.95], "_phases": ["queued", "running"], diff --git a/docker/telemetry/workload/run-full-validation.sh b/docker/telemetry/workload/run-full-validation.sh index 6fc2c36ab9..c6c292410b 100755 --- a/docker/telemetry/workload/run-full-validation.sh +++ b/docker/telemetry/workload/run-full-validation.sh @@ -300,7 +300,7 @@ trace_ledger=1 # Native OTel metrics via OTLP/HTTP. The collector has no StatsD receiver # (metrics pipeline is [otlp, spanmetrics]), so beast::insight must export # over OTLP for system metrics to reach Prometheus. prefix=xrpld matches the -# OTel resource service name and the xrpld_* names the dashboards query. +# OTel resource service name and the metric names the dashboards query. server=otel endpoint=http://localhost:4318/v1/metrics prefix=xrpld diff --git a/docker/telemetry/workload/validate_telemetry.py b/docker/telemetry/workload/validate_telemetry.py index e4e70c0acc..40f7e507a8 100644 --- a/docker/telemetry/workload/validate_telemetry.py +++ b/docker/telemetry/workload/validate_telemetry.py @@ -541,20 +541,30 @@ async def validate_metrics( ) as resp: label_data = await resp.json() all_metrics = label_data.get("data", []) - # Log xrpld-related and Phase 9 metrics for debugging. + # Log relevant metrics for debugging. relevant = [ m for m in all_metrics - if "xrpld" in m.lower() - or m.startswith( + if m.startswith( ( + "span_", "rpc_method", "cache_", "txq_", "object_count", "load_factor", "nodestore", - "traces_span", + "ledgermaster", + "peer_finder", + "jobq_", + "total_bytes", + "total_messages", + "validation_agreement", + "validator_health", + "peer_quality", + "ledger_economy", + "state_tracking", + "storage_detail", ) ) ] @@ -936,8 +946,8 @@ async def validate_span_durations( # dotted xrpl.* reserved for resource attributes). The amendment_blocked, # server_state, and proposers_validated values that earlier external-dashboard # work tracked are NOT span attributes — they exist only as MetricsRegistry -# metrics (xrpld_validator_health{metric="amendment_blocked"}, -# xrpld_state_tracking{metric="state_value"}, etc.), so they are validated by +# metrics (validator_health{metric="amendment_blocked"}, +# state_tracking{metric="state_value"}, etc.), so they are validated by # PARITY_VALUE_SANITY below rather than as span attributes here. PARITY_SPAN_ATTRS: list[dict[str, str]] = [ {"span": "tx.receive", "attr": "peer_version"}, @@ -955,26 +965,26 @@ PARITY_SPAN_ATTRS: list[dict[str, str]] = [ PARITY_VALUE_SANITY: list[dict[str, Any]] = [ { "name": "validation_agreement_pct_1h", - "query": 'xrpld_validation_agreement{metric="agreement_pct_1h"}', + "query": 'validation_agreement{metric="agreement_pct_1h"}', "lo": 0, "hi": 100, }, { "name": "unl_expiry_days", - "query": 'xrpld_validator_health{metric="unl_expiry_days"}', + "query": 'validator_health{metric="unl_expiry_days"}', "lo": 0, "hi": None, "exclusive_lo": True, }, { "name": "peer_latency_p90_ms", - "query": 'xrpld_peer_quality{metric="peer_latency_p90_ms"}', + "query": 'peer_quality{metric="peer_latency_p90_ms"}', "lo": 0, "hi": None, }, { "name": "state_value", - "query": 'xrpld_state_tracking{metric="state_value"}', + "query": 'state_tracking{metric="state_value"}', "lo": 0, "hi": 7, }, diff --git a/docker/telemetry/workload/xrpld-validator.cfg.template b/docker/telemetry/workload/xrpld-validator.cfg.template index 0181ddd437..725faf550c 100644 --- a/docker/telemetry/workload/xrpld-validator.cfg.template +++ b/docker/telemetry/workload/xrpld-validator.cfg.template @@ -76,7 +76,7 @@ trace_ledger=1 # --- Native OTel metrics (beast::insight over OTLP/HTTP) --- # The collector has no StatsD receiver (metrics pipeline is [otlp, spanmetrics]), # so beast::insight exports natively over OTLP. prefix=xrpld matches the OTel -# resource service name and the xrpld_* names the dashboards query. +# resource service name and the metric names the dashboards query. [insight] server=otel endpoint={{OTEL_METRICS_ENDPOINT}} diff --git a/src/libxrpl/beast/insight/OTelCollector.cpp b/src/libxrpl/beast/insight/OTelCollector.cpp index 0dab6d0ff1..5e6509ff52 100644 --- a/src/libxrpl/beast/insight/OTelCollector.cpp +++ b/src/libxrpl/beast/insight/OTelCollector.cpp @@ -130,7 +130,7 @@ public: /** * @param name Export-ready metric name, already run through * formatName() by the collector: prefix prepended and - * dots replaced with underscores (e.g. "xrpld_rpc_size"). + * dots replaced with underscores (e.g. "rpc_size"). * @param meter OTel Meter used to create the counter instrument. */ OTelCounterImpl( @@ -172,7 +172,7 @@ public: /** * @param name Export-ready metric name, already run through * formatName() by the collector: prefix prepended and - * dots replaced with underscores (e.g. "xrpld_rpc_size"). + * dots replaced with underscores (e.g. "rpc_size"). * @param meter OTel Meter used to create the histogram instrument. */ OTelEventImpl( @@ -294,7 +294,7 @@ public: /** * @param name Export-ready metric name, already run through * formatName() by the collector: prefix prepended and - * dots replaced with underscores (e.g. "xrpld_rpc_size"). + * dots replaced with underscores (e.g. "rpc_size"). * @param meter OTel Meter used to create the counter instrument. */ OTelMeterImpl( @@ -374,7 +374,7 @@ private: * "node-1", "xrpld", "mainnet", journal); * auto counter = collector->makeCounter("rpc.requests"); * counter.increment(1); - * // Metric "xrpld_rpc_requests" exported via OTLP every 1s. + * // Metric "rpc_requests" exported via OTLP every 1s. * @endcode */ class OTelCollectorImp : public OTelCollector, public std::enable_shared_from_this @@ -484,7 +484,7 @@ public: * * Replaces dots with underscores to match StatsD->Prometheus naming. * Example: prefix="xrpld", name="LedgerMaster.Validated_Ledger_Age" - * -> "xrpld_LedgerMaster_Validated_Ledger_Age" + * -> "ledgermaster_validated_ledger_age" * * @param name Raw metric name from beast::insight callers. * @return Fully-qualified metric name. diff --git a/src/libxrpl/telemetry/Telemetry.cpp b/src/libxrpl/telemetry/Telemetry.cpp index c9376a806b..72447c0a33 100644 --- a/src/libxrpl/telemetry/Telemetry.cpp +++ b/src/libxrpl/telemetry/Telemetry.cpp @@ -414,7 +414,7 @@ public: // histogram instruments align with the collector's SpanMetrics. auto histogramSelector = metrics_sdk::InstrumentSelectorFactory::Create( metrics_sdk::InstrumentType::kHistogram, "*", "ms"); - auto meterSelector = metrics_sdk::MeterSelectorFactory::Create("xrpld_metrics", "", ""); + auto meterSelector = metrics_sdk::MeterSelectorFactory::Create("metrics", "", ""); auto histogramConfig = std::make_shared(); histogramConfig->boundaries_ = std::vector{1.0, 5.0, 10.0, 25.0, 50.0, 100.0, 250.0, 500.0, 1000.0, 5000.0}; diff --git a/src/xrpld/telemetry/MetricsRegistry.h b/src/xrpld/telemetry/MetricsRegistry.h index 8ee3348863..7aa99f50fb 100644 --- a/src/xrpld/telemetry/MetricsRegistry.h +++ b/src/xrpld/telemetry/MetricsRegistry.h @@ -23,23 +23,23 @@ | +-- OtlpHttpMetricExporter | +-- Counters / Histograms (synchronous instruments) - | +-- xrpld_rpc_method_started_total - | +-- xrpld_rpc_method_finished_total - | +-- xrpld_rpc_method_errored_total - | +-- xrpld_rpc_method_duration_us (Histogram) - | +-- xrpld_job_queued_total - | +-- xrpld_job_started_total - | +-- xrpld_job_finished_total - | +-- xrpld_job_queued_duration_us (Histogram) - | +-- xrpld_job_running_duration_us (Histogram) - | +-- xrpld_ledgers_closed_total - | +-- xrpld_validations_sent_total - | +-- xrpld_validations_checked_total - | +-- xrpld_state_changes_total - | +-- xrpld_jq_trans_overflow_total - | +-- xrpld_ledger_history_mismatch_total{reason} - | +-- xrpld_txq_expired_total - | +-- xrpld_txq_dropped_total{reason} + | +-- rpc_method_started_total + | +-- rpc_method_finished_total + | +-- rpc_method_errored_total + | +-- rpc_method_us (Histogram) + | +-- job_queued_total + | +-- job_started_total + | +-- job_finished_total + | +-- job_queued_us (Histogram) + | +-- job_running_us (Histogram) + | +-- ledgers_closed_total + | +-- validations_sent_total + | +-- validations_checked_total + | +-- state_changes_total + | +-- jq_trans_overflow_total + | +-- ledger_history_mismatch_total{reason} + | +-- txq_expired_total + | +-- txq_dropped_total{reason} | +-- ValidationTracker (validation agreement tracker) | @@ -504,38 +504,38 @@ private: validationAgreementGauge_; // --- External dashboard parity counters (Task 7.14) --- - /// Counter: xrpld_ledgers_closed_total — incremented each consensus round. + /// Counter: ledgers_closed_total — incremented each consensus round. opentelemetry::nostd::unique_ptr> ledgersClosedCounter_; - /// Counter: xrpld_validations_sent_total — incremented when this node sends a validation. + /// Counter: validations_sent_total — incremented when this node sends a validation. opentelemetry::nostd::unique_ptr> validationsSentCounter_; - /// Counter: xrpld_validations_checked_total — incremented for each network validation + /// Counter: validations_checked_total — incremented for each network validation /// received. opentelemetry::nostd::unique_ptr> validationsCheckedCounter_; - /// Counter: xrpld_state_changes_total — incremented on operating mode transitions. + /// Counter: state_changes_total — incremented on operating mode transitions. opentelemetry::nostd::unique_ptr> stateChangesCounter_; - /// Counter: xrpld_jq_trans_overflow_total — incremented on job queue transaction overflows. + /// Counter: jq_trans_overflow_total — incremented on job queue transaction overflows. opentelemetry::nostd::unique_ptr> jqTransOverflowCounter_; - /// Counter: xrpld_ledger_history_mismatch_total{reason} — incremented per classified + /// Counter: ledger_history_mismatch_total{reason} — incremented per classified /// built-vs-validated ledger mismatch. opentelemetry::nostd::unique_ptr> ledgerHistoryMismatchCounter_; - /// Counter: xrpld_txq_expired_total — incremented per transaction expired out of the + /// Counter: txq_expired_total — incremented per transaction expired out of the /// transaction queue. opentelemetry::nostd::unique_ptr> txqExpiredCounter_; - /// Counter: xrpld_txq_dropped_total{reason} — incremented when a transaction is refused + /// Counter: txq_dropped_total{reason} — incremented when a transaction is refused /// admission to the queue. opentelemetry::nostd::unique_ptr> txqDroppedCounter_; - /// ObservableCounter: xrpld_validation_agreements_total — observed from + /// ObservableCounter: validation_agreements_total — observed from /// ValidationTracker::totalAgreementsEver() (monotonic gross lifetime /// tally, initial-classification semantics). opentelemetry::nostd::shared_ptr validationAgreementsObservable_; - /// ObservableCounter: xrpld_validation_missed_total — observed from + /// ObservableCounter: validation_missed_total — observed from /// ValidationTracker::totalMissedEver() (monotonic gross lifetime tally, /// initial-classification semantics). opentelemetry::nostd::shared_ptr diff --git a/src/xrpld/telemetry/ValidationTracker.h b/src/xrpld/telemetry/ValidationTracker.h index 301ad31fe0..32956235f7 100644 --- a/src/xrpld/telemetry/ValidationTracker.h +++ b/src/xrpld/telemetry/ValidationTracker.h @@ -190,7 +190,7 @@ public: * @note Unlike totalAgreements(), this is strictly monotonic: it is * incremented only when a ledger is first reconciled as an agreement and * is never adjusted by a late repair. It backs the monotonic Prometheus - * counter xrpld_validation_agreements_total. See the counting-semantics + * counter validation_agreements_total. See the counting-semantics * note in detail/ValidationTracker.cpp. */ uint64_t @@ -200,7 +200,7 @@ public: * @note Unlike totalMissed(), this is strictly monotonic: it is * incremented only when a ledger is first reconciled as a miss and is * never decremented by a late repair. It backs the monotonic Prometheus - * counter xrpld_validation_missed_total. See the counting-semantics note + * counter validation_missed_total. See the counting-semantics note * in detail/ValidationTracker.cpp. */ uint64_t @@ -291,13 +291,13 @@ private: // _total must never decrease) and additive: // totalAgreementsGross_ + totalMissedGross_ == ledgers reconciled. // The repaired/agreement view is still available from the windowed gauge - // (xrpld_validation_agreement) and the net totals above. + // (validation_agreement) and the net totals above. /// Monotonic lifetime initial agreements; backs - /// xrpld_validation_agreements_total. Never adjusted on repair. + /// validation_agreements_total. Never adjusted on repair. std::atomic totalAgreementsGross_{0}; - /// Monotonic lifetime initial misses; backs xrpld_validation_missed_total. + /// Monotonic lifetime initial misses; backs validation_missed_total. /// Never decremented on repair. std::atomic totalMissedGross_{0};