diff --git a/.cspell.config.yaml b/.cspell.config.yaml index d20b1ae7b3..01a51ea0e6 100644 --- a/.cspell.config.yaml +++ b/.cspell.config.yaml @@ -164,6 +164,7 @@ words: - itype - jemalloc - jlog + - jmeter - jtnofill - keylet - keylets diff --git a/.github/scripts/telemetry/check_bucket_parity.py b/.github/scripts/telemetry/check_bucket_parity.py new file mode 100755 index 0000000000..4c723fb4c0 --- /dev/null +++ b/.github/scripts/telemetry/check_bucket_parity.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +"""Assert the C++ millisecond ladder agrees with the collector's spanmetrics ladder. + +The two are specified to match so a span-derived latency panel and a native +histogram panel can be read on the same scale. They *were* identical when first +shipped. Then the collector ladder alone was extended -- sub-millisecond edges +below 1ms and second-scale edges up to 30s -- and nothing checked the other +side, so the C++ ladder stayed capped at 5s. Every quantile above 5s then read +back as a flat 5000, because Prometheus returns the second-highest edge for a +quantile landing in the `+Inf` bucket. That looks like a measurement rather +than an error, which is why it survived for eleven phases. + +The rule is containment, not equality: + + * every representable collector edge MUST appear in the C++ ladder, so the + shared range reads identically; + * the C++ ladder MAY carry extra edges ABOVE the collector's highest edge, + because jobs outlive spans -- the updatepaths job type was measured + averaging ~60s, which no span approaches. Demanding equality would force a + ceiling that censors it, reintroducing the bug this guards against; + * collector edges below 1ms are expected to be ABSENT rather than missing: + beast::insight::Event rounds every duration up to a whole millisecond + before it reaches the histogram, so those edges could never collect a + sample. + +Exit 0 when the ladders agree, 1 with a diff when they do not. +""" + +import re +import sys +from pathlib import Path + +HEADER = Path("include/xrpl/telemetry/HistogramBuckets.h") +COLLECTOR = Path("docker/telemetry/otel-collector-config.yaml") + +# beast::insight::Event applies ceil, so anything below 1ms +# collapses onto the 1ms edge. +REPRESENTABLE_FLOOR_MS = 1.0 + +UNIT_TO_MS = {"ms": 1.0, "s": 1000.0} + + +def collector_edges_ms(): + """Parse the spanmetrics bucket list, normalising each edge to milliseconds.""" + text = COLLECTOR.read_text() + match = re.search(r"buckets:\s*\[(.*?)\]", text, re.S) + if not match: + sys.exit(f"{COLLECTOR}: no 'buckets:' list found") + + edges = [] + for raw in match.group(1).split(","): + token = raw.strip() + if not token: + continue + parsed = re.fullmatch(r"([0-9.]+)(ms|s)", token) + if not parsed: + sys.exit(f"{COLLECTOR}: cannot parse bucket edge {token!r}") + edges.append(float(parsed.group(1)) * UNIT_TO_MS[parsed.group(2)]) + return edges + + +def cpp_edges_ms(): + """Parse kMillisecondBuckets out of the header that owns every ladder.""" + text = HEADER.read_text() + match = re.search(r"kMillisecondBuckets\{(.*?)\};", text, re.S) + if not match: + sys.exit(f"{HEADER}: kMillisecondBuckets not found") + return [ + float(token.strip().replace("'", "")) + for token in match.group(1).split(",") + if token.strip() + ] + + +def main(): + collector = collector_edges_ms() + cpp = cpp_edges_ms() + required = [edge for edge in collector if edge >= REPRESENTABLE_FLOOR_MS] + if not required: + sys.exit(f"{COLLECTOR}: no edges at or above {REPRESENTABLE_FLOOR_MS} ms") + collector_top = max(required) + + missing = [edge for edge in required if edge not in cpp] + # An extra C++ edge inside the collector's range means the two scales + # disagree where they overlap. Above the collector's top it is a deliberate + # extension. + inside_range = [e for e in cpp if e not in required and e < collector_top] + + if not missing and not inside_range: + extensions = [e for e in cpp if e > collector_top] + summary = f"OK: all {len(required)} representable collector edges present" + if extensions: + pretty = ", ".join(f"{e:g}" for e in extensions) + summary += ( + f"; {len(extensions)} extension edge(s) above " + f"{collector_top:g} ms: [{pretty}]" + ) + print(summary) + return 0 + + print("Bucket ladder parity violated.", file=sys.stderr) + print( + f" collector (>= {REPRESENTABLE_FLOOR_MS:g} ms): " + f"{[f'{e:g}' for e in required]}", + file=sys.stderr, + ) + print( + f" HistogramBuckets.h : {[f'{e:g}' for e in cpp]}", file=sys.stderr + ) + for edge in missing: + print(f" MISSING from the C++ ladder: {edge:g} ms", file=sys.stderr) + for edge in inside_range: + print( + f" C++ edge {edge:g} ms lies inside the collector's range but is not " + "a collector edge -- add it to the collector or drop it here", + file=sys.stderr, + ) + print( + "\nThe two ladders must agree over their shared range. Extra C++ edges are\n" + "permitted only ABOVE the collector's highest edge. Change both sides, or\n" + "change the spec in OpenTelemetryPlan/Phase7_taskList.md.", + file=sys.stderr, + ) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/on-pr.yml b/.github/workflows/on-pr.yml index 9fd8b14b27..5135f45b43 100644 --- a/.github/workflows/on-pr.yml +++ b/.github/workflows/on-pr.yml @@ -72,6 +72,7 @@ jobs: .github/scripts/levelization/** .github/scripts/otel-naming/** .github/scripts/rename/** + .github/scripts/telemetry/** .github/workflows/reusable-check-levelization.yml .github/workflows/reusable-check-otel-naming.yml .github/workflows/reusable-check-rename.yml diff --git a/.github/workflows/reusable-check-otel-naming.yml b/.github/workflows/reusable-check-otel-naming.yml index 54cab30640..a37e7e1632 100644 --- a/.github/workflows/reusable-check-otel-naming.yml +++ b/.github/workflows/reusable-check-otel-naming.yml @@ -33,3 +33,11 @@ jobs: # it enforces each rule only when the layer it needs is present, so it # works whether telemetry changes land in one PR or several. run: python .github/scripts/otel-naming/check_otel_naming.py + - name: Check histogram bucket parity + # The C++ millisecond ladder and the collector's spanmetrics ladder are + # specified to agree over their shared range. They were identical when + # first shipped, then the collector side alone was extended and nothing + # noticed for eleven phases: native histograms stayed capped at 5s while + # spans reached 30s, so every quantile above 5s reported a flat 5000. + # Nothing but a check keeps two lists in step. + run: python .github/scripts/telemetry/check_bucket_parity.py diff --git a/OpenTelemetryPlan/05-configuration-reference.md b/OpenTelemetryPlan/05-configuration-reference.md index e3b8ae5fe6..2210a9a268 100644 --- a/OpenTelemetryPlan/05-configuration-reference.md +++ b/OpenTelemetryPlan/05-configuration-reference.md @@ -68,13 +68,13 @@ The authoritative `[telemetry]` example lives in `cfg/xrpld-example.cfg`. Teleme | Option | Type | Default | Description | | -------------------------- | ------ | ---------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `enabled` | bool | `false` | Enable/disable telemetry | +| `enabled` | 0 or 1 | `0` | Enable/disable telemetry | | `endpoint` | string | `http://localhost:4318/v1/traces` | OTLP/HTTP collector endpoint for **traces** | | `metrics_endpoint` | string | `http://localhost:4318/v1/metrics` | OTLP/HTTP collector endpoint for the native metrics pipeline (`MetricsRegistry`). Read in `Application.cpp:1670` | -| `use_tls` | bool | `false` | Enable TLS for exporter connection | +| `use_tls` | 0 or 1 | `0` | Enable TLS for exporter connection | | `tls_ca_cert` | string | `""` | Path to CA certificate file | -| `tls_client_cert` | string | `""` | Path to node's client certificate (PEM) for mutual TLS; requires `use_tls=1`; empty = one-way TLS | -| `tls_client_key` | string | `""` | Path to private key (PEM) for `tls_client_cert`; requires `use_tls=1`; required when the cert is set | +| `tls_client_cert` | string | `""` | Client cert (PEM) for mTLS; empty = one-way; if `enabled=1`, needs key + `use_tls=1` or startup fails | +| `tls_client_key` | string | `""` | Private key (PEM) for `tls_client_cert`; if set with `enabled=1`, needs the cert + `use_tls=1` or fails | | `batch_size` | uint | `512` | Spans per export batch | | `batch_delay_ms` | uint | `5000` | Max delay before sending batch (ms) | | `max_queue_size` | uint | `2048` | Maximum queued spans | diff --git a/OpenTelemetryPlan/09-data-collection-reference.md b/OpenTelemetryPlan/09-data-collection-reference.md index bee2152837..20576e2f35 100644 --- a/OpenTelemetryPlan/09-data-collection-reference.md +++ b/OpenTelemetryPlan/09-data-collection-reference.md @@ -531,12 +531,12 @@ a destructor must not depend on still existing. A query that only groups by The OTel Collector's SpanMetrics connector automatically generates RED (Rate, Errors, Duration) metrics from every span. No custom metrics code in xrpld is needed. -| Prometheus Metric | Type | Description | -| ----------------------------------- | --------- | ------------------------------------------------------------------------------ | -| `span_calls_total` | Counter | Total span invocations | -| `span_duration_milliseconds_bucket` | Histogram | Latency distribution (buckets: 1, 5, 10, 25, 50, 100, 250, 500, 1000, 5000 ms) | -| `span_duration_milliseconds_count` | Histogram | Observation count | -| `span_duration_milliseconds_sum` | Histogram | Cumulative latency | +| Prometheus Metric | Type | Description | +| ----------------------------------- | --------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `span_calls_total` | Counter | Total span invocations | +| `span_duration_milliseconds_bucket` | Histogram | Latency distribution. Buckets come from the collector's spanmetrics config: 0.01, 0.05, 0.1, 0.25, 0.5, 1, 5, 10, 25, 50, 100, 250, 500 ms then 1, 2, 3, 4, 5, 10, 30 s. The sub-millisecond edges exist because most xrpld spans are far below 1 ms; without them every p95/p99 pinned to a constant 0.95 ms | +| `span_duration_milliseconds_count` | Histogram | Observation count | +| `span_duration_milliseconds_sum` | Histogram | Cumulative latency | **Standard labels on every metric**: `span_name`, `status_code`, `service_name`, `span_kind` @@ -684,13 +684,14 @@ prefix=xrpld Quantiles collected: 0th, 50th, 90th, 95th, 99th, 100th percentile. -\* **`rpc_size` instrument mismatch (known issue):** response size in bytes is -recorded through the millisecond-scaled event histogram (`makeEvent`), so it is -exported as `rpc_size_milliseconds_bucket` with time-scaled boundaries that top -out at 5000. Byte values above ~5 KB saturate in the last bucket, so the -percentiles are not true byte sizes. The _RPC & Pathfinding_ panel is flagged -accordingly. A dedicated byte-unit histogram is needed to fix this; tracked -separately. +\* **`rpc_size` now records bytes as bytes (fixed).** It used to go through the +millisecond-scaled event histogram and export as `rpc_size_milliseconds_bucket` +on a ladder topping out at 5000, so the 24.9% of responses larger than 5 kB all +landed in the last bucket and every percentile read back as a flat 5000 — a +plausible-looking constant rather than a byte size. `beast::insight::Event` now +declares a `Unit`, so this instrument is created with unit `By` and exports as +**`rpc_size_bytes_bucket`** on `kByteBuckets` (512 B to 1 MiB, placed from the +measured distribution). Queries and panels must use the new name. **Grafana dashboards**: _Node Health_ (`ios_latency`), _RPC & Pathfinding_ (`rpc_time`, `rpc_size`, `pathfind_*`) diff --git a/OpenTelemetryPlan/Phase7_taskList.md b/OpenTelemetryPlan/Phase7_taskList.md index be90294ff8..a9e9abef99 100644 --- a/OpenTelemetryPlan/Phase7_taskList.md +++ b/OpenTelemetryPlan/Phase7_taskList.md @@ -53,7 +53,7 @@ - **OTelCounterImpl**: Wraps `opentelemetry::metrics::Counter`. `increment(amount)` calls `counter->Add(amount)`. - **OTelGaugeImpl**: Uses `opentelemetry::metrics::ObservableGauge` with an async callback. `set(value)` stores value atomically; callback reads it during collection. - **OTelMeterImpl**: Wraps `opentelemetry::metrics::Counter`. `increment(amount)` calls `counter->Add(amount)`. Semantically identical to Counter but unsigned. - - **OTelEventImpl**: Wraps `opentelemetry::metrics::Histogram`. `notify(duration)` calls `histogram->Record(duration.count())`. Uses explicit bucket boundaries matching SpanMetrics: [1, 5, 10, 25, 50, 100, 250, 500, 1000, 5000] ms. + - **OTelEventImpl**: Wraps `opentelemetry::metrics::Histogram`. `notify()` calls `histogram->Record(value.count())`. Declares its unit from `beast::insight::Unit`, which is what selects its bucket ladder: the histogram views in `Telemetry.cpp` match on unit, so a `ms` instrument gets the millisecond ladder and a `By` instrument the byte ladder. Bucket edges live in `include/xrpl/telemetry/HistogramBuckets.h` — do not restate them here. The millisecond ladder must contain every representable edge of the collector's spanmetrics ladder and may extend above it (jobs outlive spans); `.github/scripts/telemetry/check_bucket_parity.py` enforces that. An earlier version of this line specified `[1, 5, 10, 25, 50, 100, 250, 500, 1000, 5000] ms` as "matching SpanMetrics" — true when written, then silently false once the collector ladder was extended on its own, which capped every quantile above 5s at a flat 5000. - **OTelHookImpl**: Stores handler function. Called during periodic metric collection (same 1s pattern via PeriodicMetricReader). - **OTelCollectorImp**: Main class. - Creates `MeterProvider` with `PeriodicMetricReader` (1s export interval) diff --git a/cfg/xrpld-example.cfg b/cfg/xrpld-example.cfg index ed06f46241..843126386a 100644 --- a/cfg/xrpld-example.cfg +++ b/cfg/xrpld-example.cfg @@ -1744,13 +1744,21 @@ validators.txt # tls_client_cert= # # Path to this node's PEM-encoded client certificate, presented to the -# collector for mutual TLS (mTLS). Only used when use_tls=1. Leave empty +# collector for mutual TLS (mTLS). Requires use_tls=1. Leave empty # for one-way (server-only) TLS. Default: empty. # +# To enable mTLS, both tls_client_cert and tls_client_key must be +# specified. If only one is provided, xrpld will fail to start. Providing +# them while use_tls=0 also fails to start, rather than being ignored. +# Both checks apply only when enabled=1; with telemetry disabled these +# settings are read but never validated. +# # tls_client_key= # # Path to the PEM-encoded private key for tls_client_cert. Required -# whenever tls_client_cert is set. Only used when use_tls=1. +# whenever tls_client_cert is set. Requires use_tls=1. Both conditions +# are enforced exactly as described under tls_client_cert above: when +# enabled=1, breaking either one makes xrpld fail to start. # Default: empty. # # Head sampling is intentionally fixed at 1.0 (sample everything) and is diff --git a/docker/telemetry/grafana/dashboards/consensus-health.json b/docker/telemetry/grafana/dashboards/consensus-health.json index 2dc19084e3..fbadb7f1fb 100644 --- a/docker/telemetry/grafana/dashboards/consensus-health.json +++ b/docker/telemetry/grafana/dashboards/consensus-health.json @@ -21,11 +21,28 @@ "enable": true, "hide": false, "iconColor": "rgb(70, 70, 70)", - "name": "Annotate perf-iac runs", + "name": "Perf Runs (JMeter)", "target": { "limit": 100, "matchAny": false, - "tags": ["perf-iac"], + "tags": ["perf-iac", "jmeter"], + "type": "tags" + }, + "type": "tags" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(15, 122, 102)", + "name": "Perf Runs (Locust)", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac", "locust"], "type": "tags" }, "type": "tags" diff --git a/docker/telemetry/grafana/dashboards/fee-market.json b/docker/telemetry/grafana/dashboards/fee-market.json index 71dd0ba6ed..8faf0acc7c 100644 --- a/docker/telemetry/grafana/dashboards/fee-market.json +++ b/docker/telemetry/grafana/dashboards/fee-market.json @@ -21,11 +21,28 @@ "enable": true, "hide": false, "iconColor": "rgb(70, 70, 70)", - "name": "Annotate perf-iac runs", + "name": "Perf Runs (JMeter)", "target": { "limit": 100, "matchAny": false, - "tags": ["perf-iac"], + "tags": ["perf-iac", "jmeter"], + "type": "tags" + }, + "type": "tags" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(15, 122, 102)", + "name": "Perf Runs (Locust)", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac", "locust"], "type": "tags" }, "type": "tags" diff --git a/docker/telemetry/grafana/dashboards/job-queue.json b/docker/telemetry/grafana/dashboards/job-queue.json index 52319e1df1..d7baefa1b1 100644 --- a/docker/telemetry/grafana/dashboards/job-queue.json +++ b/docker/telemetry/grafana/dashboards/job-queue.json @@ -1,6 +1,53 @@ { "annotations": { - "list": [] + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(70, 70, 70)", + "name": "Perf Runs (JMeter)", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac", "jmeter"], + "type": "tags" + }, + "type": "tags" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(15, 122, 102)", + "name": "Perf Runs (Locust)", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac", "locust"], + "type": "tags" + }, + "type": "tags" + } + ] }, "description": "What this shows: Per-job-type throughput, queue wait times, and execution times for the node's internal worker job queue.\nUse it to: Find job types that are backing up or running slowly and causing internal processing delays.", "editable": true, diff --git a/docker/telemetry/grafana/dashboards/ledger-data-sync.json b/docker/telemetry/grafana/dashboards/ledger-data-sync.json index 5b2d4ed048..8f9e33efc5 100644 --- a/docker/telemetry/grafana/dashboards/ledger-data-sync.json +++ b/docker/telemetry/grafana/dashboards/ledger-data-sync.json @@ -1,6 +1,53 @@ { "annotations": { - "list": [] + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(70, 70, 70)", + "name": "Perf Runs (JMeter)", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac", "jmeter"], + "type": "tags" + }, + "type": "tags" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(15, 122, 102)", + "name": "Perf Runs (Locust)", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac", "locust"], + "type": "tags" + }, + "type": "tags" + } + ] }, "description": "What this shows: Ledger data exchange and object-fetch traffic between this node and its peers: ledger sync, tree-node retrieval, and transaction-set exchange.\nUse it to: See how much ledger data the node is pulling or serving and spot catch-up activity.", "editable": true, @@ -1204,7 +1251,7 @@ }, { "title": "Job Queue Wait p95 By Type", - "description": "###### What this is:\n*95th-percentile time a job waits in the queue before a worker thread picks it up, for the sync-critical job types. This is the metric form of the 'ProcessLData wait: NNNNms' warnings in the debug log.*\n\n###### How it's computed:\n*histogram_quantile(0.95, rate(jobq__q_milliseconds_bucket[$__rate_interval])) for ledgerdata, acceptledger, fetchtxndata, transaction, advanceledger, ledgerrequest.*\n\n###### Reading it:\n*Queue wait should be single-digit to low-tens of ms. High ledgerdata/fetchtxndata wait = the node cannot process inbound ledger data fast enough.*\n\n###### Healthy range:\n*< ~50ms p95 per type on a healthy node.*\n\n###### Watch for:\n*ledgerdata or fetchtxndata q-wait spiking to seconds = worker threads are blocked (usually on NuDB reads - see the cause tier).*\n\n###### Keywords:\n- **Job queue / job type** *(per node)* \u2014 xrpld's worker-thread pool; every unit of background work is enqueued under a named job type.\n- **Deferred job** *(per node)* \u2014 a job held back because its type is already at its concurrency limit; the leading indicator of queue backpressure.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[core/JobQueue.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/core/detail/JobQueue.cpp)\n\n###### Function:\n`JobQueue::getJson (per-type queue timing)`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#job-queue-job-type)", + "description": "###### What this is:\n*95th-percentile time a job waits in the queue before a worker thread picks it up, for the sync-critical job types. This is the metric form of the 'ProcessLData wait: NNNNms' warnings in the debug log.*\n\n###### How it's computed:\n*histogram_quantile(0.95, rate(job_queued_us_bucket{job_type=\"\"}[$__rate_interval])) for ledgerData, acceptLedger, fetchTxnData, transaction, advanceLedger, ledgerRequest. Reads the OTel-native microsecond instrument rather than the beast jobq_* pair: beast Events round every duration up to a whole millisecond, so 94-100% of their samples landed in the first bucket and every percentile was an interpolation inside it rather than a measurement.*\n\n###### Reading it:\n*Queue wait is normally tens to hundreds of microseconds. High ledgerData/fetchTxnData wait = the node cannot process inbound ledger data fast enough.*\n\n###### Healthy range:\n*< ~500us p95 per type on a healthy node; sustained milliseconds is already backpressure.*\n\n###### Watch for:\n*ledgerData or fetchTxnData q-wait spiking to seconds = worker threads are blocked (usually on NuDB reads - see the cause tier).*\n\n###### Keywords:\n- **Job queue / job type** *(per node)* \u2014 xrpld's worker-thread pool; every unit of background work is enqueued under a named job type.\n- **Deferred job** *(per node)* \u2014 a job held back because its type is already at its concurrency limit; the leading indicator of queue backpressure.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code by MetricsRegistry as an OTel-native histogram in microseconds; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[core/JobQueue.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/core/detail/JobQueue.cpp)\n\n###### Function:\n`JobQueue::getJson (per-type queue timing)`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#job-queue-job-type)", "type": "timeseries", "gridPos": { "h": 10, @@ -1225,48 +1272,48 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(jobq_ledgerdata_q_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"ledgerdata q-wait p95\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(job_queued_us_bucket{job_type=\"ledgerData\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"ledgerData q-wait p95\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(jobq_acceptledger_q_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"acceptledger q-wait p95\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(job_queued_us_bucket{job_type=\"acceptLedger\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"acceptLedger q-wait p95\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(jobq_fetchtxndata_q_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"fetchtxndata q-wait p95\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(job_queued_us_bucket{job_type=\"fetchTxnData\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"fetchTxnData q-wait p95\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(jobq_transaction_q_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"transaction q-wait p95\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(job_queued_us_bucket{job_type=\"transaction\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"transaction q-wait p95\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(jobq_advanceledger_q_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"advanceledger q-wait p95\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(job_queued_us_bucket{job_type=\"advanceLedger\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"advanceLedger q-wait p95\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(jobq_ledgerrequest_q_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"ledgerrequest q-wait p95\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(job_queued_us_bucket{job_type=\"ledgerRequest\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"ledgerRequest q-wait p95\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" } ], "fieldConfig": { "defaults": { "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", - "unit": "ms", + "unit": "µs", "custom": { "axisLabel": "p95 Wait (ms)", "spanNulls": 1800000, diff --git a/docker/telemetry/grafana/dashboards/ledger-operations.json b/docker/telemetry/grafana/dashboards/ledger-operations.json index 9cc7726cb0..7abc014089 100644 --- a/docker/telemetry/grafana/dashboards/ledger-operations.json +++ b/docker/telemetry/grafana/dashboards/ledger-operations.json @@ -21,11 +21,28 @@ "enable": true, "hide": false, "iconColor": "rgb(70, 70, 70)", - "name": "Annotate perf-iac runs", + "name": "Perf Runs (JMeter)", "target": { "limit": 100, "matchAny": false, - "tags": ["perf-iac"], + "tags": ["perf-iac", "jmeter"], + "type": "tags" + }, + "type": "tags" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(15, 122, 102)", + "name": "Perf Runs (Locust)", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac", "locust"], "type": "tags" }, "type": "tags" diff --git a/docker/telemetry/grafana/dashboards/ledger-sync-health.json b/docker/telemetry/grafana/dashboards/ledger-sync-health.json index d68c55fe27..c810d08fa2 100644 --- a/docker/telemetry/grafana/dashboards/ledger-sync-health.json +++ b/docker/telemetry/grafana/dashboards/ledger-sync-health.json @@ -21,11 +21,28 @@ "enable": true, "hide": false, "iconColor": "rgb(70, 70, 70)", - "name": "Annotate perf-iac runs", + "name": "Perf Runs (JMeter)", "target": { "limit": 100, "matchAny": false, - "tags": ["perf-iac"], + "tags": ["perf-iac", "jmeter"], + "type": "tags" + }, + "type": "tags" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(15, 122, 102)", + "name": "Perf Runs (Locust)", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac", "locust"], "type": "tags" }, "type": "tags" diff --git a/docker/telemetry/grafana/dashboards/log-derived-insights.json b/docker/telemetry/grafana/dashboards/log-derived-insights.json index 6acda66c69..2fa1b84eb3 100644 --- a/docker/telemetry/grafana/dashboards/log-derived-insights.json +++ b/docker/telemetry/grafana/dashboards/log-derived-insights.json @@ -14,7 +14,54 @@ "uid": "log-derived-insights", "refresh": "60s", "annotations": { - "list": [] + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(70, 70, 70)", + "name": "Perf Runs (JMeter)", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac", "jmeter"], + "type": "tags" + }, + "type": "tags" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(15, 122, 102)", + "name": "Perf Runs (Locust)", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac", "locust"], + "type": "tags" + }, + "type": "tags" + } + ] }, "templating": { "list": [ diff --git a/docker/telemetry/grafana/dashboards/network-traffic.json b/docker/telemetry/grafana/dashboards/network-traffic.json index f258ff8db1..272c9cec3a 100644 --- a/docker/telemetry/grafana/dashboards/network-traffic.json +++ b/docker/telemetry/grafana/dashboards/network-traffic.json @@ -21,11 +21,28 @@ "enable": true, "hide": false, "iconColor": "rgb(70, 70, 70)", - "name": "Annotate perf-iac runs", + "name": "Perf Runs (JMeter)", "target": { "limit": 100, "matchAny": false, - "tags": ["perf-iac"], + "tags": ["perf-iac", "jmeter"], + "type": "tags" + }, + "type": "tags" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(15, 122, 102)", + "name": "Perf Runs (Locust)", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac", "locust"], "type": "tags" }, "type": "tags" diff --git a/docker/telemetry/grafana/dashboards/node-health.json b/docker/telemetry/grafana/dashboards/node-health.json index 9a21fca75c..08a693c524 100644 --- a/docker/telemetry/grafana/dashboards/node-health.json +++ b/docker/telemetry/grafana/dashboards/node-health.json @@ -21,11 +21,28 @@ "enable": true, "hide": false, "iconColor": "rgb(70, 70, 70)", - "name": "Annotate perf-iac runs", + "name": "Perf Runs (JMeter)", "target": { "limit": 100, "matchAny": false, - "tags": ["perf-iac"], + "tags": ["perf-iac", "jmeter"], + "type": "tags" + }, + "type": "tags" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(15, 122, 102)", + "name": "Perf Runs (Locust)", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac", "locust"], "type": "tags" }, "type": "tags" @@ -810,7 +827,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "###### What this is:\n*Transitions into each operating mode, per interval.*\n\n###### How it's computed:\n*increase() over each per-mode transition counter, so each point is the number of transitions in that bucket and the series stays correct across an xrpld restart (the counters reset to 0).*\n\n###### Reading it:\n*Zero is healthy; each point is a mode change within that bucket. Brief flaps show up here even when they are too short to appear on Operating Mode (State Timeline), which can only sample state once per scrape.*\n\n###### Healthy range:\n*Few transitions once the node is stable in Full mode.*\n\n###### Watch for:\n*Frequent transitions out of Full, or into Disconnected or Syncing, indicate instability.*\n\n###### Keywords:\n- **Operating mode / server state** *(per node)* \u2014 the node's sync level: Disconnected, Connected, Syncing, Tracking, Full (and Validating/Proposing).\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`NetworkOPsImp::Stats`\n\n###### References:\n[Operating mode / server state](https://xrpl.org/docs/references/http-websocket-apis/api-conventions/xrpld-server-states) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#operating-mode-server-state)", + "description": "###### What this is:\n*Transitions into each operating mode, per interval.*\n\n###### How it's computed:\n*round(increase(...[$__interval])) over each per-mode transition counter. $__interval tiles the buckets exactly, so each bar is the transitions in that bucket and the legend Total is the true count; $__rate_interval would overlap each bucket by one scrape and inflate it (measured +5% at a 36h range, +26% zoomed in). round() removes increase()'s extrapolation, which otherwise reports fractional counts. The series stays correct across an xrpld restart (the counters reset to 0).*\n\n###### Reading it:\n*Zero is healthy; each bar is the mode changes within that bucket. Brief flaps show up here even when they are too short to appear on Operating Mode (State Timeline), which can only sample state once per scrape.*\n\n###### Healthy range:\n*Few transitions once the node is stable in Full mode.*\n\n###### Watch for:\n*Frequent transitions out of Full, or into Disconnected or Syncing, indicate instability.*\n\n###### Keywords:\n- **Operating mode / server state** *(per node)* — the node's sync level: Disconnected, Connected, Syncing, Tracking, Full (and Validating/Proposing).\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`NetworkOPsImp::Stats`\n\n###### References:\n[Operating mode / server state](https://xrpl.org/docs/references/http-websocket-apis/api-conventions/xrpld-server-states) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#operating-mode-server-state)", "fieldConfig": { "defaults": { "color": { @@ -824,8 +841,8 @@ "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, + "drawStyle": "bars", + "fillOpacity": 70, "gradientMode": "none", "hideFrom": { "legend": false, @@ -834,7 +851,7 @@ }, "insertNulls": false, "lineInterpolation": "linear", - "lineWidth": 1, + "lineWidth": 0, "pointSize": 5, "scaleDistribution": { "type": "linear" @@ -864,7 +881,8 @@ } ] }, - "unit": "short" + "unit": "short", + "decimals": 0 } }, "gridPos": { @@ -880,8 +898,8 @@ "multiLane": false }, "legend": { - "calcs": [], - "displayMode": "list", + "calcs": ["sum", "max"], + "displayMode": "table", "enableFacetedFilter": false, "overflow": "ellipsis", "placement": "bottom", @@ -901,7 +919,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "label_replace(label_join(label_replace(increase(state_accounting_full_transitions{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Full\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "expr": "label_replace(label_join(label_replace(round(increase(state_accounting_full_transitions{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__interval])), \"series\", \"Full\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", "refId": "A" }, { @@ -909,7 +927,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "label_replace(label_join(label_replace(increase(state_accounting_tracking_transitions{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Tracking\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "expr": "label_replace(label_join(label_replace(round(increase(state_accounting_tracking_transitions{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__interval])), \"series\", \"Tracking\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", "refId": "B" }, { @@ -917,7 +935,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "label_replace(label_join(label_replace(increase(state_accounting_syncing_transitions{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Syncing\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "expr": "label_replace(label_join(label_replace(round(increase(state_accounting_syncing_transitions{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__interval])), \"series\", \"Syncing\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", "refId": "C" }, { @@ -925,7 +943,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "label_replace(label_join(label_replace(increase(state_accounting_connected_transitions{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Connected\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "expr": "label_replace(label_join(label_replace(round(increase(state_accounting_connected_transitions{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__interval])), \"series\", \"Connected\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", "refId": "D" }, { @@ -933,12 +951,13 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "label_replace(label_join(label_replace(increase(state_accounting_disconnected_transitions{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Disconnected\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "expr": "label_replace(label_join(label_replace(round(increase(state_accounting_disconnected_transitions{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__interval])), \"series\", \"Disconnected\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", "refId": "E" } ], "title": "Operating Mode Transitions", - "type": "timeseries" + "type": "timeseries", + "interval": "1m" }, { "datasource": { diff --git a/docker/telemetry/grafana/dashboards/overlay-traffic-detail.json b/docker/telemetry/grafana/dashboards/overlay-traffic-detail.json index df48858eab..c74e58235e 100644 --- a/docker/telemetry/grafana/dashboards/overlay-traffic-detail.json +++ b/docker/telemetry/grafana/dashboards/overlay-traffic-detail.json @@ -21,11 +21,28 @@ "enable": true, "hide": false, "iconColor": "rgb(70, 70, 70)", - "name": "Annotate perf-iac runs", + "name": "Perf Runs (JMeter)", "target": { "limit": 100, "matchAny": false, - "tags": ["perf-iac"], + "tags": ["perf-iac", "jmeter"], + "type": "tags" + }, + "type": "tags" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(15, 122, 102)", + "name": "Perf Runs (Locust)", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac", "locust"], "type": "tags" }, "type": "tags" diff --git a/docker/telemetry/grafana/dashboards/peer-network.json b/docker/telemetry/grafana/dashboards/peer-network.json index ca4d55239a..fc2f98b6d4 100644 --- a/docker/telemetry/grafana/dashboards/peer-network.json +++ b/docker/telemetry/grafana/dashboards/peer-network.json @@ -21,11 +21,28 @@ "enable": true, "hide": false, "iconColor": "rgb(70, 70, 70)", - "name": "Annotate perf-iac runs", + "name": "Perf Runs (JMeter)", "target": { "limit": 100, "matchAny": false, - "tags": ["perf-iac"], + "tags": ["perf-iac", "jmeter"], + "type": "tags" + }, + "type": "tags" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(15, 122, 102)", + "name": "Perf Runs (Locust)", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac", "locust"], "type": "tags" }, "type": "tags" diff --git a/docker/telemetry/grafana/dashboards/peer-quality.json b/docker/telemetry/grafana/dashboards/peer-quality.json index d45f516796..6fa96c0547 100644 --- a/docker/telemetry/grafana/dashboards/peer-quality.json +++ b/docker/telemetry/grafana/dashboards/peer-quality.json @@ -21,11 +21,28 @@ "enable": true, "hide": false, "iconColor": "rgb(70, 70, 70)", - "name": "Annotate perf-iac runs", + "name": "Perf Runs (JMeter)", "target": { "limit": 100, "matchAny": false, - "tags": ["perf-iac"], + "tags": ["perf-iac", "jmeter"], + "type": "tags" + }, + "type": "tags" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(15, 122, 102)", + "name": "Perf Runs (Locust)", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac", "locust"], "type": "tags" }, "type": "tags" diff --git a/docker/telemetry/grafana/dashboards/rpc-pathfinding.json b/docker/telemetry/grafana/dashboards/rpc-pathfinding.json index b751c16959..8feac10762 100644 --- a/docker/telemetry/grafana/dashboards/rpc-pathfinding.json +++ b/docker/telemetry/grafana/dashboards/rpc-pathfinding.json @@ -21,11 +21,28 @@ "enable": true, "hide": false, "iconColor": "rgb(70, 70, 70)", - "name": "Annotate perf-iac runs", + "name": "Perf Runs (JMeter)", "target": { "limit": 100, "matchAny": false, - "tags": ["perf-iac"], + "tags": ["perf-iac", "jmeter"], + "type": "tags" + }, + "type": "tags" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(15, 122, 102)", + "name": "Perf Runs (Locust)", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac", "locust"], "type": "tags" }, "type": "tags" @@ -137,7 +154,7 @@ }, { "title": "RPC Response Size", - "description": "\u26a0 Instrument mismatch \u2014 values unreliable. Response size is recorded through the millisecond-scaled event histogram (rpc_size_milliseconds_bucket), so byte values saturate at the top time bucket (5000) and the percentiles are not true byte sizes. A dedicated byte-unit histogram is needed to fix this; tracked separately. Treat this panel as indicative only until then.\n\n###### What this is:\n*The 95th-percentile size of RPC response payloads in bytes.*\n\n###### How it's computed:\n*95th-percentile of response payload sizes over the dashboard rate interval, per node.*\n\n###### Reading it:\n*Smaller is cheaper; large responses cost bandwidth and memory.*\n\n###### Healthy range:\n*Workload-dependent; small for status queries, large for bulk data queries.*\n\n###### Watch for:\n*Growth in large responses, consistent with expensive queries or API misuse.*\n\n###### Keywords:\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[ServerHandler.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/ServerHandler.cpp)\n\n###### Function:\n`ServerHandler ctor`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rpc-command-method)", + "description": "\u26a0 Instrument mismatch \u2014 values unreliable. Response size is recorded through the millisecond-scaled event histogram (rpc_size_bytes_bucket), so byte values saturate at the top time bucket (5000) and the percentiles are not true byte sizes. A dedicated byte-unit histogram is needed to fix this; tracked separately. Treat this panel as indicative only until then.\n\n###### What this is:\n*The 95th-percentile size of RPC response payloads in bytes.*\n\n###### How it's computed:\n*95th-percentile of response payload sizes over the dashboard rate interval, per node.*\n\n###### Reading it:\n*Smaller is cheaper; large responses cost bandwidth and memory.*\n\n###### Healthy range:\n*Workload-dependent; small for status queries, large for bulk data queries.*\n\n###### Watch for:\n*Growth in large responses, consistent with expensive queries or API misuse.*\n\n###### Keywords:\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[ServerHandler.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/ServerHandler.cpp)\n\n###### Function:\n`ServerHandler ctor`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rpc-command-method)", "type": "timeseries", "gridPos": { "h": 10, @@ -158,7 +175,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(rpc_size_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"P95 Response Size\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(rpc_size_bytes_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"P95 Response Size\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" } ], "fieldConfig": { diff --git a/docker/telemetry/grafana/dashboards/rpc-performance.json b/docker/telemetry/grafana/dashboards/rpc-performance.json index 8228c9283b..57f9d6b005 100644 --- a/docker/telemetry/grafana/dashboards/rpc-performance.json +++ b/docker/telemetry/grafana/dashboards/rpc-performance.json @@ -21,11 +21,28 @@ "enable": true, "hide": false, "iconColor": "rgb(70, 70, 70)", - "name": "Annotate perf-iac runs", + "name": "Perf Runs (JMeter)", "target": { "limit": 100, "matchAny": false, - "tags": ["perf-iac"], + "tags": ["perf-iac", "jmeter"], + "type": "tags" + }, + "type": "tags" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(15, 122, 102)", + "name": "Perf Runs (Locust)", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac", "locust"], "type": "tags" }, "type": "tags" diff --git a/docker/telemetry/grafana/dashboards/transaction-overview.json b/docker/telemetry/grafana/dashboards/transaction-overview.json index f5e6820a81..c637f172e6 100644 --- a/docker/telemetry/grafana/dashboards/transaction-overview.json +++ b/docker/telemetry/grafana/dashboards/transaction-overview.json @@ -21,11 +21,28 @@ "enable": true, "hide": false, "iconColor": "rgb(70, 70, 70)", - "name": "Annotate perf-iac runs", + "name": "Perf Runs (JMeter)", "target": { "limit": 100, "matchAny": false, - "tags": ["perf-iac"], + "tags": ["perf-iac", "jmeter"], + "type": "tags" + }, + "type": "tags" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(15, 122, 102)", + "name": "Perf Runs (Locust)", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac", "locust"], "type": "tags" }, "type": "tags" diff --git a/docker/telemetry/grafana/dashboards/validator-health.json b/docker/telemetry/grafana/dashboards/validator-health.json index 15254cb9ca..603ef3cda9 100644 --- a/docker/telemetry/grafana/dashboards/validator-health.json +++ b/docker/telemetry/grafana/dashboards/validator-health.json @@ -21,11 +21,28 @@ "enable": true, "hide": false, "iconColor": "rgb(70, 70, 70)", - "name": "Annotate perf-iac runs", + "name": "Perf Runs (JMeter)", "target": { "limit": 100, "matchAny": false, - "tags": ["perf-iac"], + "tags": ["perf-iac", "jmeter"], + "type": "tags" + }, + "type": "tags" + }, + { + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": false, + "iconColor": "rgb(15, 122, 102)", + "name": "Perf Runs (Locust)", + "target": { + "limit": 100, + "matchAny": false, + "tags": ["perf-iac", "locust"], "type": "tags" }, "type": "tags" diff --git a/docker/telemetry/grafana/provisioning/alerting/rules.yaml b/docker/telemetry/grafana/provisioning/alerting/rules.yaml index 9746559b28..7d909074b2 100644 --- a/docker/telemetry/grafana/provisioning/alerting/rules.yaml +++ b/docker/telemetry/grafana/provisioning/alerting/rules.yaml @@ -480,6 +480,15 @@ groups: # p99 time a job waits in the queue before running. A sustained p99 # above 1s means the node is saturated and work is backing up. `le` must # stay inside the inner sum or histogram_quantile cannot interpolate. + # + # Threshold re-validated after the microsecond ladder was re-cut. Do NOT + # tune it down against a casual reading of this p99: before that change + # the ladder's first edge was 100us with 99.3% of samples beneath it, so + # p99 reported 99.7us -- the bucket edge scaled by the quantile, not a + # latency. Measured cumulative distribution: 99.26% of samples land + # within 100us, 99.969% within 5ms, 99.990% within 100ms, and only + # 0.0022% exceed 1s. So 1s sits about four orders of magnitude above the + # healthy p99 and fires only on genuine saturation, which is the intent. - uid: xrpld-jobqueue-latency-high title: JobQueueLatencyHigh condition: C @@ -544,6 +553,12 @@ groups: # first and explains the others. Measured p99-of-p95 is 37-49ms on # healthy nodes and 488-566ms on nodes that are actively flapping, so # 1000ms flags genuine degradation rather than the current baseline. + # + # Still valid after the millisecond ladder was extended: that change only + # ADDED edges above 5s (2s/3s/4s/10s/30s/60s/120s) and removed none, so + # every edge bracketing this threshold -- 25/50/100/250/500/1000ms -- is + # unchanged and the measurements above still hold. ios_latency's own mean + # is 12.9ms, far below the threshold. - uid: xrpld-nodestore-io-latency-high title: NodeStoreIOLatencyHigh condition: C diff --git a/docker/telemetry/otel-collector-config.grafanacloud.yaml b/docker/telemetry/otel-collector-config.grafanacloud.yaml index 5f5f88d4e1..88f5a97a43 100644 --- a/docker/telemetry/otel-collector-config.grafanacloud.yaml +++ b/docker/telemetry/otel-collector-config.grafanacloud.yaml @@ -139,6 +139,7 @@ processors: - context: datapoint statements: - set(attributes["service_instance_id"], resource.attributes["service.instance.id"]) + - set(attributes["xrpl_node_id"], resource.attributes["xrpl.node.id"]) - set(attributes["deployment_environment"], resource.attributes["deployment.environment"]) - set(attributes["xrpl_network_type"], resource.attributes["xrpl.network.type"]) @@ -159,6 +160,7 @@ connectors: # series from distinct nodes/tiers grouped separately. resource_metrics_key_attributes: - service.instance.id + - xrpl.node.id - deployment.environment - xrpl.network.type histogram: diff --git a/docker/telemetry/otel-collector-config.yaml b/docker/telemetry/otel-collector-config.yaml index c6bc3a2804..ddc3b93ff0 100644 --- a/docker/telemetry/otel-collector-config.yaml +++ b/docker/telemetry/otel-collector-config.yaml @@ -136,6 +136,7 @@ connectors: # nodes/tiers grouped separately. resource_metrics_key_attributes: - service.instance.id + - xrpl.node.id - deployment.environment - xrpl.network.type histogram: diff --git a/docker/telemetry/workload/baselines/baseline-timings.json b/docker/telemetry/workload/baselines/baseline-timings.json index 4784953609..4e9633568b 100644 --- a/docker/telemetry/workload/baselines/baseline-timings.json +++ b/docker/telemetry/workload/baselines/baseline-timings.json @@ -1,23 +1,8 @@ { + "_note": "job.* entries were removed on 2026-08-21. They were captured against the old microsecond ladder whose first edge was 100us, with 99.3% of job_queued_us samples beneath it, so job.acceptLedger.queued.p95 = 96.79us was 0.95/0.9926 x 100 -- arithmetic on the bucket edge, not a latency. Recapture them on a node running the re-cut ladder (floor 1us); until then the comparer reports them as \"new metric (not in baseline)\" and gates only the span metrics, which are unaffected. Removed values, for reference: job.acceptLedger.queued.p95=96.79us, job.acceptLedger.running.p95=10562.50us, job.transaction.queued.p95=478.97us, job.transaction.running.p95=494.14us.", "captured_at": "2026-06-05T18:41:52Z", "git_sha": "fd1c8c6060f7a15cc9e65b16f99629d9ab7ac7dc", "metrics": { - "job.acceptLedger.queued.p95": { - "unit": "us", - "value": 96.78571428571428 - }, - "job.acceptLedger.running.p95": { - "unit": "us", - "value": 10562.499999999945 - }, - "job.transaction.queued.p95": { - "unit": "us", - "value": 478.96551724137925 - }, - "job.transaction.running.p95": { - "unit": "us", - "value": 494.1361256544502 - }, "span.consensus.accept.p50": { "unit": "ms", "value": 1.059405940594059 diff --git a/docker/telemetry/workload/regression-thresholds.json b/docker/telemetry/workload/regression-thresholds.json index ae6789f1ba..0dba5b6845 100644 --- a/docker/telemetry/workload/regression-thresholds.json +++ b/docker/telemetry/workload/regression-thresholds.json @@ -1,6 +1,6 @@ { "_description": "Per-metric regression thresholds. A metric regresses when current - baseline exceeds BOTH the percentage and absolute bounds (AND, not OR — this tolerates small-value noise). Defaults apply unless a per-metric override exists.", - "_bucket_note": "SpanMetrics latency histograms use explicit buckets [1,5,10,25,50,100,250,500,1000,2000,3000,4000,5000,10000,30000]ms. A quantile sitting near a low-end boundary can jump a full bucket (e.g. 1ms->5ms) between runs with no real change, so absolute span bounds are set to ~2 low-end bucket widths (10ms) to tolerate that quantization noise while still catching genuine multi-bucket regressions. Second-scale consensus spans now have 2s/3s/4s boundaries (previously all fell in one 1s-5s bucket); their quantiles quantize to ~1s widths there. The job_queue running bound is widened similarly — per-ledger apply work scales with TxQ burst load.", + "_bucket_note": "SpanMetrics latency histograms use explicit buckets [0.01,0.05,0.1,0.25,0.5,1,5,10,25,50,100,250,500]ms then [1,2,3,4,5,10,30]s (20 edges; docker/telemetry/otel-collector-config.yaml is the authoritative list). An earlier version of this note claimed 15 edges starting at 1ms and justified the 10ms absolute span bound as \"~2 low-end bucket widths\" — that derivation is void, because the sub-millisecond edges make the low-end bucket width 0.01ms, not 5ms. The 10ms bound is retained on its own merit: it is roughly two bucket widths in the 5-25ms band where most span quantiles actually sit, so it still absorbs single-bucket quantization jitter while catching multi-bucket regressions. Second-scale consensus spans have 2s/3s/4s boundaries, so their quantiles quantize to ~1s widths there. The job_queue running bound is widened similarly — per-ledger apply work scales with TxQ burst load. NOTE: the native job_queue histograms are microsecond-valued and their ladder was re-cut (floor 100us → 1us), so any job_queue baseline captured before that change is an interpolation artefact, not a latency.", "defaults": { "span": { "p50": { "max_pct_increase": 50.0, "max_abs_increase_ms": 10.0 }, diff --git a/docker/telemetry/workload/validate_telemetry.py b/docker/telemetry/workload/validate_telemetry.py index 0fea028ede..2862c6df68 100644 --- a/docker/telemetry/workload/validate_telemetry.py +++ b/docker/telemetry/workload/validate_telemetry.py @@ -35,6 +35,7 @@ import asyncio import fnmatch import json import logging +import re import sys import time from dataclasses import dataclass, field @@ -43,6 +44,11 @@ from typing import Any import aiohttp +# Loki's default query window is the last hour. A validation run finishes in +# minutes, but bounding the range explicitly keeps the query reproducible when +# someone re-runs it later to investigate a result. +LOG_QUERY_WINDOW_SECONDS = 4 * 60 * 60 + logger = logging.getLogger("validate_telemetry") # --------------------------------------------------------------------------- @@ -175,6 +181,19 @@ class ValidationReport: # --------------------------------------------------------------------------- +def _log_query_window() -> dict[str, str]: + """Loki query_range bounds covering a validation run. + + Returns: + start/end parameters in nanoseconds since the epoch. + """ + now = time.time() + return { + "start": str(int((now - LOG_QUERY_WINDOW_SECONDS) * 1_000_000_000)), + "end": str(int(now * 1_000_000_000)), + } + + async def _tempo_search( session: aiohttp.ClientSession, tempo_url: str, @@ -1092,9 +1111,13 @@ async def validate_log_trace_correlation( # Check 1: Any logs with trace_id exist. try: params = { - "query": '{job="xrpld"} |= "trace_id="', + # Loki's OTLP ingestion promotes service.name to the label + # `service_name`. A `job` attribute is structured metadata, which a + # stream selector cannot match — see otel-collector-config.yaml. + "query": '{service_name="xrpld"} |= "trace_id="', "limit": 5, "direction": "backward", + **_log_query_window(), } async with session.get( f"{loki_url}/loki/api/v1/query_range", params=params @@ -1125,56 +1148,79 @@ async def validate_log_trace_correlation( ) ) - # Check 2: Cross-reference a trace_id from Tempo to Loki. + # Check 2: Cross-reference a trace_id from a log line back to Tempo. + # + # Driven from the log side on purpose. A trace_id only reaches a log line + # when that line is emitted inside a sampled span, and at `warning` level + # most spans produce no log output at all — so picking an arbitrary trace + # from Tempo and expecting it in Loki fails even when correlation works. + # Starting from a logged trace_id tests the invariant that matters: an id + # written to a log must resolve to a trace that was actually exported. try: - # Get a recent trace from Tempo. - traces = await _tempo_search( - session, - tempo_url, - '{resource.service.name="xrpld"}', - limit=1, - ) + loki_params = { + "query": '{service_name="xrpld"} |= "trace_id="', + "limit": 5, + "direction": "backward", + **_log_query_window(), + } + async with session.get( + f"{loki_url}/loki/api/v1/query_range", params=loki_params + ) as resp: + data = await resp.json() + streams = data.get("data", {}).get("result", []) - if traces: - trace_id = traces[0].get("traceID", "") - if trace_id: - # Search Loki for this trace_id. - loki_params = { - "query": f'{{job="xrpld"}} |= "{trace_id}"', - "limit": 5, - "direction": "backward", - } - async with session.get( - f"{loki_url}/loki/api/v1/query_range", - params=loki_params, - ) as loki_resp: - loki_data = await loki_resp.json() - loki_streams = loki_data.get("data", {}).get("result", []) - loki_count = sum(len(s.get("values", [])) for s in loki_streams) - report.add( - CheckResult( - name="log.trace_id_cross_reference", - category="log", - passed=loki_count > 0, - message=( - f"trace_id {trace_id[:16]}... found in " - f"{loki_count} Loki entries" - if loki_count > 0 - else f"trace_id {trace_id[:16]}... not found " "in Loki" - ), - details={ - "trace_id": trace_id, - "loki_count": loki_count, - }, - ) - ) - else: + logged_ids = [ + match.group(1) + for stream in streams + for _, line in stream.get("values", []) + if (match := re.search(r"trace_id=([0-9a-f]{32})", line)) + ] + + if not logged_ids: report.add( CheckResult( name="log.trace_id_cross_reference", category="log", passed=False, - message="No traces in Tempo to cross-reference", + message=( + "No logged trace_id to cross-reference. Log lines carry one only " + "when emitted inside a sampled span; raise the log level or widen " + "the workload if this persists." + ), + ) + ) + else: + # Try every id found, not just the first: one unexported trace + # should not fail the check while correlation demonstrably works. + resolved: str | None = None + span_count = 0 + unique_ids = list(dict.fromkeys(logged_ids)) + for candidate in unique_ids: + try: + spans = await _tempo_get_trace(session, tempo_url, candidate) + except Exception: # noqa: BLE001 - a 404 is "not found", not an error + continue + if spans: + resolved, span_count = candidate, len(spans) + break + + report.add( + CheckResult( + name="log.trace_id_cross_reference", + category="log", + passed=resolved is not None, + message=( + f"logged trace_id {resolved[:16]}... resolves to " + f"{span_count} spans in Tempo" + if resolved + else f"none of {len(unique_ids)} logged trace_id(s) resolve in " + "Tempo; the spans they name were not exported" + ), + details={ + "trace_id": resolved, + "span_count": span_count, + "candidates": len(unique_ids), + }, ) ) except Exception as exc: diff --git a/docs/telemetry-runbook.md b/docs/telemetry-runbook.md index c70ae15c69..d4e73914e0 100644 --- a/docs/telemetry-runbook.md +++ b/docs/telemetry-runbook.md @@ -143,8 +143,16 @@ curl -s http://localhost:5015 -d '{"method":"server_info"}' | | `max_queue_size` | `2048` | Max spans queued before dropping | | `use_tls` | `0` | Use TLS for exporter connection | | `tls_ca_cert` | (empty) | Path to CA certificate bundle | -| `tls_client_cert` | (empty) | Client cert (PEM) for mutual TLS; empty = one-way TLS | -| `tls_client_key` | (empty) | Private key (PEM) for `tls_client_cert` | +| `tls_client_cert` | (empty) | Client cert (PEM) for mTLS; empty = one-way. See note | +| `tls_client_key` | (empty) | Private key (PEM) for `tls_client_cert`. See note | + +> **mTLS (mutual TLS) note**: `tls_client_cert` and `tls_client_key` are optional — leaving both empty gives one-way (server-only) TLS. **If either one is set**, `enabled=1` requires both of them **and** `use_tls=1`, or the node exits at startup; see the Troubleshooting entry for `Unable to start ...: [telemetry] ...`. When `enabled=0` they are read but never validated. + +> **Traces and metrics also carry `xrpl.node.id`.** xrpld sets it as a resource +> attribute alongside `service.instance.id`; the value is the node public key +> (base58, begins with `n`). It comes from the node identity unconditionally, so +> it is present even when `[telemetry] service_instance_id` is configured. +> TraceQL filters on it as `resource.xrpl.node.id`. > **`consensus_trace_strategy` is not validated.** The parser copies the raw > string through (`TelemetryConfig.cpp:155-156`) and the only equality test in @@ -2004,6 +2012,55 @@ attributes from their own alloy pipeline. Outside those runs the labels are absent; leaving the filters on **All** keeps every dashboard rendering normally. +### Perf Run Annotations + +Perf load windows are drawn on the dashboards as shaded region annotations +rather than single markers. perf-iac's +`.github/scripts/post_grafana_annotation.sh` opens an annotation when a load +phase starts and closes it with an end time when that phase finishes, so the +shaded band covers exactly the interval over which the load was applied. + +Every dashboard carries two tag-matched annotation layers, one per load driver: + +| Layer | Tags | Color | +| -------------------- | --------------------- | --------- | +| `Perf Runs (JMeter)` | `perf-iac` + `jmeter` | grey | +| `Perf Runs (Locust)` | `perf-iac` + `locust` | dark teal | + +Both layers set `matchAny: false`, so a region is drawn only if it carries +**both** of the layer's tags — the tag list is an AND, not an OR. Grafana tag +matching is a superset AND-match with no negation, so a layer listing only +`perf-iac` would also match every driver region, and "`perf-iac` but neither +driver" cannot be expressed at all. That is why there is no catch-all layer +beside these two: a generic layer could only ever redraw the same regions the +driver layers already show, giving two overlapping bands and two tooltips for +one load window. + +JMeter posts **two** regions per load job, one around the warm-up phase and one +around the measured phase. Locust posts **one**, for the measured phase only, +because it has no warm-up step — so a Locust leg shows a single band where a +JMeter leg shows two. + +The driver tag is not something a run supplies. Each load workflow hardcodes it +as a `LOAD_DRIVER` environment value (`reusable-jmeter-test.yml` sets `jmeter`, +`reusable-locust-test.yml` sets `locust`), so it is never a dispatch input and no +current workflow can omit it; the script warns in CI if one ever does. Alongside +the driver, each region also carries the ticket (work item), the side (`test` or +`baseline`), the ref, the commit, and the phase; blank values are dropped. The +tooltip lists those, which is how one band is told from another when several runs +overlap. The driver is carried only as a tag, not in the tooltip — which layer +drew the band is what identifies it. + +Two rendering limits are worth knowing. Grafana draws annotations only on time +series, state timeline and candlestick panels, so on a board of mostly stats and +gauges most panels show no band. And the shaded fill is rendered at 10% opacity, +so the two drivers' colours are near-identical inside the band; the region's two +full-colour dashed edges and the toolbar toggles are what tell them apart. Both +colours are deliberately muted so a band never competes with the data; the Locust +teal is the darkest step that still separates from the JMeter grey by a readable +margin. The grey itself sits below the 3:1 contrast floor on the dark theme, so +its edges read faint there. + ### Who owns which attribute - **Node and service** come from xrpld config (`service_instance_id`, @@ -2160,7 +2217,7 @@ Requires `trace_peer=1` in the `[telemetry]` config section. | Validated Ledger Age | stat | `ledgermaster_validated_ledger_age` | — | | Published Ledger Age | stat | `ledgermaster_published_ledger_age` | — | | Operating Mode (Time Share) | timeseries | `rate(state_accounting_X_duration) / sum(rate(all modes))` | — | -| Operating Mode Transitions | timeseries | `increase(state_accounting_*_transitions[$__rate_interval])` | — | +| Operating Mode Transitions | timeseries | `round(increase(state_accounting_*_transitions[$__interval]))` (bars, Min step 1m) | — | | I/O Latency | timeseries | `histogram_quantile(0.95, ios_latency_milliseconds_bucket)` | — | | Job Queue Depth | timeseries | `jobq_job_count` | — | | Ledger Fetch Rate | stat | `rate(ledger_fetches[5m])` | — | @@ -2232,22 +2289,68 @@ Requires `trace_peer=1` in the `[telemetry]` config section. | ------------------------- | ---------- | ------------------------------------------------------------- | ----------- | | RPC Request Rate | stat | `rate(rpc_requests[5m])` | — | | RPC Response Time | timeseries | `histogram_quantile(0.95, rpc_time_milliseconds_bucket)` | — | -| RPC Response Size | timeseries | `histogram_quantile(0.95, rpc_size_milliseconds_bucket)` | — | +| RPC Response Size | timeseries | `histogram_quantile(0.95, rpc_size_bytes_bucket)` | — | | RPC Response Time Heatmap | heatmap | `rpc_time_milliseconds_bucket` | — | | Pathfinding Fast Duration | timeseries | `histogram_quantile(0.95, pathfind_fast_milliseconds_bucket)` | — | | Pathfinding Full Duration | timeseries | `histogram_quantile(0.95, pathfind_full_milliseconds_bucket)` | — | | Resource Warnings Rate | stat | `rate(warn_total[$__rate_interval])` | — | | Resource Drops Rate | stat | `rate(drop_total[$__rate_interval])` | — | -> **The `_milliseconds` suffix comes from the exporter, not from xrpld.** These -> histograms are created with unit `"ms"` -> ([OTelCollector.cpp:615](../src/libxrpl/beast/insight/OTelCollector.cpp#L615)), -> so the Prometheus exporter appends the unit to the family name — `rpc_time` -> becomes `rpc_time_milliseconds_bucket`. Querying the bare `rpc_time_bucket`, +> **The unit suffix comes from the exporter, not from xrpld.** Each histogram +> declares a unit, and the Prometheus exporter appends the unit's name to the +> family name — a `ms` instrument like `rpc_time` becomes +> `rpc_time_milliseconds_bucket`. Querying the bare `rpc_time_bucket`, > `ios_latency_bucket` or `pathfind_fast_bucket` returns no data and no error. -> **Known issue**: `rpc_size` counts bytes but shares the same `"ms"` histogram -> constructor, so it is exported as `rpc_size_milliseconds_bucket` — the suffix -> is wrong, the name is nonetheless the one to query. +> +> The unit an `Event` declares also selects its bucket ladder, because the +> histogram views match on unit. `rpc_size` measures bytes, so it declares +> `Unit::Bytes` and exports as **`rpc_size_bytes_bucket`** on the byte ladder. +> It used to share the `ms` constructor and export as +> `rpc_size_milliseconds_bucket` on a latency ladder — if you find that name in +> an old query or bookmark, it no longer exists. + +#### Reading A Histogram Percentile + +Two failure modes make a percentile panel lie, and neither looks like an error — +both produce a believable number. Check for them before trusting any p95/p99. + +**Saturated at the top.** If the quantile falls in the `+Inf` bucket, Prometheus +returns the **second-highest** bucket edge, not `+Inf`. A panel pinned to a round +number that happens to equal the ladder's top edge is the signature. Confirm by +comparing the top finite bucket against the total: + +```promql +1 - ( + sum(last_over_time(_bucket{le=""}[15m])) + / sum(last_over_time(_bucket{le="+Inf"}[15m])) +) +``` + +A non-trivial result means samples are being censored and the percentile is a +lower bound, not a measurement. + +**Saturated at the bottom.** If nearly every sample lands in the first bucket, +`histogram_quantile` interpolates _inside_ it and returns +`quantile / fraction_in_bucket_0 × first_edge`. The signature is a p75/p95/p99 +that sit in near-constant proportion to each other and to the first edge — for +example 75.5 / 95.7 / 99.7 against a 100 µs floor. Confirm with: + +```promql +sum(last_over_time(_bucket{le=""}[15m])) +/ sum(last_over_time(_bucket{le="+Inf"}[15m])) +``` + +Anything close to 1 means the panel is reporting arithmetic on the bucket edge. + +**After a ladder change, expect a discontinuity.** Existing series keep their old +`le` values, so a percentile panel shows a step at the restart that introduced +new edges. That break is the ladder changing, not an incident. + +Bucket edges for the native instruments live in one place — +[`include/xrpl/telemetry/HistogramBuckets.h`](../include/xrpl/telemetry/HistogramBuckets.h). +The millisecond ladder is required to contain every representable edge of the +collector's spanmetrics ladder; `.github/scripts/telemetry/check_bucket_parity.py` +enforces that in CI, because the two silently drifted once already. ### Span → Metric → Dashboard Summary @@ -3378,6 +3481,30 @@ not a sign the cache is working. - Check firewall rules for ports 4317/4318 - If using TLS, verify certificate path with `tls_ca_cert` +### Node exits at startup with `Unable to start ...: [telemetry] ...` + +- Symptom: the process exits immediately with a non-zero status (255 on POSIX) + — a clean exit, not a crash — after printing that line on stderr. Any + exception thrown while the `Application` object is constructed prints the same + `Unable to start` prefix, so confirm the text after the colon begins with + `[telemetry]` before using this entry +- Cause: the `[telemetry]` mTLS keys (`tls_client_cert` and `tls_client_key`) + contradict each other. Only these two mTLS checks are gated on `enabled=1`; + the rest of the section is still read when telemetry is off, so a malformed + value in any key — including `enabled` itself, which is read before the gate + — still fails startup with a different message +- Fix: the two checks need different remedies, and the printed message says + which one fired + - `tls_client_cert and tls_client_key must be set together` — exactly one of + the two paths is set. Either delete the one that is set, or add the missing + one **and** set `use_tls=1`. Unless `use_tls=1` is already set, adding the + missing path on its own just moves the failure to the second check + - `tls_client_cert/tls_client_key require use_tls=1` — both paths are set but + TLS is off. Either set `use_tls=1`, or delete **both** paths. Deleting only + one of them trips the first check + - If you did not mean to enable telemetry at all, set `enabled=0` — that + clears both checks whichever one fired + ### No trace_id in log output - Verify xrpld was built with `telemetry=ON` (the `XRPL_ENABLE_TELEMETRY` preprocessor flag) diff --git a/include/xrpl/beast/insight/Collector.h b/include/xrpl/beast/insight/Collector.h index 9da2a8bb74..c4ba20e3f4 100644 --- a/include/xrpl/beast/insight/Collector.h +++ b/include/xrpl/beast/insight/Collector.h @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -30,6 +31,29 @@ public: virtual ~Collector() = 0; + /** + * Called once the services that hook handlers read are constructed. + * + * Implementations that poll their producers must not do so before this: + * hook handlers read live application state. Default is a no-op, for + * collectors that only push. + */ + virtual void + onCollectionReady() + { + } + + /** + * Called before those services are shut down. + * + * Polling must have stopped by the time this returns. Paired with + * onCollectionReady(). + */ + virtual void + onCollectionStopping() + { + } + /** * Create a hook. * @@ -80,6 +104,24 @@ public: virtual Event makeEvent(std::string const& name) = 0; + /** + * Create an event whose samples measure `unit` rather than milliseconds. + * + * The default delegates to the millisecond overload, so a collector that + * cannot act on a unit keeps working unchanged -- the StatsD collector + * relies on this. Collectors that map a unit onto an export format, such + * as the OTel collector, override it. + * + * @param name Metric name, already prefixed if it came through a Group. + * @param unit What the samples measure. + */ + virtual Event + makeEvent(std::string const& name, Unit unit) + { + (void)unit; + return makeEvent(name); + } + Event makeEvent(std::string const& prefix, std::string const& name) { @@ -87,6 +129,14 @@ public: return makeEvent(name); return makeEvent(prefix + "." + name); } + + Event + makeEvent(std::string const& prefix, std::string const& name, Unit unit) + { + if (prefix.empty()) + return makeEvent(name, unit); + return makeEvent(prefix + "." + name, unit); + } /** @} */ /** diff --git a/include/xrpl/beast/insight/Event.h b/include/xrpl/beast/insight/Event.h index c3ff1a8877..9640e2c1b4 100644 --- a/include/xrpl/beast/insight/Event.h +++ b/include/xrpl/beast/insight/Event.h @@ -51,6 +51,24 @@ public: impl_->notify(ceil(value)); } + /** + * Push a raw integral sample. + * + * For Events whose unit is not a duration, such as a byte count. The + * value is stored in the same integral field the duration overload uses + * and is interpreted per the Event's unit by the backend. + * + * Prefer this over constructing an `Event::value_type` at the call site: + * wrapping a byte count in a `std::chrono::milliseconds` compiles, but + * reads as a duration to everything downstream. + */ + void + notify(std::uint64_t value) const + { + if (impl_) + impl_->notify(value_type{value}); + } + [[nodiscard]] std::shared_ptr const& impl() const { diff --git a/include/xrpl/beast/insight/EventImpl.h b/include/xrpl/beast/insight/EventImpl.h index ede649d195..aa2298150f 100644 --- a/include/xrpl/beast/insight/EventImpl.h +++ b/include/xrpl/beast/insight/EventImpl.h @@ -1,5 +1,7 @@ #pragma once +#include + #include #include @@ -10,11 +12,48 @@ class Event; class EventImpl : public std::enable_shared_from_this { public: + /** + * The integral type every sample is stored as. + * + * Named for the common case -- durations -- and deliberately left as a + * duration type. Widening it would change the wire value of every + * existing StatsD timer, and metrics that need finer resolution than a + * whole millisecond use the OTel-native microsecond instruments instead. + * A sample whose unit() is not a duration is carried in the same integral + * field and interpreted per unit() by the backend. + */ using value_type = std::chrono::milliseconds; virtual ~EventImpl() = 0; virtual void notify(value_type const& value) = 0; + + /** + * @brief What this Event's samples measure. Fixed at construction. + * + * The OTel backend reads this to choose the instrument's declared unit + * and, through that, its bucket ladder. The StatsD backend ignores it. + */ + [[nodiscard]] Unit + unit() const noexcept + { + return unit_; + } + +protected: + /** + * @param unit What the samples measure. Defaults to milliseconds so + * existing implementations keep their behaviour unchanged. + */ + explicit EventImpl(Unit unit = Unit::Millis) : unit_(unit) + { + } + +private: + /** + * What the samples measure; selects the export unit and bucket ladder. + */ + Unit unit_; }; } // namespace beast::insight diff --git a/include/xrpl/beast/insight/Unit.h b/include/xrpl/beast/insight/Unit.h new file mode 100644 index 0000000000..cd9c863d1e --- /dev/null +++ b/include/xrpl/beast/insight/Unit.h @@ -0,0 +1,94 @@ +#pragma once + +#include + +namespace beast::insight { + +/** + * @brief What an Event's samples measure. + * + * `Event` documents itself as carrying "a millisecond time, or other integral + * value", but both backends used to assume the first case: the OTel bridge + * declared every instrument with unit `ms`, and StatsD tagged every sample + * `|ms`. A size metric therefore exported under a `_milliseconds` name and + * inherited a latency bucket ladder, which censored a quarter of its samples + * and pinned its p95 to a constant. + * + * Naming the unit at creation time is what lets the OTel bridge pick both the + * instrument unit and the matching bucket ladder: + * + * makeEvent("time", Unit::Millis) --> OTel unit "ms" --> millisecond ladder + * makeEvent("size", Unit::Bytes) --> OTel unit "By" --> byte ladder + * + * The StatsD backend deliberately ignores this and keeps emitting `|ms` for + * every Event. That path is retired here -- its UDP port is commented out of + * the compose file and the integration test fails if anything is listening on + * 8125 -- so changing its wire format would alter a legacy contract for no + * local benefit and with no way to verify it. + * + * @note Adding a member requires extending otelUnitCode(), which switches + * exhaustively so a new member is a compile error rather than a silent + * fallthrough to milliseconds. + */ +enum class Unit : std::uint8_t { + /** + * Whole milliseconds. The default, and what every duration Event uses. + */ + Millis, + + /** + * A byte count, such as a serialized response size. + */ + Bytes +}; + +/** + * @brief The OTel (UCUM) unit code for a Unit. + * + * The collector's Prometheus exporter derives the exported metric-name suffix + * from this code, so `ms` yields `_milliseconds` and `By` yields `_bytes`. It + * is also the key the histogram views match on, which is how each unit gets + * its own bucket ladder. + * + * @param unit The unit to translate. + * @return A static, null-terminated UCUM code. + */ +constexpr char const* +otelUnitCode(Unit unit) noexcept +{ + switch (unit) + { + case Unit::Bytes: + return "By"; + case Unit::Millis: + break; + } + return "ms"; +} + +/** + * @brief Human-readable description for an instrument of this unit. + * + * Exported alongside the metric, so this is the text an operator reads in a + * metric catalogue. A byte-valued instrument that describes itself as a + * duration is exactly the confusion this whole type exists to remove, so the + * description is derived from the unit rather than written out at each + * instrument site. + * + * @param unit The unit to describe. + * @return A static, null-terminated description. + */ +constexpr char const* +otelUnitDescription(Unit unit) noexcept +{ + switch (unit) + { + case Unit::Bytes: + return "Size in bytes"; + case Unit::Millis: + break; + } + return "Duration in ms"; +} + +} // namespace beast::insight diff --git a/include/xrpl/server/Wallet.h b/include/xrpl/server/Wallet.h index 95486cc468..af6c92b83d 100644 --- a/include/xrpl/server/Wallet.h +++ b/include/xrpl/server/Wallet.h @@ -16,6 +16,7 @@ #include #include +#include #include #include #include @@ -88,6 +89,19 @@ addValidatorManifest(soci::session& session, std::string const& serialized); void clearNodeIdentity(soci::session& session); +/** + * Returns this node's stored keypair, if the database holds a valid one. + * + * Read-only: unlike getNodeIdentity(), never generates or persists a key. A row + * counts only when its public and secret keys are a pair. + * + * @param session Session with the database. + * + * @return The stored keypair, or std::nullopt. + */ +std::optional> +readNodeIdentity(soci::session& session); + /** * Returns a stable public and private key for this node. * diff --git a/include/xrpl/telemetry/HistogramBuckets.h b/include/xrpl/telemetry/HistogramBuckets.h new file mode 100644 index 0000000000..6a00417961 --- /dev/null +++ b/include/xrpl/telemetry/HistogramBuckets.h @@ -0,0 +1,244 @@ +#pragma once + +#include +#include +#include +#include + +namespace xrpl::telemetry::buckets { + +/** + * @file HistogramBuckets.h + * @brief Explicit histogram bucket edges for xrpld's OTel instruments. + * + * One header owns every ladder so a reviewer sees all of them at once and a + * test can assert their invariants. Before this existed the edges lived as + * file-local `namespace {}` constants, unreachable from any test, and they + * drifted apart. + * + * Why a ladder is worth this much care: when a quantile falls in the `+Inf` + * bucket, Prometheus returns the *second-highest* edge, not `+Inf`. A + * saturated histogram therefore reports a believable constant instead of an + * obvious error. The same trap exists at the bottom -- if nearly every + * sample lands in bucket 0, `histogram_quantile` interpolates inside it and + * invents a value. A ladder is correct only when its floor sits below the + * mass of the distribution and its ceiling above the tail. + * + * sample --> [ SDK lower_bound over edges ] --> per-bucket counter + * | | + * edges come from v + * THIS header OTLP export + * | + * v + * histogram_quantile() in Grafana + * + * Ladders are `std::array` so they are constant-initialised and + * usable in a `static_assert`. The OTel SDK wants `std::vector` in + * its aggregation config, so call toVector() at the registration site + * rather than storing vectors here. + * + * Example -- register a view with the millisecond ladder: + * @code + * auto config = std::make_shared(); + * config->boundaries_ = buckets::toVector(buckets::kMillisecondBuckets); + * @endcode + * + * Example -- the edge case that motivated a second ladder. An Event whose + * samples are sizes rather than durations must not borrow a latency ladder, + * or a quarter of its samples land in `+Inf` and every quantile reads back + * as the top edge: + * @code + * config->boundaries_ = buckets::toVector(buckets::kByteBuckets); + * @endcode + * + * @note Thread safety: every member is `constexpr` and immutable, so + * reading them from any thread is safe. toVector() allocates and is + * meant for start-up registration paths, never for a record path. + * @note Limitation: changing a ladder changes the exported series count and + * ends bucket comparability across the change -- existing series keep + * their old `le` values, so panels show a break at restart. Grafana + * Cloud bills per series, so re-measure the series count after any + * edit here. + */ + +/** + * Bucket edges, in milliseconds, for whole-millisecond `beast::insight` + * Events: job queue wait and run times, io latency, RPC time, pathfinding. + * + * **This list must contain every representable edge of the collector's + * spanmetrics ladder, and may extend above it.** Agreement over the shared + * range is deliberate: it lets a span-derived latency panel and a native + * histogram panel be read on the same scale. It was specified that way + * originally, then silently broken when the collector ladder alone was + * extended, which left this side capped at 5 s while spans reached 30 s and + * censored every quantile above 5 s. `check_bucket_parity.py` now enforces + * the containment -- add a collector edge, add it here too. + * + * The sub-millisecond edges the collector carries (0.01 to 0.5 ms) are + * deliberately absent. `beast::insight::Event` rounds every duration up to + * a whole millisecond before it reaches the histogram, so those edges would + * collect nothing. Metrics that genuinely need finer resolution belong on + * the microsecond ladder, on the OTel-native path. + * + * The 60 s and 120 s edges exceed the collector's 30 s top on purpose, + * because jobs outlive spans: the updatepaths job type was measured + * averaging about 60 s, so a 30 s ceiling would censor its quantiles just + * as 5 s censors them today. All these Events share one ladder, so its + * ceiling has to cover the slowest member rather than the typical one. + * + * The 2, 3 and 4 s edges resolve second-scale work that previously had to + * interpolate across a single four-second-wide bucket. + */ +inline constexpr std::array kMillisecondBuckets{ + 1.0, + 5.0, + 10.0, + 25.0, + 50.0, + 100.0, + 250.0, + 500.0, + 1'000.0, + 2'000.0, + 3'000.0, + 4'000.0, + 5'000.0, + 10'000.0, + 30'000.0, + 60'000.0, + 120'000.0}; + +/** + * Bucket edges, in bytes, for `beast::insight` Events whose samples are + * sizes rather than durations. Currently only the RPC response size. + * + * Placed from the measured distribution rather than from a guess about how + * large a response could theoretically be. Measured over 24 h: mean 2131 B, + * half of all responses under 1 kB, three quarters under 5 kB. The tail + * above 5 kB has a mean of at most 7538 B, which bounds p99 near 80 kB and + * p99.75 below 256 kB. + * + * So the resolution belongs between 512 B and 64 kB, where the + * distribution actually turns, and two further edges are ample headroom. + * Spending edges at the megabyte scale would cost cardinality on a range + * nothing measured occupies. If a genuinely multi-megabyte response ever + * shows up in the top bucket, extend this -- but extend it on evidence. + */ +inline constexpr std::array kByteBuckets{ + 512.0, + 1'024.0, + 2'048.0, + 4'096.0, + 8'192.0, + 16'384.0, + 32'768.0, + 65'536.0, + 262'144.0, + 1'048'576.0}; + +/** + * Bucket edges, in microseconds, for the OTel-native duration instruments + * created directly on MetricsRegistry: job queue wait and run times, RPC + * method latency, and GetObject lookup latency. + * + * The edges from 1 to 1000 us are the ones that matter most. An earlier + * version of this ladder started at 100 us, which sat ABOVE the mass of every + * instrument using it: 99.3% of job_queued_us samples, 92.5% of + * job_running_us and 90.4% of getobject_lookup_us fell in that first bucket. + * `histogram_quantile` then interpolated inside bucket 0 and returned the + * boundary scaled by the requested quantile -- p75/p95/p99 of job_queued_us + * read 75.5/95.7/99.7 us, which is arithmetic on the bucket edge, not a + * latency. A warm nodestore read is around 1.5 us, so single-microsecond + * resolution is not excessive here. + * + * The upper edges reach a minute so multi-second stalls stay measurable. The + * SDK's own default ladder stops at 10,000, which every one of these + * instruments exceeds during catch-up. + */ +inline constexpr std::array kMicrosecondBuckets{ + 1.0, + 2.0, + 5.0, + 10.0, + 25.0, + 50.0, + 100.0, + 250.0, + 500.0, + 1'000.0, + 5'000.0, + 25'000.0, + 100'000.0, + 500'000.0, + 1'000'000.0, + 5'000'000.0, + 10'000'000.0, + 30'000'000.0, + 60'000'000.0}; + +/** + * Bucket edges for the GetObject request object count. + * + * Counts run from 1 to the hard reply cap (kHardMaxReplyNodes, 12288). The + * honest sync path asks for at most 8 objects, so the low edges are + * fine-grained; the upper ones follow the charge size bands up to the cap. + * Because the top edge IS the hard cap, this ladder cannot saturate. + */ +inline constexpr std::array + kObjectCountBuckets{1.0, 2.0, 4.0, 8.0, 16.0, 64.0, 256.0, 1'024.0, 4'096.0, 12'288.0}; + +/** + * Bucket edges for the GetObject resource charge. + * + * Charges span 0 (the free tier) to roughly 99k for a full-size all-miss + * request. The edges bracket the two thresholds that decide a peer's fate -- + * the warning threshold at 5000 and the drop threshold at 25000 -- so a + * dashboard can show how close charges run to each. + */ +inline constexpr std::array + kChargeBuckets{0.0, 100.0, 500.0, 1'000.0, 5'000.0, 10'000.0, 25'000.0, 50'000.0, 100'000.0}; + +/** + * @brief Check that a ladder is strictly ascending and non-negative. + * + * The SDK places a sample with `std::lower_bound` over the edges, which + * silently misbuckets when edges repeat or descend. Checking at compile + * time makes that class of typo impossible to ship. + * + * @param ladder Bucket upper bounds to check. + * @return true when the ladder is non-empty, starts at or above zero, and + * every later edge is strictly greater than its predecessor. + */ +constexpr bool +isAscendingNonNegative(std::span ladder) noexcept +{ + if (ladder.empty() || ladder.front() < 0.0) + return false; + + for (std::size_t i = 1; i < ladder.size(); ++i) + { + if (!(ladder[i] > ladder[i - 1])) + return false; + } + return true; +} + +static_assert(isAscendingNonNegative(kMillisecondBuckets)); +static_assert(isAscendingNonNegative(kByteBuckets)); +static_assert(isAscendingNonNegative(kMicrosecondBuckets)); +static_assert(isAscendingNonNegative(kObjectCountBuckets)); +static_assert(isAscendingNonNegative(kChargeBuckets)); + +/** + * @brief Copy a ladder into the `std::vector` the OTel SDK wants. + * + * @param ladder Bucket upper bounds. + * @return A vector holding the same edges in the same order. + */ +inline std::vector +toVector(std::span ladder) +{ + return std::vector(ladder.begin(), ladder.end()); +} + +} // namespace xrpl::telemetry::buckets diff --git a/include/xrpl/telemetry/SpanNames.h b/include/xrpl/telemetry/SpanNames.h index b848f96c03..d9f2fb0f0b 100644 --- a/include/xrpl/telemetry/SpanNames.h +++ b/include/xrpl/telemetry/SpanNames.h @@ -108,6 +108,7 @@ inline constexpr auto consensus = makeStr("consensus"); inline constexpr auto peer = makeStr("peer"); inline constexpr auto ledger = makeStr("ledger"); inline constexpr auto network = makeStr("network"); +inline constexpr auto node = makeStr("node"); inline constexpr auto link = makeStr("link"); } // namespace seg @@ -117,6 +118,16 @@ namespace attr { inline constexpr auto networkId = join(join(seg::xrpl, seg::network), makeStr("id")); inline constexpr auto networkType = join(join(seg::xrpl, seg::network), makeStr("type")); +/** + * Resource attribute `xrpl.node.id` — the node's base58 public key. + * + * Dotted form, like its siblings above, because it is a process-identity + * value stamped once on the OTel resource rather than a per-span attribute. + * It gives traces and metrics a stable per-node key alongside + * `service.instance.id`. + */ +inline constexpr auto nodeId = join(join(seg::xrpl, seg::node), makeStr("id")); + /** * Canonical shared attrs (rule 5 — _ underscore form). * diff --git a/include/xrpl/telemetry/Telemetry.h b/include/xrpl/telemetry/Telemetry.h index b9ca1dca64..33fd94ce33 100644 --- a/include/xrpl/telemetry/Telemetry.h +++ b/include/xrpl/telemetry/Telemetry.h @@ -83,7 +83,8 @@ * * @note Thread safety: The Telemetry interface is safe for concurrent reads * (isEnabled, shouldTrace*, getTracer, startSpan) after start() completes. - * setServiceInstanceId() must be called before start() and is not thread-safe. + * setServiceInstanceId() and setNodeId() must be called before start() and + * are not thread-safe. * The OTel SDK's TracerProvider and Tracer are internally thread-safe. */ @@ -193,6 +194,14 @@ public: */ std::string serviceInstanceId; + /** + * OTel resource attribute `xrpl.node.id`: the node's base58-encoded + * public key. Always the node identity, never config-supplied, so it + * stays a stable per-node key even when serviceInstanceId is + * overridden by [telemetry] service_instance_id. + */ + std::string nodeId; + /** * OTLP/HTTP endpoint URL where spans are sent. */ @@ -313,6 +322,24 @@ public: (void)id; } + /** + * Update the node ID (OTel resource attribute `xrpl.node.id`). + * + * Must be called before start(). A setter is needed for the same reason + * setServiceInstanceId() needs one: the node public key is not available + * when Telemetry is constructed (during the ApplicationImp member + * initializer list), so Application::setup() injects it once + * nodeIdentity_ is known. + * + * @param id The node's base58-encoded public key. + */ + virtual void + setNodeId(std::string const& id) + { + // Default no-op for NullTelemetry implementations. + (void)id; + } + /** * Initialize the tracing pipeline (exporter, processor, provider). * Call after construction. @@ -436,8 +463,10 @@ public: /** * Create a Telemetry instance. * - * Returns a TelemetryImpl when setup.enabled is true, or a - * NullTelemetry no-op stub otherwise. + * With XRPL_ENABLE_TELEMETRY defined, returns a TelemetryImpl when + * setup.enabled is true, or a no-op stub otherwise. Without it, the only + * definition of this factory always returns the no-op stub and never reads + * setup.enabled. * * @param setup Configuration from the [telemetry] config section. * @param journal Journal for log output during initialization. @@ -454,6 +483,14 @@ makeTelemetry(Telemetry::Setup const& setup, beast::Journal journal); * @param networkId Network identifier from [network_id] config * (0 = mainnet, 1 = testnet, 2 = devnet). * @return A populated Setup struct with defaults for missing values. + * @throws std::runtime_error If `enabled` is set and the mutual TLS (mTLS) + * settings contradict each other: only one of `tls_client_cert`/`tls_client_key` + * is given, or a client certificate is given while `use_tls` is 0. Those two + * checks are skipped when `enabled` is 0. + * @throws boost::bad_lexical_cast If any numeric key (`enabled`, `use_tls`, + * `batch_size`, the trace switches, ...) holds a value Section::valueOr cannot + * convert. None of the numeric reads sit inside the `enabled` branch, so this + * escapes whether telemetry is on or off. */ Telemetry::Setup makeTelemetrySetup( diff --git a/src/libxrpl/beast/insight/Groups.cpp b/src/libxrpl/beast/insight/Groups.cpp index 6a60c75aa2..a3126d1d75 100644 --- a/src/libxrpl/beast/insight/Groups.cpp +++ b/src/libxrpl/beast/insight/Groups.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -56,12 +57,25 @@ public: return collector_->makeCounter(makeName(name)); } + using Collector::makeEvent; + Event makeEvent(std::string const& name) override { return collector_->makeEvent(makeName(name)); } + // Forwards the unit as well as the prefixed name. Without this override + // the base-class default would delegate to the single-argument overload + // above and silently drop the unit, which is how a byte-valued Event ends + // up declared as milliseconds -- call sites reach a collector through a + // Group, so this is the hop that actually matters. + Event + makeEvent(std::string const& name, Unit unit) override + { + return collector_->makeEvent(makeName(name), unit); + } + Gauge makeGauge(std::string const& name) override { diff --git a/src/libxrpl/beast/insight/NullCollector.cpp b/src/libxrpl/beast/insight/NullCollector.cpp index 03a12ee498..f5b444b3d3 100644 --- a/src/libxrpl/beast/insight/NullCollector.cpp +++ b/src/libxrpl/beast/insight/NullCollector.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -49,7 +50,15 @@ public: class NullEventImpl : public EventImpl { public: - explicit NullEventImpl() = default; + /** + * @param unit What the samples would measure. Recorded even though + * nothing is collected, so a caller can still read back the + * unit it asked for -- which is what makes the null collector + * usable for testing the unit plumbing. + */ + explicit NullEventImpl(Unit unit = Unit::Millis) : EventImpl(unit) + { + } void notify(value_type const&) override @@ -119,12 +128,20 @@ public: return Counter(std::make_shared()); } + using Collector::makeEvent; + Event makeEvent(std::string const&) override { return Event(std::make_shared()); } + Event + makeEvent(std::string const&, Unit unit) override + { + return Event(std::make_shared(unit)); + } + Gauge makeGauge(std::string const&) override { diff --git a/src/libxrpl/beast/insight/OTelCollector.cpp b/src/libxrpl/beast/insight/OTelCollector.cpp index e28dc60ad2..1d12dd0422 100644 --- a/src/libxrpl/beast/insight/OTelCollector.cpp +++ b/src/libxrpl/beast/insight/OTelCollector.cpp @@ -41,6 +41,7 @@ #include #include #include +#include #include #include @@ -57,7 +58,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -166,10 +169,17 @@ private: /** * @brief OTel-backed implementation of beast::insight::EventImpl. * - * Wraps an OTel Histogram instrument. Each notify() call - * records the duration in milliseconds. Uses explicit bucket boundaries - * matching the SpanMetrics connector configuration: - * [1, 5, 10, 25, 50, 100, 250, 500, 1000, 5000] ms + * Wraps an OTel Histogram instrument. Each notify() call records one + * sample, interpreted per the Event's unit(). + * + * The instrument's declared unit is what selects its bucket ladder: the + * histogram views registered in Telemetry.cpp match on unit, so a `ms` + * instrument gets the millisecond ladder and a `By` instrument the byte + * ladder. The edges themselves live in xrpl/telemetry/HistogramBuckets.h -- + * do not restate them here. An earlier version of this comment listed + * `[1, 5, ..., 1000, 5000] ms` as "matching the SpanMetrics connector"; that + * was true when written and silently became false when the connector's + * ladder was extended, which is why the edges now have one owner. * * Thread safety: OTel Histogram::Record() is thread-safe by specification. */ @@ -181,10 +191,14 @@ public: * formatName() by the collector: lowercase, with `.` and * ` ` mapped to `_` (e.g. "rpc_size"). * @param meter OTel Meter used to create the histogram instrument. + * @param unit What the samples measure. Selects the instrument's + * declared unit, its description, and through the unit the + * bucket ladder a histogram view applies. */ OTelEventImpl( std::string const& name, - opentelemetry::nostd::shared_ptr const& meter); + opentelemetry::nostd::shared_ptr const& meter, + Unit unit); ~OTelEventImpl() override = default; @@ -229,13 +243,9 @@ public: * @param name Export-ready metric name, already run through * formatName() by the collector: lowercase, with `.` * and ` ` mapped to `_`. - * @param meter OTel Meter used to create the observable gauge. * @param collector Owning collector, used to invoke hooks before reads. */ - OTelGaugeImpl( - std::string const& name, - opentelemetry::nostd::shared_ptr const& meter, - std::shared_ptr const& collector); + OTelGaugeImpl(std::string name, std::shared_ptr const& collector); ~OTelGaugeImpl() override; @@ -273,6 +283,25 @@ public: static void gaugeCallback(opentelemetry::metrics::ObserverResult result, void* state); + /** + * Create the observable instrument and register the callback, once. + * + * Called when the collector is told collection is ready, because the + * callback reads live application state. + */ + void + arm(); + + /** + * Remove the callback, so the reader thread stops observing this gauge. + * + * RemoveCallback is synchronous: the SDK guards its callback list and the + * observe pass with the same mutex, so no callback is running once this + * returns. Idempotent. + */ + void + disarm(); + private: /** * Current gauge value, updated atomically by set()/increment(). @@ -280,10 +309,20 @@ private: std::atomic value_{0}; /** - * OTel observable gauge handle (prevents deregistration). + * Export-ready metric name, held until arm() creates the instrument. + */ + std::string const name_; + + /** + * OTel observable gauge handle, null until arm() runs. */ opentelemetry::nostd::shared_ptr gauge_; + /** + * Guards gauge_ against concurrent arm()/disarm(). + */ + std::mutex armMutex_; + /** * Owning collector, used to invoke hooks before reading gauge values. */ @@ -447,9 +486,18 @@ public: Event makeEvent(std::string const& name) override; + Event + makeEvent(std::string const& name, Unit unit) override; + Gauge makeGauge(std::string const& name) override; + void + onCollectionReady() override; + + void + onCollectionStopping() override; + Meter makeMeter(std::string const& name) override; /** @} */ @@ -503,6 +551,13 @@ public: removeGauge(OTelGaugeImpl* gauge); /** @} */ + /** + * @brief The shared Meter, for gauges creating their instrument in arm(). + * @return The Meter this collector resolved at construction. + */ + opentelemetry::nostd::shared_ptr const& + otelMeter() const; + /** * @brief Format a raw metric name for export. * @@ -612,8 +667,10 @@ OTelCounterImpl::increment(value_type amount) OTelEventImpl::OTelEventImpl( std::string const& name, - opentelemetry::nostd::shared_ptr const& meter) - : histogram_(meter->CreateDoubleHistogram(name, "Duration in ms", "ms")) + opentelemetry::nostd::shared_ptr const& meter, + Unit unit) + : EventImpl(unit) + , histogram_(meter->CreateDoubleHistogram(name, otelUnitDescription(unit), otelUnitCode(unit))) { } @@ -627,16 +684,37 @@ OTelEventImpl::notify(value_type const& value) // OTelGaugeImpl //------------------------------------------------------------------------------ -OTelGaugeImpl::OTelGaugeImpl( - std::string const& name, - opentelemetry::nostd::shared_ptr const& meter, - std::shared_ptr const& collector) - : gauge_(meter->CreateInt64ObservableGauge(name)), collector_(collector) +OTelGaugeImpl::OTelGaugeImpl(std::string name, std::shared_ptr const& collector) + : name_(std::move(name)), collector_(collector) { collector_->addGauge(this); +} + +void +OTelGaugeImpl::arm() +{ + // AddCallback arms the SDK reader thread against this gauge, and the + // callback runs hook handlers that read application services. The registry + // does not de-duplicate callbacks, so arm at most once. + std::scoped_lock const lock(armMutex_); + if (gauge_) + return; + + gauge_ = collector_->otelMeter()->CreateInt64ObservableGauge(name_); gauge_->AddCallback(gaugeCallback, this); } +void +OTelGaugeImpl::disarm() +{ + std::scoped_lock const lock(armMutex_); + if (!gauge_) + return; + + gauge_->RemoveCallback(gaugeCallback, this); + gauge_ = nullptr; +} + void OTelGaugeImpl::gaugeCallback(opentelemetry::metrics::ObserverResult result, void* state) { @@ -657,7 +735,8 @@ OTelGaugeImpl::~OTelGaugeImpl() // The SDK's ObservableRegistry guards its callback list and the Observe() // pass with the same mutex, so RemoveCallback cannot return while a // callback for this instrument is in flight — removal is synchronous. - gauge_->RemoveCallback(gaugeCallback, this); + // A no-op when never armed, or already disarmed at shutdown. + disarm(); collector_->removeGauge(this); } @@ -780,13 +859,19 @@ OTelCollectorImp::makeCounter(std::string const& name) Event OTelCollectorImp::makeEvent(std::string const& name) { - return Event(std::make_shared(formatName(name), otelMeter_)); + return makeEvent(name, Unit::Millis); +} + +Event +OTelCollectorImp::makeEvent(std::string const& name, Unit unit) +{ + return Event(std::make_shared(formatName(name), otelMeter_, unit)); } Gauge OTelCollectorImp::makeGauge(std::string const& name) { - return Gauge(std::make_shared(formatName(name), otelMeter_, shared_from_this())); + return Gauge(std::make_shared(formatName(name), shared_from_this())); } Meter @@ -852,6 +937,71 @@ OTelCollectorImp::removeGauge(OTelGaugeImpl* gauge) std::erase(gauges_, gauge); } +void +OTelCollectorImp::onCollectionReady() +{ + // Snapshot under the lock, arm outside it. arm() enters the SDK's + // observable registry lock, and the reader thread takes that lock before + // calling callHooks(), which wants mutex_. callHooks() copies its hook list + // for the same reason. + std::vector gauges; + { + std::scoped_lock const lock(mutex_); + gauges = gauges_; + } + + std::size_t armed = 0; + for (auto* gauge : gauges) + { + // Telemetry must never stop the node, so one bad instrument costs only + // its own metric. + try + { + gauge->arm(); + ++armed; + } + catch (std::exception const& e) + { + if (auto stream = journal_.error()) + { + stream << "OTelCollector: could not register an observable gauge, so that " + "metric will not be exported: " + << e.what(); + } + } + } + + if (auto stream = journal_.info()) + { + stream << "OTelCollector: registered " << armed << " of " << gauges.size() + << " observable gauges"; + } +} + +void +OTelCollectorImp::onCollectionStopping() +{ + // Same lock discipline as onCollectionReady(): snapshot, then act outside + // the lock, because disarm() enters the SDK's observable registry lock. + std::vector gauges; + { + std::scoped_lock const lock(mutex_); + gauges = gauges_; + } + + for (auto* gauge : gauges) + gauge->disarm(); + + if (auto stream = journal_.info()) + stream << "OTelCollector: stopped observing " << gauges.size() << " gauges"; +} + +opentelemetry::nostd::shared_ptr const& +OTelCollectorImp::otelMeter() const +{ + return otelMeter_; +} + std::string OTelCollectorImp::formatName(std::string const& name) { diff --git a/src/libxrpl/beast/insight/StatsDCollector.cpp b/src/libxrpl/beast/insight/StatsDCollector.cpp index dc19aa8953..bc2640ca77 100644 --- a/src/libxrpl/beast/insight/StatsDCollector.cpp +++ b/src/libxrpl/beast/insight/StatsDCollector.cpp @@ -23,6 +23,7 @@ #include #include +#include #include #include #include @@ -218,6 +219,13 @@ private: std::recursive_mutex metricsLock_; List metrics_; + /** + * Whether hook handlers may be called. False until onCollectionReady(), + * because the handlers read application services that are still being + * constructed while this collector exists. + */ + std::atomic polling_{false}; + // Must come last for order of init std::thread thread_; @@ -255,6 +263,22 @@ public: thread_.join(); } + void + onCollectionReady() override + { + polling_.store(true, std::memory_order_release); + } + + void + onCollectionStopping() override + { + polling_.store(false, std::memory_order_release); + + // onTimer holds metricsLock_ across the handler loop, so acquiring it + // here waits for a handler that is already running. + std::scoped_lock const _(metricsLock_); + } + Hook makeHook(HookImpl::HandlerType const& handler) override { @@ -437,12 +461,15 @@ public: return; } - std::scoped_lock const _(metricsLock_); + if (polling_.load(std::memory_order_acquire)) + { + std::scoped_lock const _(metricsLock_); - for (auto& m : metrics_) - m.doProcess(); + for (auto& m : metrics_) + m.doProcess(); - sendBuffers(); + sendBuffers(); + } setTimer(); } diff --git a/src/libxrpl/server/Wallet.cpp b/src/libxrpl/server/Wallet.cpp index 56d0db67d4..92317d40f6 100644 --- a/src/libxrpl/server/Wallet.cpp +++ b/src/libxrpl/server/Wallet.cpp @@ -32,6 +32,7 @@ #include #include #include +#include #include #include #include @@ -147,27 +148,34 @@ clearNodeIdentity(soci::session& session) session << "DELETE FROM NodeIdentity;"; } +std::optional> +readNodeIdentity(soci::session& session) +{ + // SOCI requires boost::optional (not std::optional) as the parameter. + boost::optional pubKO, priKO; + soci::statement st = + (session.prepare << "SELECT PublicKey, PrivateKey FROM NodeIdentity;", + soci::into(pubKO), + soci::into(priKO)); + st.execute(); + while (st.fetch()) + { + auto const sk = parseBase58(TokenType::NodePrivate, priKO.value_or("")); + auto const pk = parseBase58(TokenType::NodePublic, pubKO.value_or("")); + + // Only use if the public and secret keys are a pair + if (sk && pk && (*pk == derivePublicKey(KeyType::Secp256k1, *sk))) + return std::pair{*pk, *sk}; + } + + return std::nullopt; +} + std::pair getNodeIdentity(soci::session& session) { - { - // SOCI requires boost::optional (not std::optional) as the parameter. - boost::optional pubKO, priKO; - soci::statement st = - (session.prepare << "SELECT PublicKey, PrivateKey FROM NodeIdentity;", - soci::into(pubKO), - soci::into(priKO)); - st.execute(); - while (st.fetch()) - { - auto const sk = parseBase58(TokenType::NodePrivate, priKO.value_or("")); - auto const pk = parseBase58(TokenType::NodePublic, pubKO.value_or("")); - - // Only use if the public and secret keys are a pair - if (sk && pk && (*pk == derivePublicKey(KeyType::Secp256k1, *sk))) - return {*pk, *sk}; - } - } + if (auto const stored = readNodeIdentity(session)) + return *stored; // If a valid identity wasn't found, we randomly generate a new one: auto [newpublicKey, newsecretKey] = randomKeyPair(KeyType::Secp256k1); diff --git a/src/libxrpl/telemetry/Telemetry.cpp b/src/libxrpl/telemetry/Telemetry.cpp index 1d3eb13c7f..30c9a22b50 100644 --- a/src/libxrpl/telemetry/Telemetry.cpp +++ b/src/libxrpl/telemetry/Telemetry.cpp @@ -19,10 +19,12 @@ #include #include +#include #include #include #include #include +#include #include #include @@ -66,6 +68,7 @@ #include #include +#include #include #include #include @@ -283,8 +286,8 @@ class TelemetryImpl : public Telemetry { /** * Configuration from the [telemetry] config section. - * Non-const so setServiceInstanceId() can update the instance ID - * before start() creates the OTel resource. + * Non-const so setServiceInstanceId() and setNodeId() can update the + * identity attributes before start() creates the OTel resource. */ Setup setup_; @@ -317,32 +320,53 @@ class TelemetryImpl : public Telemetry */ opentelemetry::nostd::shared_ptr contextStorage_; + /** + * Set by stop(), so a second call does nothing. + */ + bool stopped_{false}; + public: TelemetryImpl(Setup setup, beast::Journal journal) : setup_(std::move(setup)), journal_(journal) { - // Build the metrics pipeline NOW, in the constructor, so the global - // MeterProvider is published before any subsystem is constructed. - // beast::insight instruments are created eagerly in subsystem - // constructors (e.g. LedgerMaster, NetworkOPs, ServerHandler), which - // run during ApplicationImp's member-init list — long before start(). - // opentelemetry-cpp has no proxy MeterProvider, so an instrument - // created before SetMeterProvider() binds to the noop provider forever. - // Tracing does not have this problem because getTracer() is called - // fresh at each span creation (runtime, after start()). + // Publish the MeterProvider before any subsystem is constructed; see + // initMetrics(). setup_.serviceInstanceId is already resolved by the + // caller, so the resource is complete. // - // The metrics resource uses setup_.serviceInstanceId as provided by - // config. A later setServiceInstanceId() (node-key fallback) cannot - // change this immutable resource, so operators relying on the node-key - // identity should set [telemetry] service_instance_id explicitly. - initMetrics(); + // A failure must never stop the node starting: the global provider + // stays noop and every instrument call remains valid. + try + { + initMetrics(); + } + catch (std::exception const& e) + { + JLOG(journal_.error()) << "Telemetry metrics pipeline failed to initialise, " + "continuing without metrics: " + << e.what(); + } } + /** + * Override the service instance id, for callers that learn it late. + * + * Affects only the tracer resource, which start() builds. The metrics + * resource is built by the constructor and is immutable, so supply the id + * through Setup to have it on both. + * + * @param id The instance id to report on spans. + */ void setServiceInstanceId(std::string const& id) override { setup_.serviceInstanceId = id; } + void + setNodeId(std::string const& id) override + { + setup_.nodeId = id; + } + void start() override { @@ -377,14 +401,7 @@ public: auto processor = std::make_unique(std::move(batchProcessor)); // Configure resource attributes - auto resourceAttrs = resource::Resource::Create({ - {opentelemetry::semconv::service::kServiceName, setup_.serviceName}, - {opentelemetry::semconv::service::kServiceVersion, setup_.serviceVersion}, - {opentelemetry::semconv::service::kServiceInstanceId, setup_.serviceInstanceId}, - {std::string(attr::networkId), - static_cast(setup_.networkId)}, // LCOV_EXCL_LINE - {std::string(attr::networkType), setup_.networkType}, // LCOV_EXCL_LINE - }); + auto resourceAttrs = makeTracerResource(); // Configure sampler. Head sampling is fixed at 1.0 (sample everything); // setup_.samplingRatio is not config-driven. Wrap the ratio sampler in a @@ -433,6 +450,56 @@ public: JLOG(journal_.info()) << "Telemetry started successfully"; } + /** + * Build the tracer resource: the process-identity attributes stamped on + * every exported span. + * + * Called from start(), which runs after Application::setup() has injected + * the node identity, so setup_.nodeId is populated by then. + * + * @return The resource attached to the TracerProvider. + */ + [[nodiscard]] resource::Resource + makeTracerResource() const + { + return resource::Resource::Create({ + {opentelemetry::semconv::service::kServiceName, setup_.serviceName}, + {opentelemetry::semconv::service::kServiceVersion, setup_.serviceVersion}, + {opentelemetry::semconv::service::kServiceInstanceId, setup_.serviceInstanceId}, + {std::string(attr::networkId), + static_cast(setup_.networkId)}, // LCOV_EXCL_LINE + {std::string(attr::networkType), setup_.networkType}, // LCOV_EXCL_LINE + {std::string(attr::nodeId), setup_.nodeId}, // LCOV_EXCL_LINE + }); + } + + /** + * Build the metrics resource: the same attributes as the tracer resource + * in start(), so metrics and traces share one identity. + * + * xrpl.node.id is added only when setup_.nodeId already holds a value. + * setNodeId() runs after the constructor that calls this, so on the normal + * startup path the attribute is left off rather than stamped blank. + * + * @return The resource attached to the MeterProvider. + */ + [[nodiscard]] resource::Resource + makeMetricsResource() const + { + resource::ResourceAttributes attrs{ + {opentelemetry::semconv::service::kServiceName, setup_.serviceName}, + {opentelemetry::semconv::service::kServiceVersion, setup_.serviceVersion}, + {opentelemetry::semconv::service::kServiceInstanceId, setup_.serviceInstanceId}, + {std::string(attr::networkId), static_cast(setup_.networkId)}, + {std::string(attr::networkType), setup_.networkType}, + }; + + if (!setup_.nodeId.empty()) + attrs[std::string(attr::nodeId)] = setup_.nodeId; + + return resource::Resource::Create(attrs); + } + /** * Build and publish the metrics pipeline (MeterProvider + periodic * reader + OTLP exporter + histogram view). @@ -442,6 +509,8 @@ public: * during ApplicationImp's member-init list. The metrics resource uses * setup_.serviceInstanceId from config; it is immutable once the provider * is built, so a later node-key setServiceInstanceId() does not affect it. + * The same applies to setNodeId(): xrpl.node.id reaches this resource only + * if setup_.nodeId is already populated when the constructor runs. */ void initMetrics() @@ -479,47 +548,53 @@ public: auto reader = metrics_sdk::PeriodicExportingMetricReaderFactory::Create( std::move(metricExporter), readerOpts); - // Metrics resource: same attributes as the tracer resource so metrics - // and traces share one identity. Built here (not shared with start()) - // because start() runs later; serviceInstanceId comes from config. - auto resourceAttrs = resource::Resource::Create({ - {opentelemetry::semconv::service::kServiceName, setup_.serviceName}, - {opentelemetry::semconv::service::kServiceVersion, setup_.serviceVersion}, - {opentelemetry::semconv::service::kServiceInstanceId, setup_.serviceInstanceId}, - {std::string(attr::networkId), static_cast(setup_.networkId)}, - {std::string(attr::networkType), setup_.networkType}, - }); + auto resourceAttrs = makeMetricsResource(); // Create MeterProvider with the shared resource, then attach reader. meterProvider_ = metrics_sdk::MeterProviderFactory::Create( std::make_unique(), resourceAttrs); meterProvider_->AddMetricReader(std::move(reader)); - // Histogram view: SpanMetrics-compatible bucket boundaries (ms) so - // histogram instruments align with the collector's SpanMetrics. The - // view is created with an EMPTY name so it applies the buckets WITHOUT - // renaming instruments — a non-empty view name would collapse every - // matching histogram (ios_latency, rpc_size, rpc_time, pathfind_*) - // into a single series under that one name. - auto histogramSelector = metrics_sdk::InstrumentSelectorFactory::Create( - metrics_sdk::InstrumentType::kHistogram, "*", "ms"); - // Meter selector MUST match the meter name used by getMeter() and the - // beast OTelCollector (kMeterName = "xrpld"); otherwise this histogram - // view never applies and duration histograms fall back to the SDK - // default boundaries instead of these SpanMetrics-aligned buckets. - auto meterSelector = - metrics_sdk::MeterSelectorFactory::Create(std::string(kMeterName), "", ""); - auto histogramConfig = std::make_shared(); - histogramConfig->boundaries_ = - std::vector{1.0, 5.0, 10.0, 25.0, 50.0, 100.0, 250.0, 500.0, 1000.0, 5000.0}; - auto histogramView = metrics_sdk::ViewFactory::Create( - "", // empty name: keep each instrument's own name, only set buckets - "SpanMetrics-compatible histogram buckets", - metrics_sdk::AggregationType::kHistogram, - histogramConfig); + // One histogram view per unit. The unit is the selector, so an + // instrument gets the ladder that fits what it measures -- a byte + // count no longer inherits a latency ladder. Edges come from + // HistogramBuckets.h, which owns every ladder. + // + // Each view keeps the "*" name pattern and an EMPTY view name: a + // non-empty view name would rename every matching histogram to it and + // collapse them (ios_latency, rpc_size, rpc_time, pathfind_*, and all + // the jobq_* pairs) into a single series. + // + // The meter selector MUST match the meter name used by getMeter() and + // the beast OTelCollector (kMeterName = "xrpld"); otherwise a view + // never applies and instruments fall back to the SDK default ladder, + // whose ceiling is 10,000. + auto const addUnitView = [this]( + std::string const& unitCode, + std::vector boundaries, + std::string const& description) { + auto selector = metrics_sdk::InstrumentSelectorFactory::Create( + metrics_sdk::InstrumentType::kHistogram, "*", unitCode); + auto meterSelector = + metrics_sdk::MeterSelectorFactory::Create(std::string(kMeterName), "", ""); + auto config = std::make_shared(); + config->boundaries_ = std::move(boundaries); + auto view = metrics_sdk::ViewFactory::Create( + "", // empty name: keep each instrument's own name, only set buckets + description, + metrics_sdk::AggregationType::kHistogram, + std::move(config)); + meterProvider_->AddView(std::move(selector), std::move(meterSelector), std::move(view)); + }; - meterProvider_->AddView( - std::move(histogramSelector), std::move(meterSelector), std::move(histogramView)); + addUnitView( + beast::insight::otelUnitCode(beast::insight::Unit::Millis), + buckets::toVector(buckets::kMillisecondBuckets), + "Duration buckets, 1 ms to 120 s"); + addUnitView( + beast::insight::otelUnitCode(beast::insight::Unit::Bytes), + buckets::toVector(buckets::kByteBuckets), + "Size buckets, 512 B to 1 MiB"); // Publish as the global meter provider so developers (and the beast // OTelCollector shim) reach the same pipeline. @@ -530,10 +605,16 @@ public: void stop() override { + if (stopped_) + return; + stopped_ = true; + JLOG(journal_.info()) << "Telemetry stopping"; - // Unregister global instance before tearing down the pipeline. - Telemetry::setInstance(nullptr); + // Unregister global instance before tearing down the pipeline, but only + // if this object is the one that published it. + if (Telemetry::getInstance() == this) + Telemetry::setInstance(nullptr); if (sdkProvider_) { diff --git a/src/libxrpl/telemetry/TelemetryConfig.cpp b/src/libxrpl/telemetry/TelemetryConfig.cpp index d122393e51..11af837c52 100644 --- a/src/libxrpl/telemetry/TelemetryConfig.cpp +++ b/src/libxrpl/telemetry/TelemetryConfig.cpp @@ -112,24 +112,37 @@ makeTelemetrySetup( setup.tlsClientCertPath = section.valueOr(key::tlsClientCert, ""); setup.tlsClientKeyPath = section.valueOr(key::tlsClientKey, ""); - // Mutual TLS needs both the client certificate and its private key. - // Supplying only one fails later with a cryptic SSL handshake error, so - // reject the partial configuration here with an actionable message. - if (setup.tlsClientCertPath.empty() != setup.tlsClientKeyPath.empty()) + // The mutual TLS (mTLS) checks below are fatal, so gate them on the one + // thing this parser can know: `enabled` is 1. With `enabled` 0 a leftover + // cert line must never stop the node from booting. + // + // The predicate is only that config switch, not whether an exporter can + // exist. This file has no preprocessor guard, so both checks also run in a + // -Dtelemetry=OFF build, where makeTelemetry() returns the null + // implementation whatever `enabled` says. + if (setup.enabled) { - Throw( - "[telemetry] tls_client_cert and tls_client_key must be set together " - "(set both for mutual TLS, or neither for one-way TLS)."); - } + // mTLS needs both the client certificate and its private key. + // Supplying only one fails later with a cryptic SSL handshake error, so + // reject the partial configuration here with an actionable message. + if (setup.tlsClientCertPath.empty() != setup.tlsClientKeyPath.empty()) + { + Throw( + "[telemetry] tls_client_cert and tls_client_key must be set together " + "(set both for mutual TLS, or neither for one-way TLS)."); + } - // Mutual TLS only takes effect when TLS is on. Certificate paths set with - // use_tls=0 would be silently ignored and the exporter would connect in - // plaintext, so reject that contradiction instead of failing open. - if (!setup.tlsClientCertPath.empty() && !setup.useTls) - { - Throw( - "[telemetry] tls_client_cert/tls_client_key require use_tls=1 " - "(set use_tls=1 to enable mutual TLS, or remove the cert paths)."); + // Still inside the enabled branch. mTLS only takes effect when TLS is + // on, so a client certificate set with use_tls=0 would be ignored and + // any exporter that did run would connect in plaintext. Reject that + // contradiction instead of failing open. tls_ca_cert is deliberately + // not checked this way. + if (!setup.tlsClientCertPath.empty() && !setup.useTls) + { + Throw( + "[telemetry] tls_client_cert/tls_client_key require use_tls=1 " + "(set use_tls=1 to enable mutual TLS, or remove the cert paths)."); + } } // Head sampling is intentionally fixed at 1.0 (sample everything) and is diff --git a/src/test/nodestore/DatabaseConfig_test.cpp b/src/test/nodestore/DatabaseConfig_test.cpp index 8021351b0d..0f132081f0 100644 --- a/src/test/nodestore/DatabaseConfig_test.cpp +++ b/src/test/nodestore/DatabaseConfig_test.cpp @@ -14,12 +14,12 @@ #include #include +#include #include #include #include #include #include -#include #include #include #include @@ -705,7 +705,7 @@ public: DummyScheduler scheduler; - beast::TempDir const nodeDb; + TempDir const nodeDb; Section nodeParams; nodeParams.set(Keys::kType, "memory"); nodeParams.set(Keys::kPath, nodeDb.path()); @@ -774,7 +774,7 @@ public: CountingScheduler scheduler; - beast::TempDir const nodeDb; + TempDir const nodeDb; Section nodeParams; nodeParams.set(Keys::kType, "nudb"); nodeParams.set(Keys::kPath, nodeDb.path()); @@ -863,8 +863,8 @@ public: std::unique_ptr makeRotatingDatabase( Scheduler& scheduler, - beast::TempDir const& writableDir, - beast::TempDir const& archiveDir) + TempDir const& writableDir, + TempDir const& archiveDir) { Section writableParams; writableParams.set(Keys::kType, "nudb"); @@ -907,8 +907,8 @@ public: CountingScheduler scheduler; - beast::TempDir const writableDir; - beast::TempDir const archiveDir; + TempDir const writableDir; + TempDir const archiveDir; auto rotating = makeRotatingDatabase(scheduler, writableDir, archiveDir); if (!BEAST_EXPECT(rotating)) @@ -1037,7 +1037,7 @@ public: * @return The database, or nullptr on failure. */ std::unique_ptr - makeMeasuredDatabase(beast::TempDir const& dir, Scheduler& scheduler, int readThreads) + makeMeasuredDatabase(TempDir const& dir, Scheduler& scheduler, int readThreads) { Section params; params.set(Keys::kType, "nudb"); @@ -1064,7 +1064,7 @@ public: testcase("nodestore_state totals labels"); DummyScheduler scheduler; - beast::TempDir const nodeDb; + TempDir const nodeDb; Section nodeParams; nodeParams.set(Keys::kType, "nudb"); nodeParams.set(Keys::kPath, nodeDb.path()); @@ -1182,7 +1182,7 @@ public: // publish NOTHING. Zeros here would read as a perfectly idle write // path on a node whose write path is simply not instrumented. { - beast::TempDir const memDb; + TempDir const memDb; Section memParams; memParams.set(Keys::kType, "memory"); memParams.set(Keys::kPath, memDb.path()); @@ -1203,7 +1203,7 @@ public: BEAST_EXPECT(mem->getStoreCount() == 8); } - beast::TempDir const nodeDb; + TempDir const nodeDb; Section nodeParams; nodeParams.set(Keys::kType, "nudb"); nodeParams.set(Keys::kPath, nodeDb.path()); @@ -1336,7 +1336,7 @@ public: testcase("nodestore_state read-queue labels"); DummyScheduler scheduler; - beast::TempDir const nodeDb; + TempDir const nodeDb; // Three read threads and a bundle of 7, neither of which is the // default (the bundle default is 4), so a helper reading the wrong // JSON member cannot agree by coincidence. diff --git a/src/tests/libxrpl/beast/insight/Unit.cpp b/src/tests/libxrpl/beast/insight/Unit.cpp new file mode 100644 index 0000000000..29ceac2b32 --- /dev/null +++ b/src/tests/libxrpl/beast/insight/Unit.cpp @@ -0,0 +1,174 @@ +/** + * GTest unit tests for beast::insight::Unit and its plumbing. + * + * A metric's unit decides two things that are invisible at the call site: the + * name suffix the exporter appends, and which bucket ladder the histogram + * view applies. Getting it wrong is silent -- a byte count declared as + * milliseconds still records, still exports, still draws a graph, and the + * graph is wrong. So each hop the unit has to survive is asserted here + * rather than left to inspection. + * + * The hop that matters most is the group wrapper. Call sites reach a + * collector through Groups, so a unit that reaches OTelCollector correctly + * but is dropped by the group prefixing layer would pass a naive test while + * failing in production. + */ + +#include + +#include +#include +#include +#include + +#include + +#include +#include +#include +#include + +namespace beast::insight { + +namespace { + +/** + * An EventImpl that records what it was notified with. + * + * Needed because every shipped implementation either discards the sample + * (NullCollector) or sends it somewhere external. Asserting the recorded + * value proves the raw-integral path preserves it, rather than only proving + * that notify() can be called without crashing. + */ +class RecordingEventImpl : public EventImpl +{ +public: + explicit RecordingEventImpl(Unit unit) : EventImpl(unit) + { + } + + void + notify(value_type const& value) override + { + samples.push_back(value); + } + + /** + * Every value passed to notify(), in call order. + */ + std::vector samples; +}; + +} // namespace + +// The unit code is a contract with the collector's Prometheus exporter: it +// derives the exported name suffix from this string. Assert the exact codes, +// not merely that they differ. +TEST(InsightUnit, otelCodeIsTheUcumCodeForEachUnit) +{ + EXPECT_STREQ(otelUnitCode(Unit::Millis), "ms"); + EXPECT_STREQ(otelUnitCode(Unit::Bytes), "By"); +} + +// The description is what an operator reads in the metric catalogue, so a +// byte-valued instrument must not describe itself as a duration. +TEST(InsightUnit, descriptionMatchesWhatTheUnitActuallyMeasures) +{ + EXPECT_STREQ(otelUnitDescription(Unit::Millis), "Duration in ms"); + EXPECT_STREQ(otelUnitDescription(Unit::Bytes), "Size in bytes"); +} + +TEST(InsightUnit, defaultEventUnitIsMillisForBackwardCompatibility) +{ + // Every pre-existing makeEvent(name) call site records a duration, so the + // one-argument overload must keep meaning milliseconds. + auto const collector = NullCollector::make(); + auto const event = collector->makeEvent("legacy"); + ASSERT_NE(event.impl(), nullptr); + EXPECT_EQ(event.impl()->unit(), Unit::Millis); +} + +TEST(InsightUnit, makeEventCarriesTheRequestedUnitToTheImpl) +{ + auto const collector = NullCollector::make(); + auto const event = collector->makeEvent("size", Unit::Bytes); + ASSERT_NE(event.impl(), nullptr); + EXPECT_EQ(event.impl()->unit(), Unit::Bytes); +} + +TEST(InsightUnit, prefixedMakeEventCarriesTheUnit) +{ + auto const collector = NullCollector::make(); + auto const event = collector->makeEvent("rpc", "size", Unit::Bytes); + ASSERT_NE(event.impl(), nullptr); + EXPECT_EQ(event.impl()->unit(), Unit::Bytes); +} + +TEST(InsightUnit, groupWrapperForwardsTheUnitAlongWithThePrefix) +{ + // ServerHandler creates its events through a Group, not through the + // collector directly. If the group's makeEvent override forwards only the + // name, the unit silently reverts to milliseconds and the byte histogram + // inherits the latency ladder again. + auto const collector = NullCollector::make(); + auto const groups = makeGroups(collector); + auto const event = groups->get("rpc")->makeEvent("size", Unit::Bytes); + ASSERT_NE(event.impl(), nullptr); + EXPECT_EQ(event.impl()->unit(), Unit::Bytes); +} + +TEST(InsightUnit, groupWrapperStillDefaultsToMillis) +{ + auto const collector = NullCollector::make(); + auto const groups = makeGroups(collector); + auto const event = groups->get("rpc")->makeEvent("time"); + ASSERT_NE(event.impl(), nullptr); + EXPECT_EQ(event.impl()->unit(), Unit::Millis); +} + +TEST(InsightUnit, rawIntegralNotifyPreservesTheValueExactly) +{ + // The byte path must not be rounded or scaled on its way through the + // duration-typed storage field. + auto const impl = std::make_shared(Unit::Bytes); + Event const event(impl); + + event.notify(std::uint64_t{4096}); + event.notify(std::uint64_t{0}); + event.notify(std::uint64_t{1'048'577}); + + ASSERT_EQ(impl->samples.size(), 3U); + EXPECT_EQ(impl->samples[0].count(), 4096); + EXPECT_EQ(impl->samples[1].count(), 0); + EXPECT_EQ(impl->samples[2].count(), 1'048'577); +} + +TEST(InsightUnit, durationNotifyStillRoundsUpToWholeMilliseconds) +{ + // Pre-existing behaviour, asserted so the new overload cannot quietly + // change it: Event applies ceil to whole milliseconds, which is why + // sub-millisecond resolution is impossible on this path. + auto const impl = std::make_shared(Unit::Millis); + Event const event(impl); + + event.notify(std::chrono::microseconds{40}); + event.notify(std::chrono::microseconds{1'000}); + event.notify(std::chrono::milliseconds{7}); + + ASSERT_EQ(impl->samples.size(), 3U); + EXPECT_EQ(impl->samples[0].count(), 1) << "40us must round up to 1ms, not down to 0"; + EXPECT_EQ(impl->samples[1].count(), 1); + EXPECT_EQ(impl->samples[2].count(), 7); +} + +TEST(InsightUnit, notifyOnANullEventIsSafeForBothOverloads) +{ + // A default-constructed Event has no impl. Both overloads must be no-ops + // rather than dereferencing null. + Event const none; + ASSERT_EQ(none.impl(), nullptr); + EXPECT_NO_THROW(none.notify(std::uint64_t{4096})); + EXPECT_NO_THROW(none.notify(std::chrono::milliseconds{5})); +} + +} // namespace beast::insight diff --git a/src/tests/libxrpl/nodestore/Backend.cpp b/src/tests/libxrpl/nodestore/Backend.cpp index 81ef829250..f2648f3e12 100644 --- a/src/tests/libxrpl/nodestore/Backend.cpp +++ b/src/tests/libxrpl/nodestore/Backend.cpp @@ -223,7 +223,7 @@ TEST(BackendWriteStats, non_measuring_backends_report_absence_not_zeros) DummyScheduler scheduler; beast::Journal const journal{TestSink::instance()}; - beast::TempDir const tempDir; + TempDir const tempDir; Section params; params.set("type", type); diff --git a/src/tests/libxrpl/nodestore/Database.cpp b/src/tests/libxrpl/nodestore/Database.cpp index 262d684117..2cf0e783e8 100644 --- a/src/tests/libxrpl/nodestore/Database.cpp +++ b/src/tests/libxrpl/nodestore/Database.cpp @@ -285,7 +285,7 @@ TEST_P(NodeStoreDatabaseTest, write_stats_forwarded_from_backend) TEST(NodeStoreDatabase, sub_millisecond_fetch_latency_is_reported) { CapturingScheduler scheduler; - beast::TempDir const nodeDb; + TempDir const nodeDb; Section nodeParams; nodeParams.set("type", "nudb"); nodeParams.set("path", nodeDb.path()); @@ -490,7 +490,7 @@ TEST(NodeStoreDatabase, import_accumulates_store_duration) DummyScheduler scheduler; beast::Journal const journal(TestSink::instance()); - beast::TempDir const srcDir; + TempDir const srcDir; Section srcParams; srcParams.set("type", "nudb"); srcParams.set("path", srcDir.path()); @@ -504,7 +504,7 @@ TEST(NodeStoreDatabase, import_accumulates_store_duration) storeBatch(*src, batch); ASSERT_EQ(src->getStoreCount(), batch.size()); - beast::TempDir const destDir; + TempDir const destDir; Section destParams; destParams.set("type", "nudb"); destParams.set("path", destDir.path()); diff --git a/src/tests/libxrpl/nodestore/NuDBFactory.cpp b/src/tests/libxrpl/nodestore/NuDBFactory.cpp index bbaf97c6ed..a7d6a11dca 100644 --- a/src/tests/libxrpl/nodestore/NuDBFactory.cpp +++ b/src/tests/libxrpl/nodestore/NuDBFactory.cpp @@ -356,7 +356,7 @@ TEST(NuDBFactory, configuration_parsing) TEST(NuDBFactory, write_stats_accumulate_per_insert) { - beast::TempDir const tempDir; + TempDir const tempDir; auto const params = makeSection(tempDir.path()); DummyScheduler scheduler; beast::Journal const journal(TestSink::instance()); @@ -457,7 +457,7 @@ TEST(NuDBFactory, write_stats_accumulate_per_insert) // ScopeExit runs its function during unwinding. TEST(NuDBFactory, write_stats_count_duplicate_key_inserts) { - beast::TempDir const tempDir; + TempDir const tempDir; auto const params = makeSection(tempDir.path()); DummyScheduler scheduler; beast::Journal const journal(TestSink::instance()); @@ -522,7 +522,7 @@ TEST(NuDBFactory, write_stats_count_duplicate_key_inserts) // while the real one satisfies it as soon as any two inserts overlap. TEST(NuDBFactory, write_stats_measure_depth_under_real_overlap) { - beast::TempDir const tempDir; + TempDir const tempDir; auto const params = makeSection(tempDir.path()); DummyScheduler scheduler; beast::Journal const journal(TestSink::instance()); @@ -597,7 +597,7 @@ TEST(NuDBFactory, write_stats_measure_depth_under_real_overlap) TEST(NuDBFactory, write_load_reports_writer_depth) { - beast::TempDir const tempDir; + TempDir const tempDir; auto const params = makeSection(tempDir.path()); DummyScheduler scheduler; beast::Journal const journal(TestSink::instance()); diff --git a/src/tests/libxrpl/telemetry/HistogramBuckets.cpp b/src/tests/libxrpl/telemetry/HistogramBuckets.cpp new file mode 100644 index 0000000000..f54a7ecde1 --- /dev/null +++ b/src/tests/libxrpl/telemetry/HistogramBuckets.cpp @@ -0,0 +1,235 @@ +/** + * GTest unit tests for the histogram bucket ladders. + * + * These ladders decide whether a Grafana percentile panel reports a + * measurement or an artefact, and neither failure mode is visible in the + * panel itself: a quantile that falls in the `+Inf` bucket reads back as the + * second-highest edge, and one that falls inside bucket 0 is interpolated. + * Both look like plausible numbers. So the invariants are asserted here + * rather than left to review. + * + * The ladders are `constexpr`, so most of this could be `static_assert`. + * They are runtime tests as well so that a failure names which edge is + * wrong instead of only failing the compile. + */ + +#include + +#include + +#include +#include +#include +#include +#include +#include + +namespace xrpl::telemetry::buckets { + +// Every ladder must be strictly ascending and non-negative. The SDK places a +// sample with std::lower_bound over the edges, so a duplicated or +// out-of-order edge silently sends samples to the wrong bucket. +class HistogramBucketsTest : public ::testing::TestWithParam> +{ +}; + +TEST_P(HistogramBucketsTest, isStrictlyAscending) +{ + auto const ladder = GetParam(); + ASSERT_FALSE(ladder.empty()); + for (std::size_t i = 1; i < ladder.size(); ++i) + EXPECT_LT(ladder[i - 1], ladder[i]) << "edge index " << i << " does not ascend"; +} + +TEST_P(HistogramBucketsTest, isNonNegativeAndFinite) +{ + for (double const edge : GetParam()) + { + EXPECT_GE(edge, 0.0); + EXPECT_TRUE(std::isfinite(edge)) << "edge " << edge << " is not finite"; + } +} + +TEST_P(HistogramBucketsTest, passesTheCompileTimeValidator) +{ + EXPECT_TRUE(isAscendingNonNegative(GetParam())); +} + +INSTANTIATE_TEST_SUITE_P( + AllLadders, + HistogramBucketsTest, + ::testing::Values( + std::span{kMillisecondBuckets}, + std::span{kByteBuckets}, + std::span{kMicrosecondBuckets}, + std::span{kObjectCountBuckets}, + std::span{kChargeBuckets})); + +TEST(HistogramBucketsRange, microsecondFloorLandsBelowTheMeasuredMass) +{ + // Measured: 99.3% of job_queued_us samples sat below the old 100 us floor, + // so p75/p95/p99 all interpolated inside bucket 0 and returned + // 75.5/95.7/99.7 us -- the boundary scaled by the requested quantile, + // not a latency. Warm nodestore reads are ~1.5 us, so the floor has to + // reach single microseconds and several edges must precede 100 us. + EXPECT_LE(kMicrosecondBuckets.front(), 1.0); + + auto const belowHundred = + std::ranges::count_if(kMicrosecondBuckets, [](double edge) { return edge < 100.0; }); + EXPECT_GE(belowHundred, 5) << "too little resolution below 100 us"; +} + +TEST(HistogramBucketsRange, microsecondCeilingStillReachesOneMinute) +{ + // Job waits and RPC latencies routinely exceed the SDK default ceiling of + // 10,000; multi-second stalls must stay measurable rather than censored. + EXPECT_EQ(kMicrosecondBuckets.back(), 60'000'000.0); +} + +TEST(HistogramBucketsRange, objectCountLadderCannotSaturate) +{ + // GetObject counts run 1..kHardMaxReplyNodes, so the top edge IS the hard + // cap and censoring is impossible by construction. + EXPECT_EQ(kObjectCountBuckets.front(), 1.0); + EXPECT_EQ(kObjectCountBuckets.back(), 12'288.0); +} + +TEST(HistogramBucketsRange, chargeLadderBracketsTheResourceThresholds) +{ + // The two edges that decide a peer's fate must be present so a dashboard + // can show how close charges run to each: warning at 5000, drop at 25000. + // A leading 0 separates the free tier from everything else. + EXPECT_EQ(kChargeBuckets.front(), 0.0); + for (double const threshold : {5'000.0, 25'000.0}) + { + EXPECT_NE(std::ranges::find(kChargeBuckets, threshold), kChargeBuckets.end()) + << threshold << " is a resource threshold and must be an edge"; + } +} + +// The validator must also REJECT. A predicate that only ever returns true +// would let every ladder above pass while proving nothing. +TEST(HistogramBucketsValidator, rejectsEmptyDescendingDuplicateAndNegative) +{ + EXPECT_FALSE(isAscendingNonNegative(std::span{})); + + constexpr std::array descending{5.0, 1.0}; + EXPECT_FALSE(isAscendingNonNegative(descending)); + + constexpr std::array duplicated{1.0, 1.0, 2.0}; + EXPECT_FALSE(isAscendingNonNegative(duplicated)); + + constexpr std::array negative{-1.0, 1.0}; + EXPECT_FALSE(isAscendingNonNegative(negative)); +} + +TEST(HistogramBucketsValidator, acceptsASingleEdgeAndALeadingZero) +{ + constexpr std::array single{1.0}; + EXPECT_TRUE(isAscendingNonNegative(single)); + + // A leading zero is legal: the GetObject charge ladder starts at 0 to + // separate the free tier from everything else. + constexpr std::array leadingZero{0.0, 100.0}; + EXPECT_TRUE(isAscendingNonNegative(leadingZero)); +} + +TEST(HistogramBucketsRange, millisecondFloorIsOneAndCeilingCoversTheSlowestJob) +{ + // beast::insight::Event rounds durations up to whole milliseconds, so 1 + // is the smallest edge that can ever collect a sample. + EXPECT_EQ(kMillisecondBuckets.front(), 1.0); + + // The updatepaths job type was measured averaging 59,956 ms. A 30 s + // ceiling -- the collector's top edge -- would censor it just as the old + // 5 s ceiling does, so this ladder has to reach further. + EXPECT_GE(kMillisecondBuckets.back(), 120'000.0); +} + +TEST(HistogramBucketsRange, millisecondLadderClearsTheMeasuredCensoringPoint) +{ + // rpc_size had 24.9% of samples above the old 5000 ceiling and + // jobq_updatepaths had 100%. A ceiling at or below 5000 reintroduces the + // exact defect this ladder exists to fix. + EXPECT_GT(kMillisecondBuckets.back(), 5'000.0); +} + +TEST(HistogramBucketsRange, millisecondLadderContainsEveryRepresentableCollectorEdge) +{ + // Agreement with the collector's spanmetrics ladder over the shared + // range is the invariant; edges above its 30 s top are allowed because + // jobs outlive spans. Sub-millisecond collector edges are excluded + // because Event cannot represent them. check_bucket_parity.py enforces + // this against the YAML; this test pins it for the C++ side alone so a + // local edit fails fast. + constexpr std::array collectorEdges{ + 1.0, + 5.0, + 10.0, + 25.0, + 50.0, + 100.0, + 250.0, + 500.0, + 1'000.0, + 2'000.0, + 3'000.0, + 4'000.0, + 5'000.0, + 10'000.0, + 30'000.0}; + + for (double const edge : collectorEdges) + { + EXPECT_NE(std::ranges::find(kMillisecondBuckets, edge), kMillisecondBuckets.end()) + << edge << " ms is a collector spanmetrics edge and must be present"; + } +} + +TEST(HistogramBucketsRange, millisecondLadderResolvesTheOneToFiveSecondBand) +{ + // Without these the 1 s to 5 s span was one four-second-wide bucket, so + // any quantile landing inside it was interpolated across four seconds. + for (double const edge : {2'000.0, 3'000.0, 4'000.0}) + { + EXPECT_NE(std::ranges::find(kMillisecondBuckets, edge), kMillisecondBuckets.end()) + << edge << " ms edge missing"; + } +} + +TEST(HistogramBucketsRange, byteLadderBracketsTheMeasuredResponseDistribution) +{ + // Measured: mean 2131 B, half under 1 kB, three quarters under 5 kB, and + // the tail above 5 kB has a mean of at most 7538 B -- which puts p99 + // near 80 kB. The floor must sit at or below the measured median region + // and the ceiling well past the p99 bound. + EXPECT_LE(kByteBuckets.front(), 512.0); + EXPECT_GE(kByteBuckets.back(), 1'048'576.0); + + // Most of the resolution belongs where the distribution actually turns. + auto const withinWorkingRange = + std::ranges::count_if(kByteBuckets, [](double e) { return e >= 512.0 && e <= 65'536.0; }); + EXPECT_GE(withinWorkingRange, 6) << "too little resolution between 512 B and 64 kB"; +} + +TEST(HistogramBucketsRange, byteAndMillisecondLaddersAreDistinct) +{ + // A single shared ladder is what put a byte count on a latency scale and + // censored a quarter of its samples. + EXPECT_NE(kByteBuckets.size(), kMillisecondBuckets.size()); + EXPECT_GT(kByteBuckets.back(), kMillisecondBuckets.back()); +} + +TEST(HistogramBucketsConvert, toVectorPreservesOrderAndSize) +{ + auto const converted = toVector(kByteBuckets); + ASSERT_EQ(converted.size(), kByteBuckets.size()); + EXPECT_TRUE(std::ranges::equal(converted, kByteBuckets)); +} + +TEST(HistogramBucketsConvert, toVectorHandlesAnEmptyLadder) +{ + EXPECT_TRUE(toVector(std::span{}).empty()); +} + +} // namespace xrpl::telemetry::buckets diff --git a/src/tests/libxrpl/telemetry/NodeIdResource.cpp b/src/tests/libxrpl/telemetry/NodeIdResource.cpp new file mode 100644 index 0000000000..c0f4387b7e --- /dev/null +++ b/src/tests/libxrpl/telemetry/NodeIdResource.cpp @@ -0,0 +1,162 @@ +#include +#include +#include +#include + +#include + +#include +#include + +#ifdef XRPL_ENABLE_TELEMETRY +#include +#include +#endif + +/** + * Contract tests for the `xrpl.node.id` resource attribute. + * + * `xrpl.node.id` carries the node's base58 public key on both the trace and + * the metric OTel resource, so traces and metrics resolve to one node. The + * key string is a cross-component contract: the collector, TraceQL queries + * and Grafana dashboards all name it literally, and a silent rename would + * break them with no compile error. These tests pin the literal key, the + * Setup default, and the fact that the value can only arrive through + * Telemetry::setNodeId(). + * + * Scope limit: the two production resources are built inside TelemetryImpl + * (trace resource in start(), metric resource in the constructor) and inside + * MetricsRegistry::initExporterAndProvider(). Neither is reachable from this + * binary — TelemetryImpl only exists behind an OTLP/HTTP exporter with + * background export threads, which a unit test must not spin up (see + * GetMeter.cpp), and MetricsRegistry.cpp is not compiled into xrpl_tests in + * the telemetry-enabled build. The resource test below therefore pins the SDK + * contract those three call sites rely on: the exact key, and a std::string + * value landing in the string alternative of the attribute variant rather + * than the bool one. + */ + +using namespace xrpl; +using namespace xrpl::telemetry; + +TEST(NodeIdResource, attribute_key_is_dotted_resource_form) +{ + // The literal the collector, TraceQL and the dashboards all name. + EXPECT_EQ(std::string_view(attr::nodeId), "xrpl.node.id"); + + // Dotted, not the underscore form used for span attributes. + EXPECT_EQ(std::string_view(attr::nodeId).find('_'), std::string_view::npos); + + // Sibling of the other two xrpl.* resource attributes, and distinct + // from both. + EXPECT_EQ(std::string_view(attr::networkId), "xrpl.network.id"); + EXPECT_EQ(std::string_view(attr::networkType), "xrpl.network.type"); + EXPECT_NE(std::string_view(attr::nodeId), std::string_view(attr::networkId)); + EXPECT_NE(std::string_view(attr::nodeId), std::string_view(attr::networkType)); + + // Built from the shared segments, so the segment additions are exercised + // too rather than only the joined result. + EXPECT_EQ(std::string_view(seg::node), "node"); + EXPECT_EQ(std::string_view(seg::xrpl), "xrpl"); +} + +TEST(NodeIdResource, setup_node_id_defaults_to_empty) +{ + // Negative path: nothing has called setNodeId(), so there is no value to + // stamp and the resource builders skip the attribute. + Telemetry::Setup const s; + EXPECT_TRUE(s.nodeId.empty()); + EXPECT_EQ(s.nodeId, ""); +} + +TEST(NodeIdResource, config_parsing_never_populates_node_id) +{ + // nodeId is deliberately not config-driven. Even with an explicit + // service_instance_id and a node public key argument, makeTelemetrySetup() + // must leave nodeId empty: Application::setup() is the only writer, via + // setNodeId(). + Section section; + section.set("enabled", "1"); + section.set("service_instance_id", "custom-id"); + + auto const setup = makeTelemetrySetup(section, "nHUtest123", "2.0.0", 1); + + EXPECT_EQ(setup.serviceInstanceId, "custom-id"); + EXPECT_TRUE(setup.nodeId.empty()); +} + +TEST(NodeIdResource, set_node_id_on_disabled_path_is_inert) +{ + // The disabled build/config path takes the base-class no-op. Calling it + // must be safe and must not change any observable state. + Telemetry::Setup setup; + setup.enabled = false; + + beast::Journal::Sink& sink = beast::Journal::getNullSink(); + beast::Journal const journal(sink); + auto telemetry = makeTelemetry(setup, journal); + ASSERT_NE(telemetry, nullptr); + + telemetry->setNodeId("nHUtest123"); + + EXPECT_FALSE(telemetry->isEnabled()); + EXPECT_FALSE(telemetry->shouldTraceRpc()); + EXPECT_FALSE(telemetry->shouldTraceTransactions()); + EXPECT_FALSE(telemetry->shouldTraceConsensus()); + EXPECT_FALSE(telemetry->shouldTracePeer()); + EXPECT_FALSE(telemetry->shouldTraceLedger()); + EXPECT_EQ(telemetry->getConsensusTraceStrategy(), "deterministic"); +} + +#ifdef XRPL_ENABLE_TELEMETRY + +TEST(NodeIdResource, resource_carries_node_id_as_a_string) +{ + namespace otel_resource = opentelemetry::sdk::resource; + + // A base58 node public key: 'n' prefix, 52 characters. + std::string const nodeId = "n9MozjnGB3tpULewtTsVtuudg5JqYFyV3QFdAtVLzJaxHcBaxuXM"; + ASSERT_EQ(nodeId.size(), 52u); + + otel_resource::ResourceAttributes attrs; + // std::string, never a string literal: the attribute variant's + // char-const* overload binds to bool, which would record `true`. + attrs[std::string(attr::nodeId)] = nodeId; + + auto const resource = otel_resource::Resource::Create(attrs); + auto const& out = resource.GetAttributes(); + + auto const it = out.find("xrpl.node.id"); + ASSERT_NE(it, out.end()); + + // The string alternative, not bool — the pitfall the call sites guard. + ASSERT_TRUE(opentelemetry::nostd::holds_alternative(it->second)); + EXPECT_FALSE(opentelemetry::nostd::holds_alternative(it->second)); + EXPECT_EQ(opentelemetry::nostd::get(it->second), nodeId); +} + +TEST(NodeIdResource, resource_omits_node_id_when_it_was_never_set) +{ + namespace otel_resource = opentelemetry::sdk::resource; + + // Negative path: the call sites only assign when the value is non-empty, + // so an unset node ID leaves the key off the resource entirely rather + // than stamping a blank one. + Telemetry::Setup const setup; + ASSERT_TRUE(setup.nodeId.empty()); + + otel_resource::ResourceAttributes attrs; + if (!setup.nodeId.empty()) + attrs[std::string(attr::nodeId)] = setup.nodeId; + + auto const resource = otel_resource::Resource::Create(attrs); + auto const& out = resource.GetAttributes(); + + EXPECT_EQ(out.find("xrpl.node.id"), out.end()); + + // The SDK still merges in its own defaults, so the absence above is a + // real absence and not an empty map. + EXPECT_FALSE(out.empty()); +} + +#endif // XRPL_ENABLE_TELEMETRY diff --git a/src/tests/libxrpl/telemetry/TelemetryConfig.cpp b/src/tests/libxrpl/telemetry/TelemetryConfig.cpp index bfa0b85149..2c7977a2e3 100644 --- a/src/tests/libxrpl/telemetry/TelemetryConfig.cpp +++ b/src/tests/libxrpl/telemetry/TelemetryConfig.cpp @@ -2,12 +2,83 @@ #include #include +#include #include #include using namespace xrpl; +using ::testing::HasSubstr; +using ::testing::ThrowsMessage; + +namespace { + +/** + * Shared inputs for the mutual TLS (mTLS) tests of makeTelemetrySetup(). + * + * keyClientCert and keyClientKey are the config key names, named once so every + * test below spells them the same way, mirroring the `key::` constants the + * parser itself uses. A misspelling cannot hide here: the throwing case that + * names the misspelled key stops throwing, the use_tls case throws the pairing + * message instead and fails its matcher, and the value cases see an empty path + * or an unexpected throw. Tests that never set the key are unaffected. One + * source of truth still keeps the two files from drifting apart. + * + * clientCert and clientKey are the paths written to those keys. They are + * declared as `char const*` so they pass to Section::set() (which takes + * `std::string const&`) and compare against the parsed std::string members + * without an explicit conversion, exactly as a literal would. + * + * pairingError and useTlsError are message fragments. Both guards throw + * std::runtime_error, so the exception type alone cannot tell them apart. + * Each fragment occurs in exactly one of the two messages, so matching it + * proves which guard fired. + */ +namespace mtls { +constexpr char const* keyClientCert = "tls_client_cert"; +constexpr char const* keyClientKey = "tls_client_key"; +constexpr char const* clientCert = "/etc/ssl/client.pem"; +constexpr char const* clientKey = "/etc/ssl/client.key"; +constexpr char const* pairingError = "must be set together"; +constexpr char const* useTlsError = "require use_tls=1"; + +/** + * Build a [telemetry] section carrying only the `enabled` key. + * + * Every mTLS test states `enabled` explicitly, because the validation + * guards run only when telemetry is on. Each test then adds the TLS keys its + * own case needs on top of the returned section. + * + * @param telemetryEnabled Value written to the `enabled` key. + * @return The section, ready for further set() calls. + */ +Section +makeSection(bool telemetryEnabled) +{ + Section section; + section.set("enabled", telemetryEnabled ? "1" : "0"); + return section; +} + +/** + * Parse a [telemetry] section with a fixed placeholder node identity. + * + * Keeps the node key, version and network ID out of the individual cases, + * which vary only in their TLS keys. + * + * @param section The section to parse. + * @return The populated Setup struct. + */ +telemetry::Telemetry::Setup +parseSection(Section const& section) +{ + return telemetry::makeTelemetrySetup(section, "nHUtest123", "2.0.0", 0); +} +} // namespace mtls + +} // namespace + TEST(TelemetryConfig, setup_defaults) { telemetry::Telemetry::Setup const s; @@ -15,6 +86,7 @@ TEST(TelemetryConfig, setup_defaults) EXPECT_EQ(s.serviceName, "xrpld"); EXPECT_TRUE(s.serviceVersion.empty()); EXPECT_TRUE(s.serviceInstanceId.empty()); + EXPECT_TRUE(s.nodeId.empty()); EXPECT_EQ(s.exporterEndpoint, "http://localhost:4318/v1/traces"); EXPECT_FALSE(s.useTls); EXPECT_TRUE(s.tlsCertPath.empty()); @@ -87,39 +159,110 @@ TEST(TelemetryConfig, parse_full_section) TEST(TelemetryConfig, mtls_cert_and_key_both_set) { - Section section; + // Telemetry on and use_tls=1, so both guards run and neither may fire. + Section section = mtls::makeSection(true); section.set("use_tls", "1"); - section.set("tls_client_cert", "/etc/ssl/client.pem"); - section.set("tls_client_key", "/etc/ssl/client.key"); + section.set(mtls::keyClientCert, mtls::clientCert); + section.set(mtls::keyClientKey, mtls::clientKey); - auto setup = telemetry::makeTelemetrySetup(section, "nHUtest123", "2.0.0", 0); - EXPECT_EQ(setup.tlsClientCertPath, "/etc/ssl/client.pem"); - EXPECT_EQ(setup.tlsClientKeyPath, "/etc/ssl/client.key"); + auto const setup = mtls::parseSection(section); + EXPECT_TRUE(setup.enabled); + EXPECT_TRUE(setup.useTls); + EXPECT_EQ(setup.tlsClientCertPath, mtls::clientCert); + EXPECT_EQ(setup.tlsClientKeyPath, mtls::clientKey); } TEST(TelemetryConfig, mtls_cert_without_key_throws) { - Section section; - section.set("tls_client_cert", "/etc/ssl/client.pem"); - EXPECT_THROW( - telemetry::makeTelemetrySetup(section, "nHUtest123", "2.0.0", 0), std::runtime_error); + // Only the cert is set, so the pairing guard is the one that must fire. + Section section = mtls::makeSection(true); + section.set(mtls::keyClientCert, mtls::clientCert); + + EXPECT_THAT( + [§ion] { mtls::parseSection(section); }, + ThrowsMessage(HasSubstr(mtls::pairingError))); } TEST(TelemetryConfig, mtls_key_without_cert_throws) { - Section section; - section.set("tls_client_key", "/etc/ssl/client.key"); - EXPECT_THROW( - telemetry::makeTelemetrySetup(section, "nHUtest123", "2.0.0", 0), std::runtime_error); + // Only the key is set, the mirror image of the case above. + Section section = mtls::makeSection(true); + section.set(mtls::keyClientKey, mtls::clientKey); + + EXPECT_THAT( + [§ion] { mtls::parseSection(section); }, + ThrowsMessage(HasSubstr(mtls::pairingError))); +} + +TEST(TelemetryConfig, mtls_cert_key_without_use_tls_throws) +{ + // Both paths are set, so the pairing guard cannot fire; use_tls is absent + // and defaults to 0, so the use_tls guard is the only reachable throw. + Section section = mtls::makeSection(true); + section.set(mtls::keyClientCert, mtls::clientCert); + section.set(mtls::keyClientKey, mtls::clientKey); + + EXPECT_THAT( + [§ion] { mtls::parseSection(section); }, + ThrowsMessage(HasSubstr(mtls::useTlsError))); +} + +TEST(TelemetryConfig, mtls_contradiction_ignored_when_telemetry_disabled) +{ + // The use_tls contradiction with telemetry off: parsing must succeed so a + // stale cert line cannot stop the node from booting. + Section section = mtls::makeSection(false); + section.set(mtls::keyClientCert, mtls::clientCert); + section.set(mtls::keyClientKey, mtls::clientKey); + + auto const setup = mtls::parseSection(section); + EXPECT_FALSE(setup.enabled); + EXPECT_FALSE(setup.useTls); + EXPECT_EQ(setup.tlsClientCertPath, mtls::clientCert); + EXPECT_EQ(setup.tlsClientKeyPath, mtls::clientKey); +} + +TEST(TelemetryConfig, mtls_cert_without_key_ignored_when_telemetry_disabled) +{ + // The pairing violation with telemetry off: also parsed, not rejected. + Section section = mtls::makeSection(false); + section.set(mtls::keyClientCert, mtls::clientCert); + + auto const setup = mtls::parseSection(section); + EXPECT_FALSE(setup.enabled); + EXPECT_FALSE(setup.useTls); + EXPECT_EQ(setup.tlsClientCertPath, mtls::clientCert); + EXPECT_TRUE(setup.tlsClientKeyPath.empty()); +} + +TEST(TelemetryConfig, mtls_default_no_client_tls_is_accepted) +{ + // The documented default with telemetry on: no client certificate, and + // use_tls absent so it defaults to 0. Both guards run and neither may + // fire. The use_tls guard tests the certificate path first; drop that + // conjunct and this config is rejected, so no default node could boot. + Section const section = mtls::makeSection(true); + + telemetry::Telemetry::Setup setup; + ASSERT_NO_THROW(setup = mtls::parseSection(section)); + EXPECT_TRUE(setup.enabled); + EXPECT_FALSE(setup.useTls); + EXPECT_TRUE(setup.tlsClientCertPath.empty()); + EXPECT_TRUE(setup.tlsClientKeyPath.empty()); } TEST(TelemetryConfig, mtls_neither_set_is_one_way_tls) { - Section section; + // Telemetry is on so the guards run, and this config must pass both: + // one-way TLS with a CA bundle and no client certificate. + Section section = mtls::makeSection(true); section.set("use_tls", "1"); section.set("tls_ca_cert", "/etc/ssl/ca.pem"); - auto setup = telemetry::makeTelemetrySetup(section, "nHUtest123", "2.0.0", 0); + auto const setup = mtls::parseSection(section); + EXPECT_TRUE(setup.enabled); + EXPECT_TRUE(setup.useTls); + EXPECT_EQ(setup.tlsCertPath, "/etc/ssl/ca.pem"); EXPECT_TRUE(setup.tlsClientCertPath.empty()); EXPECT_TRUE(setup.tlsClientKeyPath.empty()); } diff --git a/src/xrpld/app/main/Application.cpp b/src/xrpld/app/main/Application.cpp index e0a6ecc089..7c1effbb14 100644 --- a/src/xrpld/app/main/Application.cpp +++ b/src/xrpld/app/main/Application.cpp @@ -330,7 +330,8 @@ public: ApplicationImp( std::unique_ptr config, std::unique_ptr logs, - std::unique_ptr timeKeeper) + std::unique_ptr timeKeeper, + std::optional const& nodePublicKey) : BasicApp(numberOfThreads(*config)) , config_(std::move(config)) , logs_(std::move(logs)) @@ -344,14 +345,26 @@ public: *this, logs_->journal("PerfLog"), [this] { signalStop("PerfLog"); })) + // Telemetry publishes the MeterProvider on construction, so it must + // precede collectorManager_ below and every subsystem that creates an + // instrument. Its resource is immutable, so the instance id has to be + // supplied now; empty means this run reports none. , telemetry_( telemetry::makeTelemetry( telemetry::makeTelemetrySetup( config_->section("telemetry"), - "", // Updated later via setServiceInstanceId() + nodePublicKey.value_or(""), build_info::getVersionString(), config_->networkId), logs_->journal("Telemetry"))) + // Built here, not in setup(): getMetricsRegistry() is read from the job + // queue and io threads, which are already running, so assigning the + // handle later would race with those reads. + , metricsRegistry_( + std::make_unique( + telemetry_->isEnabled(), + *this, + logs_->journal("MetricsRegistry"))) , txMaster_(*this) , collectorManager_(makeCollectorManager( @@ -529,6 +542,33 @@ public: add(ledgerCleaner_.get()); } + /** + * Stop observing and stop telemetry before the members are destroyed. + * + * The metrics reader thread runs callbacks that read the services member + * destruction is about to tear down. telemetry_ is declared early because + * the collector needs its MeterProvider, so reverse-order member destruction + * would take it down last. + * + * run() does both on the normal path; this covers the paths that never + * reach it -- every `return false` in setup(), and the unit tests. Both + * calls are idempotent. + */ + ~ApplicationImp() override + { + // A shutdown diagnostic must never terminate the process, and a + // destructor is implicitly noexcept. + try + { + collectorManager_->collector()->onCollectionStopping(); + telemetry_->stop(); + } + catch (std::exception const& e) + { + JLOG(journal_.error()) << "Error stopping telemetry: " << e.what(); + } + } + //-------------------------------------------------------------------------- bool @@ -1270,15 +1310,15 @@ private: * * Rule for keeping this call site valid: only telemetry work that reads * NO application subsystem may run here. That holds today — this phase - * uses the config strings and the node identity, and creates only - * push-model counters and histograms, which app code records into once - * it is ready. Anything that registers a callback reading a subsystem - * must go in startTelemetryGauges() instead, because a callback - * registered here can fire on the metrics reader thread while the rest of - * the application is still being built. + * uses the config strings and creates only push-model counters and + * histograms, which app code records into once it is ready. Anything that + * registers a callback reading a subsystem must go in + * startTelemetryGauges() instead, because a callback registered here can + * fire on the metrics reader thread while the rest of the application is + * still being built. * - * @pre nodeIdentity_ is populated (needed for the service_instance_id - * fallback) and metricsRegistry_ is constructed. + * The resource attributes, including service.instance.id, were supplied at + * construction. */ void startTelemetry() const; @@ -1402,19 +1442,15 @@ ApplicationImp::setup(boost::program_options::variables_map const& cmdline) nodeIdentity_ = getNodeIdentity(*this, cmdline); - // Now that the node identity is known, inject it into the telemetry - // resource attributes — but only if the user didn't already set a - // custom service_instance_id in [telemetry]. The Telemetry object - // was constructed with an empty serviceInstanceId because - // nodeIdentity_ is not available in the member initializer list. + // The metrics resource was fixed at construction, but the tracer resource is + // built by start() below, so a key minted just now can still reach spans. if (!config_->section("telemetry").exists("service_instance_id")) telemetry_->setServiceInstanceId(toBase58(TokenType::NodePublic, nodeIdentity_->first)); - // Create the OTel MetricsRegistry for gap-fill metrics (counters, - // histograms, observable gauges). It must exist before startTelemetry(), - // which starts the metrics half of the pipeline. - metricsRegistry_ = std::make_unique( - telemetry_->isEnabled(), *this, logs_->journal("MetricsRegistry")); + // xrpl.node.id always carries the node public key. Unlike + // service_instance_id it is not configurable, so traces and metrics keep a + // stable per-node key whatever [telemetry] says. + telemetry_->setNodeId(toBase58(TokenType::NodePublic, nodeIdentity_->first)); // Start telemetry here, not in start(). Spans and metrics are both emitted // during the rest of setup() — the first consensus round in @@ -1595,14 +1631,15 @@ ApplicationImp::setup(boost::program_options::variables_map const& cmdline) collectorManager_->collector()); add(*overlay_); // add to PropertyStream - // Register the observable instruments now that overlay_ exists. This arms - // the metrics reader thread to invoke their callbacks, several of which - // read getOverlay() — registering earlier would let the reader observe a - // half-built application. The reader thread itself already started in - // startTelemetry() above; this is as early as the callbacks can safely be - // attached, and it is still before beginConsensus() so the gauges cover - // the first round. + // Register the observable instruments now that overlay_ exists — the last of + // the services their callbacks read. Registering earlier would let the + // metrics reader thread observe a half-built application. Two independent + // sets: the MetricsRegistry gauges, and the insight collector's, whose + // callbacks additionally run the hook handlers in ledgerMaster_, + // networkOPs_, the peer finder and the job queue. Both are still before + // beginConsensus() below, so they cover the first round. startTelemetryGauges(); + collectorManager_->collector()->onCollectionReady(); // start first consensus round if (!networkOPs_->beginConsensus(ledgerMaster_->getClosedLedger()->header().hash, {})) @@ -1764,7 +1801,13 @@ ApplicationImp::startTelemetry() const if (instanceId.empty() && nodeIdentity_) instanceId = toBase58(TokenType::NodePublic, nodeIdentity_->first); - metricsRegistry_->start(endpoint, instanceId); + // The node public key also goes on its own resource attribute, + // xrpl.node.id, which config cannot override. + std::string nodeId; + if (nodeIdentity_) + nodeId = toBase58(TokenType::NodePublic, nodeIdentity_->first); + + metricsRegistry_->start(endpoint, instanceId, nodeId); } } @@ -1847,21 +1890,31 @@ ApplicationImp::run() return getValidators().trustedPublisher(pubKey); }); + // Stop observing before any service below is stopped: the collector's gauge + // callbacks run hook handlers that read ledgerMaster_, networkOPs_, the peer + // finder, the job queue and overlay_. Returns once no callback is running. + collectorManager_->collector()->onCollectionStopping(); + // The order of these stop calls is delicate. // Re-ordering them risks undefined behavior. loadManager_->stop(); - // Detach MetricsRegistry observable-gauge callbacks BEFORE stopping - // any service the callbacks read from. The callbacks run on the OTel - // reader thread and touch nodeStore_, overlay_, networkOPs_, - // ledgerMaster, inboundLedgers, etc. A final tick that fires after - // one of those services has shut down would dereference dangling - // state. detachCallbacks() flips an atomic flag every callback - // acquire-loads at its entry, so subsequent ticks become no-ops. - // The final provider teardown still happens in metricsRegistry_->stop() - // farther down. + // Stop the metrics pipeline BEFORE any service its callbacks read. Those + // callbacks run on the OTel reader thread and touch nodeStore_, overlay_, + // networkOPs_, ledgerMaster, inboundLedgers and more, so a tick arriving + // after one of them has stopped would read dangling state. + // + // detachCallbacks() alone would not be enough: it flips a flag that each + // callback checks on entry, which leaves a callback that is already past + // that check running. stop() shuts the provider down, which joins the + // reader thread, so once it returns no callback is running or can start. + // The cost is that metrics recorded during the remaining shutdown steps + // are not exported. if (metricsRegistry_) + { metricsRegistry_->detachCallbacks(); + metricsRegistry_->stop(); + } shaMapStore_->stop(); jobQueue_->stop(); @@ -1876,10 +1929,6 @@ ApplicationImp::run() ledgerCleaner_->stop(); nodeStore_->stop(); perfLog_->stop(); - // Stop metrics pipeline before telemetry — gauge callbacks reference - // Application services that may be shutting down. - if (metricsRegistry_) - metricsRegistry_->stop(); // Telemetry must stop last among trace-producing components. // serverHandler_, overlay_, and jobQueue_ are already stopped above, // so no threads should be calling startSpan() at this point. @@ -2447,9 +2496,19 @@ makeApplication( std::unique_ptr config, std::unique_ptr logs, std::unique_ptr timeKeeper) +{ + return makeApplication(std::move(config), std::move(logs), std::move(timeKeeper), std::nullopt); +} + +std::unique_ptr +makeApplication( + std::unique_ptr config, + std::unique_ptr logs, + std::unique_ptr timeKeeper, + std::optional const& nodePublicKey) { return std::make_unique( - std::move(config), std::move(logs), std::move(timeKeeper)); + std::move(config), std::move(logs), std::move(timeKeeper), nodePublicKey); } void diff --git a/src/xrpld/app/main/Application.h b/src/xrpld/app/main/Application.h index 225275afe4..1d7125cd64 100644 --- a/src/xrpld/app/main/Application.h +++ b/src/xrpld/app/main/Application.h @@ -174,4 +174,19 @@ makeApplication( std::unique_ptr logs, std::unique_ptr timeKeeper); +/** + * Construct the application with a known node public key. + * + * Telemetry builds its resource attributes during construction and they are + * immutable, so the base58 node public key must be supplied here. Pass + * std::nullopt when it is unknown; that run reports no instance id. See + * resolveNodePublicKey(). + */ +std::unique_ptr +makeApplication( + std::unique_ptr config, + std::unique_ptr logs, + std::unique_ptr timeKeeper, + std::optional const& nodePublicKey); + } // namespace xrpl diff --git a/src/xrpld/app/main/Main.cpp b/src/xrpld/app/main/Main.cpp index ba6520db5f..fcae528737 100644 --- a/src/xrpld/app/main/Main.cpp +++ b/src/xrpld/app/main/Main.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -12,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -36,6 +38,7 @@ #include #include #include +#include #include #include #include @@ -804,8 +807,58 @@ run(int argc, char** argv) if (vm.contains("debug")) setDebugLogSink(logs->makeSink("Debug", beast::Severity::Trace)); - auto app = - makeApplication(std::move(config), std::move(logs), std::make_unique()); + // Telemetry needs the node public key at construction, so read it here + // where a config error can still be reported and the process can exit + // cleanly. getNodeIdentity() in setup() stays authoritative. + std::optional nodePublicKey; + try + { + nodePublicKey = resolveNodePublicKey(*config, vm, logs->journal("Application")); + } + catch (std::exception const& e) + { + std::cerr << "Unable to start " << systemName() << ": " << e.what() << std::endl; + return -1; + } + + if (!nodePublicKey) + { + JLOG(logs->journal("Application").warn()) + << "Telemetry: no node identity available yet, so this run reports an empty " + "service.instance.id. Set [telemetry] service_instance_id, or restart once " + "the node key exists."; + } + + // Application construction runs member initializers that validate + // config (for example the [telemetry] section) and can throw. A throw + // from a member-initializer list cannot be recovered inside the + // constructor, so catch it here. Left uncaught it reaches + // std::terminate, whose default handler prints a C++ terminate dump + // and raises SIGABRT, leaving a core file where the system allows one; + // the catch replaces that with two operator-readable lines on stderr + // and a non-zero exit status. + // + // Only the construction is covered. The [telemetry] section is parsed + // near the top of the member list, before the job queue and node store + // are built, so unwinding that throw destroys very little. setup() is + // left outside deliberately: it starts subsystems whose shutdown order + // is delicate, and only the normal stop sequence gets that order right. + std::unique_ptr app; + try + { + app = makeApplication( + std::move(config), std::move(logs), std::make_unique(), nodePublicKey); + } + catch (std::exception const& e) + { + std::cerr << "Unable to start " << systemName() << ": " << e.what() << std::endl; + std::cerr << "Fix the reported problem and start again." << std::endl; + return -1; + } + + // Construction succeeded, so app holds an object: makeApplication never + // returns null and the catch above is the only other way out. + XRPL_ASSERT(app, "xrpl::run : non-null application"); if (!app->setup(vm)) return -1; diff --git a/src/xrpld/app/main/NodeIdentity.cpp b/src/xrpld/app/main/NodeIdentity.cpp index 8198c43af7..3860bbe3c6 100644 --- a/src/xrpld/app/main/NodeIdentity.cpp +++ b/src/xrpld/app/main/NodeIdentity.cpp @@ -3,18 +3,29 @@ #include #include +#include #include +#include #include +#include #include +#include #include #include +#include +#include +#include #include #include +#include +#include +#include #include #include #include +#include #include namespace xrpl { @@ -58,4 +69,82 @@ getNodeIdentity(Application& app, boost::program_options::variables_map const& c return getNodeIdentity(*db); } +std::optional +resolveNodePublicKey( + Config const& config, + boost::program_options::variables_map const& cmdline, + beast::Journal journal) +{ + std::optional seed; + bool seedConfigured = false; + + if (cmdline.contains("nodeid")) + { + seedConfigured = true; + seed = parseGenericSeed(cmdline["nodeid"].as(), false); + } + else if (config.exists(Sections::kNodeSeed)) + { + seedConfigured = true; + if (auto const& lines = config.section(Sections::kNodeSeed).lines(); !lines.empty()) + seed = parseBase58(lines.front()); + } + + // A configured seed decides the identity outright. A malformed or missing + // one is reported by getNodeIdentity(), which runs later. + if (seedConfigured) + { + if (!seed) + return std::nullopt; + + auto const secretKey = generateSecretKey(KeyType::Secp256k1, *seed); + return toBase58(TokenType::NodePublic, derivePublicKey(KeyType::Secp256k1, secretKey)); + } + + // --newnodeid discards whatever is stored. + if (cmdline.contains("newnodeid")) + return std::nullopt; + + try + { + auto setup = setupDatabaseCon(config, journal); + + // Standalone uses a temporary database, so nothing is persisted and this + // run will mint a fresh key. + if (setup.standAlone && setup.startUp != StartUpType::Load && + setup.startUp != StartUpType::LoadFile && setup.startUp != StartUpType::Replay) + { + return std::nullopt; + } + + // The global pragmas include journal_mode, which rewrites the database + // header. The wallet is opened without them everywhere else. + setup.useGlobalPragma = false; + + // Only read an existing file: SQLite would otherwise create one. + if (std::error_code ec; !std::filesystem::exists(setup.dataDir / kWalletDbName, ec)) + { + return std::nullopt; + } + + // Empty init SQL: open the existing schema, never create it. + DatabaseCon walletDb{ + setup, + kWalletDbName, + std::array{}, + std::array{}, + journal}; + + auto db = walletDb.checkoutDb(); + if (auto const stored = readNodeIdentity(*db)) + return toBase58(TokenType::NodePublic, stored->first); + } + catch (std::exception const& e) + { + JLOG(journal.warn()) << "Could not read the node identity: " << e.what(); + } + + return std::nullopt; +} + } // namespace xrpl diff --git a/src/xrpld/app/main/NodeIdentity.h b/src/xrpld/app/main/NodeIdentity.h index 117acffdb1..7309f6007a 100644 --- a/src/xrpld/app/main/NodeIdentity.h +++ b/src/xrpld/app/main/NodeIdentity.h @@ -1,12 +1,16 @@ #pragma once #include +#include +#include #include #include #include +#include +#include #include namespace xrpl { @@ -20,4 +24,26 @@ namespace xrpl { std::pair getNodeIdentity(Application& app, boost::program_options::variables_map const& cmdline); +/** + * This server's public key, read without creating or modifying anything. + * + * For callers that need the identity before the Application exists, such as + * telemetry building its resource attributes in the member-init list. Derives + * from a configured seed when there is one, otherwise reads the wallet database + * only if it already exists. + * + * getNodeIdentity() remains authoritative and mints a key when none exists. + * + * @param config The server configuration. + * @param cmdline The command line parameters passed into the application. + * @param journal Journal for reporting an unreadable database. + * @return The base58-encoded node public key, or std::nullopt if none can be + * read. + */ +std::optional +resolveNodePublicKey( + Config const& config, + boost::program_options::variables_map const& cmdline, + beast::Journal journal); + } // namespace xrpl diff --git a/src/xrpld/rpc/detail/ServerHandler.cpp b/src/xrpld/rpc/detail/ServerHandler.cpp index 0376345611..ebf318b247 100644 --- a/src/xrpld/rpc/detail/ServerHandler.cpp +++ b/src/xrpld/rpc/detail/ServerHandler.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -183,7 +184,11 @@ ServerHandler::ServerHandler( { auto const& group(cm.group("rpc")); rpcRequests_ = group->makeCounter("requests"); - rpcSize_ = group->makeEvent("size"); + // "size" measures the serialized response in bytes, not a duration. It + // has to say so: the unit picks both the exported name suffix and the + // histogram bucket ladder, and borrowing the millisecond ladder censored + // a quarter of these samples. + rpcSize_ = group->makeEvent("size", beast::insight::Unit::Bytes); rpcTime_ = group->makeEvent("time"); } @@ -1125,7 +1130,7 @@ ServerHandler::processRequest( std::chrono::duration_cast( std::chrono::high_resolution_clock::now() - start)); ++rpcRequests_; - rpcSize_.notify(beast::insight::Event::value_type{response.size()}); + rpcSize_.notify(static_cast(response.size())); response += '\n'; diff --git a/src/xrpld/telemetry/MetricsRegistry.cpp b/src/xrpld/telemetry/MetricsRegistry.cpp index 1e2f9ce6dd..8642d31b23 100644 --- a/src/xrpld/telemetry/MetricsRegistry.cpp +++ b/src/xrpld/telemetry/MetricsRegistry.cpp @@ -68,6 +68,8 @@ #include #include #include +#include +#include #include #include @@ -88,7 +90,6 @@ #include #include -#include #include #include #include @@ -131,36 +132,16 @@ constexpr char kRpcMethodDurationUs[] = "rpc_method_us"; // after start(). constexpr char kConsensusRoundDurationMs[] = "consensus_round_duration_ms"; -/** - * Bucket boundaries for microsecond-valued duration instruments. - * - * 100 µs, 500 µs, 1 ms, 5 ms, 10 ms, 25 ms, 50 ms, 100 ms, 250 ms, 500 ms, - * 1 s, 2.5 s, 5 s, 10 s, 30 s, 60 s. Covers sub-millisecond jobs through - * multi-second stalls without saturating. - */ -constexpr std::array kMicrosecondBoundaries{ - 100.0, - 500.0, - 1'000.0, - 5'000.0, - 10'000.0, - 25'000.0, - 50'000.0, - 100'000.0, - 250'000.0, - 500'000.0, - 1'000'000.0, - 2'500'000.0, - 5'000'000.0, - 10'000'000.0, - 30'000'000.0, - 60'000'000.0}; - /** * Register an explicit-bucket histogram view. * * The SDK's default boundaries top out at 10,000, so any instrument whose - * values exceed that saturates and every quantile reads as the ceiling. + * values exceed that saturates and every quantile reads as the ceiling. The + * floor matters just as much and is easier to miss: a ladder whose first edge + * sits above the mass of the distribution makes every low quantile an + * interpolation inside bucket 0 -- a number derived from the bucket edge + * rather than from any sample. Both ends are chosen from measured + * distributions in HistogramBuckets.h. * * @param views The registry to add the view to. * @param name Instrument name to match (e.g. "job_running_us"). @@ -188,9 +169,7 @@ addHistogramView( * Register the microsecond-ladder view for a duration instrument. * * Job wait/run times and RPC latencies routinely exceed the SDK default - * ceiling, so they all share `kMicrosecondBoundaries`: 100µs, 500µs, 1ms, 5ms, - * 10ms, 25ms, 50ms, 100ms, 250ms, 500ms, 1s, 2.5s, 5s, 10s, 30s, 60s — - * sub-millisecond jobs through multi-second stalls, without saturating. + * ceiling, so they all share `buckets::kMicrosecondBuckets`. * * @param views The registry to add the view to. * @param name Instrument name to match (e.g. "job_running_us"). @@ -198,7 +177,10 @@ addHistogramView( void addMicrosecondHistogramView(metric_sdk::ViewRegistry& views, std::string const& name) { - addHistogramView(views, name, {kMicrosecondBoundaries.begin(), kMicrosecondBoundaries.end()}); + addHistogramView( + views, + name, + xrpl::telemetry::buckets::toVector(xrpl::telemetry::buckets::kMicrosecondBuckets)); } /** @@ -266,14 +248,17 @@ MetricsRegistry::~MetricsRegistry() } void -MetricsRegistry::start(std::string const& endpoint, std::string const& instanceId) +MetricsRegistry::start( + std::string const& endpoint, + std::string const& instanceId, + std::string const& nodeId) { #ifdef XRPL_ENABLE_TELEMETRY if (!enabled_) return; JLOG(journal_.info()) << "MetricsRegistry: starting, endpoint=" << endpoint - << ", instanceId=" << instanceId; + << ", instanceId=" << instanceId << ", nodeId=" << nodeId; // Rule for anything added below: this phase may create only instruments // whose recording is PUSHED from app code -- counters and histograms. An @@ -283,13 +268,14 @@ MetricsRegistry::start(std::string const& endpoint, std::string const& instanceI // belongs in startAsyncGauges(), not here. That includes observable // COUNTERS, not just gauges: jq_trans_overflow_total was created here and // its callback read getOverlay(), which asserts overlay_ is non-null. - initExporterAndProvider(endpoint, instanceId); + initExporterAndProvider(endpoint, instanceId, nodeId); initSyncInstruments(); JLOG(journal_.info()) << "MetricsRegistry: provider and instruments ready"; #else (void)endpoint; (void)instanceId; + (void)nodeId; (void)enabled_; #endif // XRPL_ENABLE_TELEMETRY } @@ -320,7 +306,10 @@ MetricsRegistry::startAsyncGauges() #ifdef XRPL_ENABLE_TELEMETRY void -MetricsRegistry::initExporterAndProvider(std::string const& endpoint, std::string const& instanceId) +MetricsRegistry::initExporterAndProvider( + std::string const& endpoint, + std::string const& instanceId, + std::string const& nodeId) { // Configure OTLP/HTTP metric exporter. otlp_http::OtlpHttpMetricExporterOptions exporterOpts; @@ -344,6 +333,11 @@ MetricsRegistry::initExporterAndProvider(std::string const& endpoint, std::strin attrs[opentelemetry::semconv::service::kServiceName] = std::string("xrpld"); if (!instanceId.empty()) attrs[opentelemetry::semconv::service::kServiceInstanceId] = instanceId; + // xrpl.node.id: the same per-node key the trace resource carries, so + // metrics and traces resolve to one node. std::string for the same + // variant reason as service.name above. + if (!nodeId.empty()) + attrs[std::string(attr::nodeId)] = nodeId; auto resourceAttrs = otel_resource::Resource::Create(attrs); // Build a view registry with explicit buckets for the duration @@ -425,18 +419,13 @@ MetricsRegistry::initExporterAndProvider(std::string const& endpoint, std::strin // asks for at most 8, so the low buckets are fine-grained and the upper // ones follow the charge size bands (64, 1024) up to the hard cap. addHistogramView( - *views, - kGetObjectRequestObjects, - {1.0, 2.0, 4.0, 8.0, 16.0, 64.0, 256.0, 1'024.0, 4'096.0, 12'288.0}); + *views, kGetObjectRequestObjects, buckets::toVector(buckets::kObjectCountBuckets)); // Charge values span 0 (free tier) to ~99k for a full-size all-miss // request. Boundaries bracket the resource thresholds that decide a // peer's fate -- kWarningThreshold (5000) and kDropThreshold (25000) -- // so a dashboard can show how close charges run to each. - addHistogramView( - *views, - kGetObjectCharge, - {0.0, 100.0, 500.0, 1'000.0, 5'000.0, 10'000.0, 25'000.0, 50'000.0, 100'000.0}); + addHistogramView(*views, kGetObjectCharge, buckets::toVector(buckets::kChargeBuckets)); // Create MeterProvider with resource, then attach the metric reader. provider_ = metric_sdk::MeterProviderFactory::Create(std::move(views), resourceAttrs); diff --git a/src/xrpld/telemetry/MetricsRegistry.h b/src/xrpld/telemetry/MetricsRegistry.h index fce4a56d1f..5609a58982 100644 --- a/src/xrpld/telemetry/MetricsRegistry.h +++ b/src/xrpld/telemetry/MetricsRegistry.h @@ -295,9 +295,15 @@ public: * attribute. When non-empty, Prometheus metrics * carry a service_instance_id label for per-node * filtering. + * @param nodeId Value for the xrpl.node.id resource attribute (the + * node's base58 public key). When non-empty, metrics + * carry the same per-node key that traces do. */ void - start(std::string const& endpoint, std::string const& instanceId = {}); + start( + std::string const& endpoint, + std::string const& instanceId = {}, + std::string const& nodeId = {}); /** * Register the pull-model observable instruments — the second startup @@ -1063,9 +1069,13 @@ private: * * @param endpoint OTLP/HTTP metrics endpoint URL. * @param instanceId service.instance.id resource attribute (may be empty). + * @param nodeId xrpl.node.id resource attribute (may be empty). */ void - initExporterAndProvider(std::string const& endpoint, std::string const& instanceId); + initExporterAndProvider( + std::string const& endpoint, + std::string const& instanceId, + std::string const& nodeId); /** * Create the synchronous instruments (RPC and job-queue counters and