mirror of
https://github.com/XRPLF/rippled.git
synced 2026-08-23 15:20:54 +00:00
Merge branch 'pratik/otel-phase9-metric-gap-fill' into pratik/otel-phase10-workload-validation
This commit is contained in:
Pratik Mankawde
128
.github/scripts/telemetry/check_bucket_parity.py
vendored
Executable file
128
.github/scripts/telemetry/check_bucket_parity.py
vendored
Executable file
@@ -0,0 +1,128 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Assert the C++ millisecond ladder agrees with the collector's spanmetrics ladder.
|
||||
|
||||
The two are specified to match so a span-derived latency panel and a native
|
||||
histogram panel can be read on the same scale. They *were* identical when first
|
||||
shipped. Then the collector ladder alone was extended -- sub-millisecond edges
|
||||
below 1ms and second-scale edges up to 30s -- and nothing checked the other
|
||||
side, so the C++ ladder stayed capped at 5s. Every quantile above 5s then read
|
||||
back as a flat 5000, because Prometheus returns the second-highest edge for a
|
||||
quantile landing in the `+Inf` bucket. That looks like a measurement rather
|
||||
than an error, which is why it survived for eleven phases.
|
||||
|
||||
The rule is containment, not equality:
|
||||
|
||||
* every representable collector edge MUST appear in the C++ ladder, so the
|
||||
shared range reads identically;
|
||||
* the C++ ladder MAY carry extra edges ABOVE the collector's highest edge,
|
||||
because jobs outlive spans -- the updatepaths job type was measured
|
||||
averaging ~60s, which no span approaches. Demanding equality would force a
|
||||
ceiling that censors it, reintroducing the bug this guards against;
|
||||
* collector edges below 1ms are expected to be ABSENT rather than missing:
|
||||
beast::insight::Event rounds every duration up to a whole millisecond
|
||||
before it reaches the histogram, so those edges could never collect a
|
||||
sample.
|
||||
|
||||
Exit 0 when the ladders agree, 1 with a diff when they do not.
|
||||
"""
|
||||
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
HEADER = Path("include/xrpl/telemetry/HistogramBuckets.h")
|
||||
COLLECTOR = Path("docker/telemetry/otel-collector-config.yaml")
|
||||
|
||||
# beast::insight::Event applies ceil<milliseconds>, so anything below 1ms
|
||||
# collapses onto the 1ms edge.
|
||||
REPRESENTABLE_FLOOR_MS = 1.0
|
||||
|
||||
UNIT_TO_MS = {"ms": 1.0, "s": 1000.0}
|
||||
|
||||
|
||||
def collector_edges_ms():
|
||||
"""Parse the spanmetrics bucket list, normalising each edge to milliseconds."""
|
||||
text = COLLECTOR.read_text()
|
||||
match = re.search(r"buckets:\s*\[(.*?)\]", text, re.S)
|
||||
if not match:
|
||||
sys.exit(f"{COLLECTOR}: no 'buckets:' list found")
|
||||
|
||||
edges = []
|
||||
for raw in match.group(1).split(","):
|
||||
token = raw.strip()
|
||||
if not token:
|
||||
continue
|
||||
parsed = re.fullmatch(r"([0-9.]+)(ms|s)", token)
|
||||
if not parsed:
|
||||
sys.exit(f"{COLLECTOR}: cannot parse bucket edge {token!r}")
|
||||
edges.append(float(parsed.group(1)) * UNIT_TO_MS[parsed.group(2)])
|
||||
return edges
|
||||
|
||||
|
||||
def cpp_edges_ms():
|
||||
"""Parse kMillisecondBuckets out of the header that owns every ladder."""
|
||||
text = HEADER.read_text()
|
||||
match = re.search(r"kMillisecondBuckets\{(.*?)\};", text, re.S)
|
||||
if not match:
|
||||
sys.exit(f"{HEADER}: kMillisecondBuckets not found")
|
||||
return [
|
||||
float(token.strip().replace("'", ""))
|
||||
for token in match.group(1).split(",")
|
||||
if token.strip()
|
||||
]
|
||||
|
||||
|
||||
def main():
|
||||
collector = collector_edges_ms()
|
||||
cpp = cpp_edges_ms()
|
||||
required = [edge for edge in collector if edge >= REPRESENTABLE_FLOOR_MS]
|
||||
if not required:
|
||||
sys.exit(f"{COLLECTOR}: no edges at or above {REPRESENTABLE_FLOOR_MS} ms")
|
||||
collector_top = max(required)
|
||||
|
||||
missing = [edge for edge in required if edge not in cpp]
|
||||
# An extra C++ edge inside the collector's range means the two scales
|
||||
# disagree where they overlap. Above the collector's top it is a deliberate
|
||||
# extension.
|
||||
inside_range = [e for e in cpp if e not in required and e < collector_top]
|
||||
|
||||
if not missing and not inside_range:
|
||||
extensions = [e for e in cpp if e > collector_top]
|
||||
summary = f"OK: all {len(required)} representable collector edges present"
|
||||
if extensions:
|
||||
pretty = ", ".join(f"{e:g}" for e in extensions)
|
||||
summary += (
|
||||
f"; {len(extensions)} extension edge(s) above "
|
||||
f"{collector_top:g} ms: [{pretty}]"
|
||||
)
|
||||
print(summary)
|
||||
return 0
|
||||
|
||||
print("Bucket ladder parity violated.", file=sys.stderr)
|
||||
print(
|
||||
f" collector (>= {REPRESENTABLE_FLOOR_MS:g} ms): "
|
||||
f"{[f'{e:g}' for e in required]}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
print(
|
||||
f" HistogramBuckets.h : {[f'{e:g}' for e in cpp]}", file=sys.stderr
|
||||
)
|
||||
for edge in missing:
|
||||
print(f" MISSING from the C++ ladder: {edge:g} ms", file=sys.stderr)
|
||||
for edge in inside_range:
|
||||
print(
|
||||
f" C++ edge {edge:g} ms lies inside the collector's range but is not "
|
||||
"a collector edge -- add it to the collector or drop it here",
|
||||
file=sys.stderr,
|
||||
)
|
||||
print(
|
||||
"\nThe two ladders must agree over their shared range. Extra C++ edges are\n"
|
||||
"permitted only ABOVE the collector's highest edge. Change both sides, or\n"
|
||||
"change the spec in OpenTelemetryPlan/Phase7_taskList.md.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
1
.github/workflows/on-pr.yml
vendored
1
.github/workflows/on-pr.yml
vendored
@@ -72,6 +72,7 @@ jobs:
|
||||
.github/scripts/levelization/**
|
||||
.github/scripts/otel-naming/**
|
||||
.github/scripts/rename/**
|
||||
.github/scripts/telemetry/**
|
||||
.github/workflows/reusable-check-levelization.yml
|
||||
.github/workflows/reusable-check-otel-naming.yml
|
||||
.github/workflows/reusable-check-rename.yml
|
||||
|
||||
@@ -33,3 +33,11 @@ jobs:
|
||||
# it enforces each rule only when the layer it needs is present, so it
|
||||
# works whether telemetry changes land in one PR or several.
|
||||
run: python .github/scripts/otel-naming/check_otel_naming.py
|
||||
- name: Check histogram bucket parity
|
||||
# The C++ millisecond ladder and the collector's spanmetrics ladder are
|
||||
# specified to agree over their shared range. They were identical when
|
||||
# first shipped, then the collector side alone was extended and nothing
|
||||
# noticed for eleven phases: native histograms stayed capped at 5s while
|
||||
# spans reached 30s, so every quantile above 5s reported a flat 5000.
|
||||
# Nothing but a check keeps two lists in step.
|
||||
run: python .github/scripts/telemetry/check_bucket_parity.py
|
||||
|
||||
@@ -531,12 +531,12 @@ a destructor must not depend on still existing. A query that only groups by
|
||||
|
||||
The OTel Collector's SpanMetrics connector automatically generates RED (Rate, Errors, Duration) metrics from every span. No custom metrics code in xrpld is needed.
|
||||
|
||||
| Prometheus Metric | Type | Description |
|
||||
| ----------------------------------- | --------- | ------------------------------------------------------------------------------ |
|
||||
| `span_calls_total` | Counter | Total span invocations |
|
||||
| `span_duration_milliseconds_bucket` | Histogram | Latency distribution (buckets: 1, 5, 10, 25, 50, 100, 250, 500, 1000, 5000 ms) |
|
||||
| `span_duration_milliseconds_count` | Histogram | Observation count |
|
||||
| `span_duration_milliseconds_sum` | Histogram | Cumulative latency |
|
||||
| Prometheus Metric | Type | Description |
|
||||
| ----------------------------------- | --------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `span_calls_total` | Counter | Total span invocations |
|
||||
| `span_duration_milliseconds_bucket` | Histogram | Latency distribution. Buckets come from the collector's spanmetrics config: 0.01, 0.05, 0.1, 0.25, 0.5, 1, 5, 10, 25, 50, 100, 250, 500 ms then 1, 2, 3, 4, 5, 10, 30 s. The sub-millisecond edges exist because most xrpld spans are far below 1 ms; without them every p95/p99 pinned to a constant 0.95 ms |
|
||||
| `span_duration_milliseconds_count` | Histogram | Observation count |
|
||||
| `span_duration_milliseconds_sum` | Histogram | Cumulative latency |
|
||||
|
||||
**Standard labels on every metric**: `span_name`, `status_code`, `service_name`, `span_kind`
|
||||
|
||||
@@ -684,13 +684,14 @@ prefix=xrpld
|
||||
|
||||
Quantiles collected: 0th, 50th, 90th, 95th, 99th, 100th percentile.
|
||||
|
||||
\* **`rpc_size` instrument mismatch (known issue):** response size in bytes is
|
||||
recorded through the millisecond-scaled event histogram (`makeEvent`), so it is
|
||||
exported as `rpc_size_milliseconds_bucket` with time-scaled boundaries that top
|
||||
out at 5000. Byte values above ~5 KB saturate in the last bucket, so the
|
||||
percentiles are not true byte sizes. The _RPC & Pathfinding_ panel is flagged
|
||||
accordingly. A dedicated byte-unit histogram is needed to fix this; tracked
|
||||
separately.
|
||||
\* **`rpc_size` now records bytes as bytes (fixed).** It used to go through the
|
||||
millisecond-scaled event histogram and export as `rpc_size_milliseconds_bucket`
|
||||
on a ladder topping out at 5000, so the 24.9% of responses larger than 5 kB all
|
||||
landed in the last bucket and every percentile read back as a flat 5000 — a
|
||||
plausible-looking constant rather than a byte size. `beast::insight::Event` now
|
||||
declares a `Unit`, so this instrument is created with unit `By` and exports as
|
||||
**`rpc_size_bytes_bucket`** on `kByteBuckets` (512 B to 1 MiB, placed from the
|
||||
measured distribution). Queries and panels must use the new name.
|
||||
|
||||
**Grafana dashboards**: _Node Health_ (`ios_latency`), _RPC & Pathfinding_ (`rpc_time`, `rpc_size`, `pathfind_*`)
|
||||
|
||||
|
||||
@@ -53,7 +53,7 @@
|
||||
- **OTelCounterImpl**: Wraps `opentelemetry::metrics::Counter<int64_t>`. `increment(amount)` calls `counter->Add(amount)`.
|
||||
- **OTelGaugeImpl**: Uses `opentelemetry::metrics::ObservableGauge<uint64_t>` with an async callback. `set(value)` stores value atomically; callback reads it during collection.
|
||||
- **OTelMeterImpl**: Wraps `opentelemetry::metrics::Counter<uint64_t>`. `increment(amount)` calls `counter->Add(amount)`. Semantically identical to Counter but unsigned.
|
||||
- **OTelEventImpl**: Wraps `opentelemetry::metrics::Histogram<double>`. `notify(duration)` calls `histogram->Record(duration.count())`. Uses explicit bucket boundaries matching SpanMetrics: [1, 5, 10, 25, 50, 100, 250, 500, 1000, 5000] ms.
|
||||
- **OTelEventImpl**: Wraps `opentelemetry::metrics::Histogram<double>`. `notify()` calls `histogram->Record(value.count())`. Declares its unit from `beast::insight::Unit`, which is what selects its bucket ladder: the histogram views in `Telemetry.cpp` match on unit, so a `ms` instrument gets the millisecond ladder and a `By` instrument the byte ladder. Bucket edges live in `include/xrpl/telemetry/HistogramBuckets.h` — do not restate them here. The millisecond ladder must contain every representable edge of the collector's spanmetrics ladder and may extend above it (jobs outlive spans); `.github/scripts/telemetry/check_bucket_parity.py` enforces that. An earlier version of this line specified `[1, 5, 10, 25, 50, 100, 250, 500, 1000, 5000] ms` as "matching SpanMetrics" — true when written, then silently false once the collector ladder was extended on its own, which capped every quantile above 5s at a flat 5000.
|
||||
- **OTelHookImpl**: Stores handler function. Called during periodic metric collection (same 1s pattern via PeriodicMetricReader).
|
||||
- **OTelCollectorImp**: Main class.
|
||||
- Creates `MeterProvider` with `PeriodicMetricReader` (1s export interval)
|
||||
|
||||
@@ -1251,7 +1251,7 @@
|
||||
},
|
||||
{
|
||||
"title": "Job Queue Wait p95 By Type",
|
||||
"description": "###### What this is:\n*95th-percentile time a job waits in the queue before a worker thread picks it up, for the sync-critical job types. This is the metric form of the 'ProcessLData wait: NNNNms' warnings in the debug log.*\n\n###### How it's computed:\n*histogram_quantile(0.95, rate(jobq_<type>_q_milliseconds_bucket[$__rate_interval])) for ledgerdata, acceptledger, fetchtxndata, transaction, advanceledger, ledgerrequest.*\n\n###### Reading it:\n*Queue wait should be single-digit to low-tens of ms. High ledgerdata/fetchtxndata wait = the node cannot process inbound ledger data fast enough.*\n\n###### Healthy range:\n*< ~50ms p95 per type on a healthy node.*\n\n###### Watch for:\n*ledgerdata or fetchtxndata q-wait spiking to seconds = worker threads are blocked (usually on NuDB reads - see the cause tier).*\n\n###### Keywords:\n- **Job queue / job type** *(per node)* \u2014 xrpld's worker-thread pool; every unit of background work is enqueued under a named job type.\n- **Deferred job** *(per node)* \u2014 a job held back because its type is already at its concurrency limit; the leading indicator of queue backpressure.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[core/JobQueue.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/core/detail/JobQueue.cpp)\n\n###### Function:\n`JobQueue::getJson (per-type queue timing)`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#job-queue-job-type)",
|
||||
"description": "###### What this is:\n*95th-percentile time a job waits in the queue before a worker thread picks it up, for the sync-critical job types. This is the metric form of the 'ProcessLData wait: NNNNms' warnings in the debug log.*\n\n###### How it's computed:\n*histogram_quantile(0.95, rate(job_queued_us_bucket{job_type=\"<type>\"}[$__rate_interval])) for ledgerData, acceptLedger, fetchTxnData, transaction, advanceLedger, ledgerRequest. Reads the OTel-native microsecond instrument rather than the beast jobq_* pair: beast Events round every duration up to a whole millisecond, so 94-100% of their samples landed in the first bucket and every percentile was an interpolation inside it rather than a measurement.*\n\n###### Reading it:\n*Queue wait is normally tens to hundreds of microseconds. High ledgerData/fetchTxnData wait = the node cannot process inbound ledger data fast enough.*\n\n###### Healthy range:\n*< ~500us p95 per type on a healthy node; sustained milliseconds is already backpressure.*\n\n###### Watch for:\n*ledgerData or fetchTxnData q-wait spiking to seconds = worker threads are blocked (usually on NuDB reads - see the cause tier).*\n\n###### Keywords:\n- **Job queue / job type** *(per node)* \u2014 xrpld's worker-thread pool; every unit of background work is enqueued under a named job type.\n- **Deferred job** *(per node)* \u2014 a job held back because its type is already at its concurrency limit; the leading indicator of queue backpressure.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code by MetricsRegistry as an OTel-native histogram in microseconds; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[core/JobQueue.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/core/detail/JobQueue.cpp)\n\n###### Function:\n`JobQueue::getJson (per-type queue timing)`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#job-queue-job-type)",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"h": 10,
|
||||
@@ -1272,48 +1272,48 @@
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(jobq_ledgerdata_q_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"ledgerdata q-wait p95\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
|
||||
"expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(job_queued_us_bucket{job_type=\"ledgerData\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"ledgerData q-wait p95\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(jobq_acceptledger_q_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"acceptledger q-wait p95\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
|
||||
"expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(job_queued_us_bucket{job_type=\"acceptLedger\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"acceptLedger q-wait p95\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(jobq_fetchtxndata_q_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"fetchtxndata q-wait p95\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
|
||||
"expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(job_queued_us_bucket{job_type=\"fetchTxnData\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"fetchTxnData q-wait p95\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(jobq_transaction_q_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"transaction q-wait p95\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
|
||||
"expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(job_queued_us_bucket{job_type=\"transaction\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"transaction q-wait p95\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(jobq_advanceledger_q_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"advanceledger q-wait p95\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
|
||||
"expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(job_queued_us_bucket{job_type=\"advanceLedger\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"advanceLedger q-wait p95\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(jobq_ledgerrequest_q_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"ledgerrequest q-wait p95\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
|
||||
"expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(job_queued_us_bucket{job_type=\"ledgerRequest\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"ledgerRequest q-wait p95\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}",
|
||||
"unit": "ms",
|
||||
"unit": "µs",
|
||||
"custom": {
|
||||
"axisLabel": "p95 Wait (ms)",
|
||||
"spanNulls": 1800000,
|
||||
|
||||
@@ -154,7 +154,7 @@
|
||||
},
|
||||
{
|
||||
"title": "RPC Response Size",
|
||||
"description": "\u26a0 Instrument mismatch \u2014 values unreliable. Response size is recorded through the millisecond-scaled event histogram (rpc_size_milliseconds_bucket), so byte values saturate at the top time bucket (5000) and the percentiles are not true byte sizes. A dedicated byte-unit histogram is needed to fix this; tracked separately. Treat this panel as indicative only until then.\n\n###### What this is:\n*The 95th-percentile size of RPC response payloads in bytes.*\n\n###### How it's computed:\n*95th-percentile of response payload sizes over the dashboard rate interval, per node.*\n\n###### Reading it:\n*Smaller is cheaper; large responses cost bandwidth and memory.*\n\n###### Healthy range:\n*Workload-dependent; small for status queries, large for bulk data queries.*\n\n###### Watch for:\n*Growth in large responses, consistent with expensive queries or API misuse.*\n\n###### Keywords:\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[ServerHandler.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/ServerHandler.cpp)\n\n###### Function:\n`ServerHandler ctor`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rpc-command-method)",
|
||||
"description": "\u26a0 Instrument mismatch \u2014 values unreliable. Response size is recorded through the millisecond-scaled event histogram (rpc_size_bytes_bucket), so byte values saturate at the top time bucket (5000) and the percentiles are not true byte sizes. A dedicated byte-unit histogram is needed to fix this; tracked separately. Treat this panel as indicative only until then.\n\n###### What this is:\n*The 95th-percentile size of RPC response payloads in bytes.*\n\n###### How it's computed:\n*95th-percentile of response payload sizes over the dashboard rate interval, per node.*\n\n###### Reading it:\n*Smaller is cheaper; large responses cost bandwidth and memory.*\n\n###### Healthy range:\n*Workload-dependent; small for status queries, large for bulk data queries.*\n\n###### Watch for:\n*Growth in large responses, consistent with expensive queries or API misuse.*\n\n###### Keywords:\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[ServerHandler.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/ServerHandler.cpp)\n\n###### Function:\n`ServerHandler ctor`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rpc-command-method)",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"h": 10,
|
||||
@@ -175,7 +175,7 @@
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(rpc_size_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"P95 Response Size\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
|
||||
"expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(rpc_size_bytes_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"P95 Response Size\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
|
||||
@@ -480,6 +480,15 @@ groups:
|
||||
# p99 time a job waits in the queue before running. A sustained p99
|
||||
# above 1s means the node is saturated and work is backing up. `le` must
|
||||
# stay inside the inner sum or histogram_quantile cannot interpolate.
|
||||
#
|
||||
# Threshold re-validated after the microsecond ladder was re-cut. Do NOT
|
||||
# tune it down against a casual reading of this p99: before that change
|
||||
# the ladder's first edge was 100us with 99.3% of samples beneath it, so
|
||||
# p99 reported 99.7us -- the bucket edge scaled by the quantile, not a
|
||||
# latency. Measured cumulative distribution: 99.26% of samples land
|
||||
# within 100us, 99.969% within 5ms, 99.990% within 100ms, and only
|
||||
# 0.0022% exceed 1s. So 1s sits about four orders of magnitude above the
|
||||
# healthy p99 and fires only on genuine saturation, which is the intent.
|
||||
- uid: xrpld-jobqueue-latency-high
|
||||
title: JobQueueLatencyHigh
|
||||
condition: C
|
||||
@@ -544,6 +553,12 @@ groups:
|
||||
# first and explains the others. Measured p99-of-p95 is 37-49ms on
|
||||
# healthy nodes and 488-566ms on nodes that are actively flapping, so
|
||||
# 1000ms flags genuine degradation rather than the current baseline.
|
||||
#
|
||||
# Still valid after the millisecond ladder was extended: that change only
|
||||
# ADDED edges above 5s (2s/3s/4s/10s/30s/60s/120s) and removed none, so
|
||||
# every edge bracketing this threshold -- 25/50/100/250/500/1000ms -- is
|
||||
# unchanged and the measurements above still hold. ios_latency's own mean
|
||||
# is 12.9ms, far below the threshold.
|
||||
- uid: xrpld-nodestore-io-latency-high
|
||||
title: NodeStoreIOLatencyHigh
|
||||
condition: C
|
||||
|
||||
@@ -2264,22 +2264,68 @@ Requires `trace_peer=1` in the `[telemetry]` config section.
|
||||
| ------------------------- | ---------- | ------------------------------------------------------------- | ----------- |
|
||||
| RPC Request Rate | stat | `rate(rpc_requests[5m])` | — |
|
||||
| RPC Response Time | timeseries | `histogram_quantile(0.95, rpc_time_milliseconds_bucket)` | — |
|
||||
| RPC Response Size | timeseries | `histogram_quantile(0.95, rpc_size_milliseconds_bucket)` | — |
|
||||
| RPC Response Size | timeseries | `histogram_quantile(0.95, rpc_size_bytes_bucket)` | — |
|
||||
| RPC Response Time Heatmap | heatmap | `rpc_time_milliseconds_bucket` | — |
|
||||
| Pathfinding Fast Duration | timeseries | `histogram_quantile(0.95, pathfind_fast_milliseconds_bucket)` | — |
|
||||
| Pathfinding Full Duration | timeseries | `histogram_quantile(0.95, pathfind_full_milliseconds_bucket)` | — |
|
||||
| Resource Warnings Rate | stat | `rate(warn_total[$__rate_interval])` | — |
|
||||
| Resource Drops Rate | stat | `rate(drop_total[$__rate_interval])` | — |
|
||||
|
||||
> **The `_milliseconds` suffix comes from the exporter, not from xrpld.** These
|
||||
> histograms are created with unit `"ms"`
|
||||
> ([OTelCollector.cpp:615](../src/libxrpl/beast/insight/OTelCollector.cpp#L615)),
|
||||
> so the Prometheus exporter appends the unit to the family name — `rpc_time`
|
||||
> becomes `rpc_time_milliseconds_bucket`. Querying the bare `rpc_time_bucket`,
|
||||
> **The unit suffix comes from the exporter, not from xrpld.** Each histogram
|
||||
> declares a unit, and the Prometheus exporter appends the unit's name to the
|
||||
> family name — a `ms` instrument like `rpc_time` becomes
|
||||
> `rpc_time_milliseconds_bucket`. Querying the bare `rpc_time_bucket`,
|
||||
> `ios_latency_bucket` or `pathfind_fast_bucket` returns no data and no error.
|
||||
> **Known issue**: `rpc_size` counts bytes but shares the same `"ms"` histogram
|
||||
> constructor, so it is exported as `rpc_size_milliseconds_bucket` — the suffix
|
||||
> is wrong, the name is nonetheless the one to query.
|
||||
>
|
||||
> The unit an `Event` declares also selects its bucket ladder, because the
|
||||
> histogram views match on unit. `rpc_size` measures bytes, so it declares
|
||||
> `Unit::Bytes` and exports as **`rpc_size_bytes_bucket`** on the byte ladder.
|
||||
> It used to share the `ms` constructor and export as
|
||||
> `rpc_size_milliseconds_bucket` on a latency ladder — if you find that name in
|
||||
> an old query or bookmark, it no longer exists.
|
||||
|
||||
#### Reading A Histogram Percentile
|
||||
|
||||
Two failure modes make a percentile panel lie, and neither looks like an error —
|
||||
both produce a believable number. Check for them before trusting any p95/p99.
|
||||
|
||||
**Saturated at the top.** If the quantile falls in the `+Inf` bucket, Prometheus
|
||||
returns the **second-highest** bucket edge, not `+Inf`. A panel pinned to a round
|
||||
number that happens to equal the ladder's top edge is the signature. Confirm by
|
||||
comparing the top finite bucket against the total:
|
||||
|
||||
```promql
|
||||
1 - (
|
||||
sum(last_over_time(<metric>_bucket{le="<top edge>"}[15m]))
|
||||
/ sum(last_over_time(<metric>_bucket{le="+Inf"}[15m]))
|
||||
)
|
||||
```
|
||||
|
||||
A non-trivial result means samples are being censored and the percentile is a
|
||||
lower bound, not a measurement.
|
||||
|
||||
**Saturated at the bottom.** If nearly every sample lands in the first bucket,
|
||||
`histogram_quantile` interpolates _inside_ it and returns
|
||||
`quantile / fraction_in_bucket_0 × first_edge`. The signature is a p75/p95/p99
|
||||
that sit in near-constant proportion to each other and to the first edge — for
|
||||
example 75.5 / 95.7 / 99.7 against a 100 µs floor. Confirm with:
|
||||
|
||||
```promql
|
||||
sum(last_over_time(<metric>_bucket{le="<first edge>"}[15m]))
|
||||
/ sum(last_over_time(<metric>_bucket{le="+Inf"}[15m]))
|
||||
```
|
||||
|
||||
Anything close to 1 means the panel is reporting arithmetic on the bucket edge.
|
||||
|
||||
**After a ladder change, expect a discontinuity.** Existing series keep their old
|
||||
`le` values, so a percentile panel shows a step at the restart that introduced
|
||||
new edges. That break is the ladder changing, not an incident.
|
||||
|
||||
Bucket edges for the native instruments live in one place —
|
||||
[`include/xrpl/telemetry/HistogramBuckets.h`](../include/xrpl/telemetry/HistogramBuckets.h).
|
||||
The millisecond ladder is required to contain every representable edge of the
|
||||
collector's spanmetrics ladder; `.github/scripts/telemetry/check_bucket_parity.py`
|
||||
enforces that in CI, because the two silently drifted once already.
|
||||
|
||||
### Span → Metric → Dashboard Summary
|
||||
|
||||
|
||||
@@ -66,4 +66,29 @@ otelUnitCode(Unit unit) noexcept
|
||||
return "ms";
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Human-readable description for an instrument of this unit.
|
||||
*
|
||||
* Exported alongside the metric, so this is the text an operator reads in a
|
||||
* metric catalogue. A byte-valued instrument that describes itself as a
|
||||
* duration is exactly the confusion this whole type exists to remove, so the
|
||||
* description is derived from the unit rather than written out at each
|
||||
* instrument site.
|
||||
*
|
||||
* @param unit The unit to describe.
|
||||
* @return A static, null-terminated description.
|
||||
*/
|
||||
constexpr char const*
|
||||
otelUnitDescription(Unit unit) noexcept
|
||||
{
|
||||
switch (unit)
|
||||
{
|
||||
case Unit::Bytes:
|
||||
return "Size in bytes";
|
||||
case Unit::Millis:
|
||||
break;
|
||||
}
|
||||
return "Duration in ms";
|
||||
}
|
||||
|
||||
} // namespace beast::insight
|
||||
|
||||
@@ -136,6 +136,68 @@ inline constexpr std::array kByteBuckets{
|
||||
262'144.0,
|
||||
1'048'576.0};
|
||||
|
||||
/**
|
||||
* Bucket edges, in microseconds, for the OTel-native duration instruments
|
||||
* created directly on MetricsRegistry: job queue wait and run times, RPC
|
||||
* method latency, and GetObject lookup latency.
|
||||
*
|
||||
* The edges from 1 to 1000 us are the ones that matter most. An earlier
|
||||
* version of this ladder started at 100 us, which sat ABOVE the mass of every
|
||||
* instrument using it: 99.3% of job_queued_us samples, 92.5% of
|
||||
* job_running_us and 90.4% of getobject_lookup_us fell in that first bucket.
|
||||
* `histogram_quantile` then interpolated inside bucket 0 and returned the
|
||||
* boundary scaled by the requested quantile -- p75/p95/p99 of job_queued_us
|
||||
* read 75.5/95.7/99.7 us, which is arithmetic on the bucket edge, not a
|
||||
* latency. A warm nodestore read is around 1.5 us, so single-microsecond
|
||||
* resolution is not excessive here.
|
||||
*
|
||||
* The upper edges reach a minute so multi-second stalls stay measurable. The
|
||||
* SDK's own default ladder stops at 10,000, which every one of these
|
||||
* instruments exceeds during catch-up.
|
||||
*/
|
||||
inline constexpr std::array kMicrosecondBuckets{
|
||||
1.0,
|
||||
2.0,
|
||||
5.0,
|
||||
10.0,
|
||||
25.0,
|
||||
50.0,
|
||||
100.0,
|
||||
250.0,
|
||||
500.0,
|
||||
1'000.0,
|
||||
5'000.0,
|
||||
25'000.0,
|
||||
100'000.0,
|
||||
500'000.0,
|
||||
1'000'000.0,
|
||||
5'000'000.0,
|
||||
10'000'000.0,
|
||||
30'000'000.0,
|
||||
60'000'000.0};
|
||||
|
||||
/**
|
||||
* Bucket edges for the GetObject request object count.
|
||||
*
|
||||
* Counts run from 1 to the hard reply cap (kHardMaxReplyNodes, 12288). The
|
||||
* honest sync path asks for at most 8 objects, so the low edges are
|
||||
* fine-grained; the upper ones follow the charge size bands up to the cap.
|
||||
* Because the top edge IS the hard cap, this ladder cannot saturate.
|
||||
*/
|
||||
inline constexpr std::array
|
||||
kObjectCountBuckets{1.0, 2.0, 4.0, 8.0, 16.0, 64.0, 256.0, 1'024.0, 4'096.0, 12'288.0};
|
||||
|
||||
/**
|
||||
* Bucket edges for the GetObject resource charge.
|
||||
*
|
||||
* Charges span 0 (the free tier) to roughly 99k for a full-size all-miss
|
||||
* request. The edges bracket the two thresholds that decide a peer's fate --
|
||||
* the warning threshold at 5000 and the drop threshold at 25000 -- so a
|
||||
* dashboard can show how close charges run to each.
|
||||
*/
|
||||
inline constexpr std::array
|
||||
kChargeBuckets{0.0, 100.0, 500.0, 1'000.0, 5'000.0, 10'000.0, 25'000.0, 50'000.0, 100'000.0};
|
||||
|
||||
/**
|
||||
* @brief Check that a ladder is strictly ascending and non-negative.
|
||||
*
|
||||
@@ -163,6 +225,9 @@ isAscendingNonNegative(std::span<double const> ladder) noexcept
|
||||
|
||||
static_assert(isAscendingNonNegative(kMillisecondBuckets));
|
||||
static_assert(isAscendingNonNegative(kByteBuckets));
|
||||
static_assert(isAscendingNonNegative(kMicrosecondBuckets));
|
||||
static_assert(isAscendingNonNegative(kObjectCountBuckets));
|
||||
static_assert(isAscendingNonNegative(kChargeBuckets));
|
||||
|
||||
/**
|
||||
* @brief Copy a ladder into the `std::vector<double>` the OTel SDK wants.
|
||||
|
||||
@@ -41,6 +41,7 @@
|
||||
#include <xrpl/beast/insight/Hook.h>
|
||||
#include <xrpl/beast/insight/HookImpl.h>
|
||||
#include <xrpl/beast/insight/MeterImpl.h>
|
||||
#include <xrpl/beast/insight/Unit.h>
|
||||
#include <xrpl/beast/utility/Journal.h>
|
||||
|
||||
#include <opentelemetry/metrics/async_instruments.h>
|
||||
@@ -168,10 +169,17 @@ private:
|
||||
/**
|
||||
* @brief OTel-backed implementation of beast::insight::EventImpl.
|
||||
*
|
||||
* Wraps an OTel Histogram<double> instrument. Each notify() call
|
||||
* records the duration in milliseconds. Uses explicit bucket boundaries
|
||||
* matching the SpanMetrics connector configuration:
|
||||
* [1, 5, 10, 25, 50, 100, 250, 500, 1000, 5000] ms
|
||||
* Wraps an OTel Histogram<double> instrument. Each notify() call records one
|
||||
* sample, interpreted per the Event's unit().
|
||||
*
|
||||
* The instrument's declared unit is what selects its bucket ladder: the
|
||||
* histogram views registered in Telemetry.cpp match on unit, so a `ms`
|
||||
* instrument gets the millisecond ladder and a `By` instrument the byte
|
||||
* ladder. The edges themselves live in xrpl/telemetry/HistogramBuckets.h --
|
||||
* do not restate them here. An earlier version of this comment listed
|
||||
* `[1, 5, ..., 1000, 5000] ms` as "matching the SpanMetrics connector"; that
|
||||
* was true when written and silently became false when the connector's
|
||||
* ladder was extended, which is why the edges now have one owner.
|
||||
*
|
||||
* Thread safety: OTel Histogram::Record() is thread-safe by specification.
|
||||
*/
|
||||
@@ -183,10 +191,14 @@ public:
|
||||
* formatName() by the collector: lowercase, with `.` and
|
||||
* ` ` mapped to `_` (e.g. "rpc_size").
|
||||
* @param meter OTel Meter used to create the histogram instrument.
|
||||
* @param unit What the samples measure. Selects the instrument's
|
||||
* declared unit, its description, and through the unit the
|
||||
* bucket ladder a histogram view applies.
|
||||
*/
|
||||
OTelEventImpl(
|
||||
std::string const& name,
|
||||
opentelemetry::nostd::shared_ptr<metrics_api::Meter> const& meter);
|
||||
opentelemetry::nostd::shared_ptr<metrics_api::Meter> const& meter,
|
||||
Unit unit);
|
||||
|
||||
~OTelEventImpl() override = default;
|
||||
|
||||
@@ -474,6 +486,9 @@ public:
|
||||
Event
|
||||
makeEvent(std::string const& name) override;
|
||||
|
||||
Event
|
||||
makeEvent(std::string const& name, Unit unit) override;
|
||||
|
||||
Gauge
|
||||
makeGauge(std::string const& name) override;
|
||||
|
||||
@@ -652,8 +667,10 @@ OTelCounterImpl::increment(value_type amount)
|
||||
|
||||
OTelEventImpl::OTelEventImpl(
|
||||
std::string const& name,
|
||||
opentelemetry::nostd::shared_ptr<metrics_api::Meter> const& meter)
|
||||
: histogram_(meter->CreateDoubleHistogram(name, "Duration in ms", "ms"))
|
||||
opentelemetry::nostd::shared_ptr<metrics_api::Meter> const& meter,
|
||||
Unit unit)
|
||||
: EventImpl(unit)
|
||||
, histogram_(meter->CreateDoubleHistogram(name, otelUnitDescription(unit), otelUnitCode(unit)))
|
||||
{
|
||||
}
|
||||
|
||||
@@ -842,7 +859,13 @@ OTelCollectorImp::makeCounter(std::string const& name)
|
||||
Event
|
||||
OTelCollectorImp::makeEvent(std::string const& name)
|
||||
{
|
||||
return Event(std::make_shared<OTelEventImpl>(formatName(name), otelMeter_));
|
||||
return makeEvent(name, Unit::Millis);
|
||||
}
|
||||
|
||||
Event
|
||||
OTelCollectorImp::makeEvent(std::string const& name, Unit unit)
|
||||
{
|
||||
return Event(std::make_shared<OTelEventImpl>(formatName(name), otelMeter_, unit));
|
||||
}
|
||||
|
||||
Gauge
|
||||
|
||||
@@ -19,10 +19,12 @@
|
||||
#include <xrpl/telemetry/Telemetry.h>
|
||||
|
||||
#include <xrpl/basics/Log.h>
|
||||
#include <xrpl/beast/insight/Unit.h>
|
||||
#include <xrpl/beast/utility/Journal.h>
|
||||
#include <xrpl/telemetry/CoroAwareContextStorage.h>
|
||||
#include <xrpl/telemetry/DeterministicIdGenerator.h>
|
||||
#include <xrpl/telemetry/DiscardFlag.h>
|
||||
#include <xrpl/telemetry/HistogramBuckets.h>
|
||||
#include <xrpl/telemetry/SpanNames.h>
|
||||
|
||||
#include <opentelemetry/context/context.h>
|
||||
@@ -553,31 +555,46 @@ public:
|
||||
std::make_unique<metrics_sdk::ViewRegistry>(), resourceAttrs);
|
||||
meterProvider_->AddMetricReader(std::move(reader));
|
||||
|
||||
// Histogram view: SpanMetrics-compatible bucket boundaries (ms) so
|
||||
// histogram instruments align with the collector's SpanMetrics. The
|
||||
// view is created with an EMPTY name so it applies the buckets WITHOUT
|
||||
// renaming instruments — a non-empty view name would collapse every
|
||||
// matching histogram (ios_latency, rpc_size, rpc_time, pathfind_*)
|
||||
// into a single series under that one name.
|
||||
auto histogramSelector = metrics_sdk::InstrumentSelectorFactory::Create(
|
||||
metrics_sdk::InstrumentType::kHistogram, "*", "ms");
|
||||
// Meter selector MUST match the meter name used by getMeter() and the
|
||||
// beast OTelCollector (kMeterName = "xrpld"); otherwise this histogram
|
||||
// view never applies and duration histograms fall back to the SDK
|
||||
// default boundaries instead of these SpanMetrics-aligned buckets.
|
||||
auto meterSelector =
|
||||
metrics_sdk::MeterSelectorFactory::Create(std::string(kMeterName), "", "");
|
||||
auto histogramConfig = std::make_shared<metrics_sdk::HistogramAggregationConfig>();
|
||||
histogramConfig->boundaries_ =
|
||||
std::vector<double>{1.0, 5.0, 10.0, 25.0, 50.0, 100.0, 250.0, 500.0, 1000.0, 5000.0};
|
||||
auto histogramView = metrics_sdk::ViewFactory::Create(
|
||||
"", // empty name: keep each instrument's own name, only set buckets
|
||||
"SpanMetrics-compatible histogram buckets",
|
||||
metrics_sdk::AggregationType::kHistogram,
|
||||
histogramConfig);
|
||||
// One histogram view per unit. The unit is the selector, so an
|
||||
// instrument gets the ladder that fits what it measures -- a byte
|
||||
// count no longer inherits a latency ladder. Edges come from
|
||||
// HistogramBuckets.h, which owns every ladder.
|
||||
//
|
||||
// Each view keeps the "*" name pattern and an EMPTY view name: a
|
||||
// non-empty view name would rename every matching histogram to it and
|
||||
// collapse them (ios_latency, rpc_size, rpc_time, pathfind_*, and all
|
||||
// the jobq_* pairs) into a single series.
|
||||
//
|
||||
// The meter selector MUST match the meter name used by getMeter() and
|
||||
// the beast OTelCollector (kMeterName = "xrpld"); otherwise a view
|
||||
// never applies and instruments fall back to the SDK default ladder,
|
||||
// whose ceiling is 10,000.
|
||||
auto const addUnitView = [this](
|
||||
std::string const& unitCode,
|
||||
std::vector<double> boundaries,
|
||||
std::string const& description) {
|
||||
auto selector = metrics_sdk::InstrumentSelectorFactory::Create(
|
||||
metrics_sdk::InstrumentType::kHistogram, "*", unitCode);
|
||||
auto meterSelector =
|
||||
metrics_sdk::MeterSelectorFactory::Create(std::string(kMeterName), "", "");
|
||||
auto config = std::make_shared<metrics_sdk::HistogramAggregationConfig>();
|
||||
config->boundaries_ = std::move(boundaries);
|
||||
auto view = metrics_sdk::ViewFactory::Create(
|
||||
"", // empty name: keep each instrument's own name, only set buckets
|
||||
description,
|
||||
metrics_sdk::AggregationType::kHistogram,
|
||||
std::move(config));
|
||||
meterProvider_->AddView(std::move(selector), std::move(meterSelector), std::move(view));
|
||||
};
|
||||
|
||||
meterProvider_->AddView(
|
||||
std::move(histogramSelector), std::move(meterSelector), std::move(histogramView));
|
||||
addUnitView(
|
||||
beast::insight::otelUnitCode(beast::insight::Unit::Millis),
|
||||
buckets::toVector(buckets::kMillisecondBuckets),
|
||||
"Duration buckets, 1 ms to 120 s");
|
||||
addUnitView(
|
||||
beast::insight::otelUnitCode(beast::insight::Unit::Bytes),
|
||||
buckets::toVector(buckets::kByteBuckets),
|
||||
"Size buckets, 512 B to 1 MiB");
|
||||
|
||||
// Publish as the global meter provider so developers (and the beast
|
||||
// OTelCollector shim) reach the same pipeline.
|
||||
|
||||
@@ -70,6 +70,14 @@ TEST(InsightUnit, otelCodeIsTheUcumCodeForEachUnit)
|
||||
EXPECT_STREQ(otelUnitCode(Unit::Bytes), "By");
|
||||
}
|
||||
|
||||
// The description is what an operator reads in the metric catalogue, so a
|
||||
// byte-valued instrument must not describe itself as a duration.
|
||||
TEST(InsightUnit, descriptionMatchesWhatTheUnitActuallyMeasures)
|
||||
{
|
||||
EXPECT_STREQ(otelUnitDescription(Unit::Millis), "Duration in ms");
|
||||
EXPECT_STREQ(otelUnitDescription(Unit::Bytes), "Size in bytes");
|
||||
}
|
||||
|
||||
TEST(InsightUnit, defaultEventUnitIsMillisForBackwardCompatibility)
|
||||
{
|
||||
// Every pre-existing makeEvent(name) call site records a duration, so the
|
||||
|
||||
@@ -60,7 +60,52 @@ INSTANTIATE_TEST_SUITE_P(
|
||||
HistogramBucketsTest,
|
||||
::testing::Values(
|
||||
std::span<double const>{kMillisecondBuckets},
|
||||
std::span<double const>{kByteBuckets}));
|
||||
std::span<double const>{kByteBuckets},
|
||||
std::span<double const>{kMicrosecondBuckets},
|
||||
std::span<double const>{kObjectCountBuckets},
|
||||
std::span<double const>{kChargeBuckets}));
|
||||
|
||||
TEST(HistogramBucketsRange, microsecondFloorLandsBelowTheMeasuredMass)
|
||||
{
|
||||
// Measured: 99.3% of job_queued_us samples sat below the old 100 us floor,
|
||||
// so p75/p95/p99 all interpolated inside bucket 0 and returned
|
||||
// 75.5/95.7/99.7 us -- the boundary scaled by the requested quantile,
|
||||
// not a latency. Warm nodestore reads are ~1.5 us, so the floor has to
|
||||
// reach single microseconds and several edges must precede 100 us.
|
||||
EXPECT_LE(kMicrosecondBuckets.front(), 1.0);
|
||||
|
||||
auto const belowHundred =
|
||||
std::ranges::count_if(kMicrosecondBuckets, [](double edge) { return edge < 100.0; });
|
||||
EXPECT_GE(belowHundred, 5) << "too little resolution below 100 us";
|
||||
}
|
||||
|
||||
TEST(HistogramBucketsRange, microsecondCeilingStillReachesOneMinute)
|
||||
{
|
||||
// Job waits and RPC latencies routinely exceed the SDK default ceiling of
|
||||
// 10,000; multi-second stalls must stay measurable rather than censored.
|
||||
EXPECT_EQ(kMicrosecondBuckets.back(), 60'000'000.0);
|
||||
}
|
||||
|
||||
TEST(HistogramBucketsRange, objectCountLadderCannotSaturate)
|
||||
{
|
||||
// GetObject counts run 1..kHardMaxReplyNodes, so the top edge IS the hard
|
||||
// cap and censoring is impossible by construction.
|
||||
EXPECT_EQ(kObjectCountBuckets.front(), 1.0);
|
||||
EXPECT_EQ(kObjectCountBuckets.back(), 12'288.0);
|
||||
}
|
||||
|
||||
TEST(HistogramBucketsRange, chargeLadderBracketsTheResourceThresholds)
|
||||
{
|
||||
// The two edges that decide a peer's fate must be present so a dashboard
|
||||
// can show how close charges run to each: warning at 5000, drop at 25000.
|
||||
// A leading 0 separates the free tier from everything else.
|
||||
EXPECT_EQ(kChargeBuckets.front(), 0.0);
|
||||
for (double const threshold : {5'000.0, 25'000.0})
|
||||
{
|
||||
EXPECT_NE(std::ranges::find(kChargeBuckets, threshold), kChargeBuckets.end())
|
||||
<< threshold << " is a resource threshold and must be an edge";
|
||||
}
|
||||
}
|
||||
|
||||
// The validator must also REJECT. A predicate that only ever returns true
|
||||
// would let every ladder above pass while proving nothing.
|
||||
|
||||
@@ -63,6 +63,7 @@
|
||||
#include <xrpl/server/LoadFeeTrack.h>
|
||||
#include <xrpl/server/NetworkOPs.h>
|
||||
#include <xrpl/telemetry/GetObjectMetricNames.h>
|
||||
#include <xrpl/telemetry/HistogramBuckets.h>
|
||||
#include <xrpl/telemetry/SpanNames.h>
|
||||
|
||||
#include <opentelemetry/context/context.h>
|
||||
@@ -84,7 +85,6 @@
|
||||
#include <opentelemetry/semconv/incubating/service_attributes.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <atomic>
|
||||
#include <chrono>
|
||||
#include <cstddef>
|
||||
@@ -119,66 +119,16 @@ constexpr char kRpcMethodDurationUs[] = "rpc_method_us";
|
||||
constexpr char kJobTypeLabel[] = "job_type";
|
||||
constexpr char kHandlerLabel[] = "handler";
|
||||
|
||||
/**
|
||||
* Bucket boundaries for microsecond-valued duration instruments.
|
||||
*
|
||||
* 100 µs, 500 µs, 1 ms, 5 ms, 10 ms, 25 ms, 50 ms, 100 ms, 250 ms, 500 ms,
|
||||
* 1 s, 2.5 s, 5 s, 10 s, 30 s, 60 s. Covers sub-millisecond jobs through
|
||||
* multi-second stalls without saturating.
|
||||
*/
|
||||
constexpr std::array kMicrosecondBoundaries{
|
||||
100.0,
|
||||
500.0,
|
||||
1'000.0,
|
||||
5'000.0,
|
||||
10'000.0,
|
||||
25'000.0,
|
||||
50'000.0,
|
||||
100'000.0,
|
||||
250'000.0,
|
||||
500'000.0,
|
||||
1'000'000.0,
|
||||
2'500'000.0,
|
||||
5'000'000.0,
|
||||
10'000'000.0,
|
||||
30'000'000.0,
|
||||
60'000'000.0};
|
||||
|
||||
/**
|
||||
* Bucket boundaries for latencies that are normally sub-millisecond.
|
||||
*
|
||||
* 1 µs, 2 µs, 5 µs, 10 µs, 25 µs, 50 µs, 100 µs, 250 µs, 500 µs, 1 ms, 5 ms,
|
||||
* 25 ms.
|
||||
*
|
||||
* kMicrosecondBoundaries starts at 100 µs, which is above the entire range a
|
||||
* healthy nodestore read occupies, so every warm read falls in its first
|
||||
* bucket and the distribution reads as flat. These edges resolve the warm
|
||||
* range instead, while still reaching far enough to show a cold tail against
|
||||
* it.
|
||||
*
|
||||
* Currently unused: no sub-millisecond histogram instrument exists yet. The
|
||||
* edges live here so the instrument that records nodestore read latency gets
|
||||
* a ladder that fits it, rather than silently inheriting the wrong one.
|
||||
*/
|
||||
[[maybe_unused]] constexpr std::array kSubMillisecondBoundaries{
|
||||
1.0,
|
||||
2.0,
|
||||
5.0,
|
||||
10.0,
|
||||
25.0,
|
||||
50.0,
|
||||
100.0,
|
||||
250.0,
|
||||
500.0,
|
||||
1'000.0,
|
||||
5'000.0,
|
||||
25'000.0};
|
||||
|
||||
/**
|
||||
* Register an explicit-bucket histogram view.
|
||||
*
|
||||
* The SDK's default boundaries top out at 10,000, so any instrument whose
|
||||
* values exceed that saturates and every quantile reads as the ceiling.
|
||||
* values exceed that saturates and every quantile reads as the ceiling. The
|
||||
* floor matters just as much and is easier to miss: a ladder whose first edge
|
||||
* sits above the mass of the distribution makes every low quantile an
|
||||
* interpolation inside bucket 0 -- a number derived from the bucket edge
|
||||
* rather than from any sample. Both ends are chosen from measured
|
||||
* distributions in HistogramBuckets.h.
|
||||
*
|
||||
* @param views The registry to add the view to.
|
||||
* @param name Instrument name to match (e.g. "job_running_us").
|
||||
@@ -206,7 +156,7 @@ addHistogramView(
|
||||
* Register the microsecond-ladder view for a duration instrument.
|
||||
*
|
||||
* Job wait/run times and RPC latencies routinely exceed the SDK default
|
||||
* ceiling, so they all share `kMicrosecondBoundaries`.
|
||||
* ceiling, so they all share `buckets::kMicrosecondBuckets`.
|
||||
*
|
||||
* @param views The registry to add the view to.
|
||||
* @param name Instrument name to match.
|
||||
@@ -214,7 +164,10 @@ addHistogramView(
|
||||
void
|
||||
addMicrosecondHistogramView(metric_sdk::ViewRegistry& views, std::string const& name)
|
||||
{
|
||||
addHistogramView(views, name, {kMicrosecondBoundaries.begin(), kMicrosecondBoundaries.end()});
|
||||
addHistogramView(
|
||||
views,
|
||||
name,
|
||||
xrpl::telemetry::buckets::toVector(xrpl::telemetry::buckets::kMicrosecondBuckets));
|
||||
}
|
||||
|
||||
} // namespace
|
||||
@@ -352,18 +305,13 @@ MetricsRegistry::initExporterAndProvider(
|
||||
// asks for at most 8, so the low buckets are fine-grained and the upper
|
||||
// ones follow the charge size bands (64, 1024) up to the hard cap.
|
||||
addHistogramView(
|
||||
*views,
|
||||
kGetObjectRequestObjects,
|
||||
{1.0, 2.0, 4.0, 8.0, 16.0, 64.0, 256.0, 1'024.0, 4'096.0, 12'288.0});
|
||||
*views, kGetObjectRequestObjects, buckets::toVector(buckets::kObjectCountBuckets));
|
||||
|
||||
// Charge values span 0 (free tier) to ~99k for a full-size all-miss
|
||||
// request. Boundaries bracket the resource thresholds that decide a
|
||||
// peer's fate -- kWarningThreshold (5000) and kDropThreshold (25000) --
|
||||
// so a dashboard can show how close charges run to each.
|
||||
addHistogramView(
|
||||
*views,
|
||||
kGetObjectCharge,
|
||||
{0.0, 100.0, 500.0, 1'000.0, 5'000.0, 10'000.0, 25'000.0, 50'000.0, 100'000.0});
|
||||
addHistogramView(*views, kGetObjectCharge, buckets::toVector(buckets::kChargeBuckets));
|
||||
|
||||
// Create MeterProvider with resource, then attach the metric reader.
|
||||
provider_ = metric_sdk::MeterProviderFactory::Create(std::move(views), resourceAttrs);
|
||||
|
||||
Reference in New Issue
Block a user