ARCHIVE/rippled
1
0
Fork 0
mirror of https://github.com/XRPLF/rippled.git synced 2026-08-21 22:30:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'pratik/otel-sync-diagnostics' into pratik/otel-devbox-two-node-configs

Browse Source
This commit is contained in:
Pratik Mankawde
2026-08-21 13:12:11 +01:00
parent 8340e95c1e 7c70e142e9
commit 0b9684d472
62 changed files with 2858 additions and 394 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
docker/telemetry/grafana/dashboards/consensus-health.json
View File

@@ -21,11 +21,28 @@
"enable": true,
"hide": false,
"iconColor": "rgb(70, 70, 70)",
"name": "Annotate perf-iac runs",
"name": "Perf Runs (JMeter)",
"target": {
"limit": 100,
"matchAny": false,
"tags": ["perf-iac"],
"tags": ["perf-iac", "jmeter"],
"type": "tags"
},
"type": "tags"
},
{
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": false,
"iconColor": "rgb(15, 122, 102)",
"name": "Perf Runs (Locust)",
"target": {
"limit": 100,
"matchAny": false,
"tags": ["perf-iac", "locust"],
"type": "tags"
},
"type": "tags"

21
docker/telemetry/grafana/dashboards/fee-market.json
View File

@@ -21,11 +21,28 @@
"enable": true,
"hide": false,
"iconColor": "rgb(70, 70, 70)",
"name": "Annotate perf-iac runs",
"name": "Perf Runs (JMeter)",
"target": {
"limit": 100,
"matchAny": false,
"tags": ["perf-iac"],
"tags": ["perf-iac", "jmeter"],
"type": "tags"
},
"type": "tags"
},
{
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": false,
"iconColor": "rgb(15, 122, 102)",
"name": "Perf Runs (Locust)",
"target": {
"limit": 100,
"matchAny": false,
"tags": ["perf-iac", "locust"],
"type": "tags"
},
"type": "tags"

49
docker/telemetry/grafana/dashboards/job-queue.json
View File

@@ -1,6 +1,53 @@
{
"annotations": {
"list": []
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
},
{
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": false,
"iconColor": "rgb(70, 70, 70)",
"name": "Perf Runs (JMeter)",
"target": {
"limit": 100,
"matchAny": false,
"tags": ["perf-iac", "jmeter"],
"type": "tags"
},
"type": "tags"
},
{
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": false,
"iconColor": "rgb(15, 122, 102)",
"name": "Perf Runs (Locust)",
"target": {
"limit": 100,
"matchAny": false,
"tags": ["perf-iac", "locust"],
"type": "tags"
},
"type": "tags"
}
]
},
"description": "What this shows: Per-job-type throughput, queue wait times, and execution times for the node's internal worker job queue.\nUse it to: Find job types that are backing up or running slowly and causing internal processing delays.",
"editable": true,

65
docker/telemetry/grafana/dashboards/ledger-data-sync.json
View File

@@ -1,6 +1,53 @@
{
"annotations": {
"list": []
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
},
{
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": false,
"iconColor": "rgb(70, 70, 70)",
"name": "Perf Runs (JMeter)",
"target": {
"limit": 100,
"matchAny": false,
"tags": ["perf-iac", "jmeter"],
"type": "tags"
},
"type": "tags"
},
{
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": false,
"iconColor": "rgb(15, 122, 102)",
"name": "Perf Runs (Locust)",
"target": {
"limit": 100,
"matchAny": false,
"tags": ["perf-iac", "locust"],
"type": "tags"
},
"type": "tags"
}
]
},
"description": "What this shows: Ledger data exchange and object-fetch traffic between this node and its peers: ledger sync, tree-node retrieval, and transaction-set exchange.\nUse it to: See how much ledger data the node is pulling or serving and spot catch-up activity.",
"editable": true,
@@ -1204,7 +1251,7 @@
},
{
"title": "Job Queue Wait p95 By Type",
"description": "###### What this is:\n*95th-percentile time a job waits in the queue before a worker thread picks it up, for the sync-critical job types. This is the metric form of the 'ProcessLData wait: NNNNms' warnings in the debug log.*\n\n###### How it's computed:\n*histogram_quantile(0.95, rate(jobq_<type>_q_milliseconds_bucket[$__rate_interval])) for ledgerdata, acceptledger, fetchtxndata, transaction, advanceledger, ledgerrequest.*\n\n###### Reading it:\n*Queue wait should be single-digit to low-tens of ms. High ledgerdata/fetchtxndata wait = the node cannot process inbound ledger data fast enough.*\n\n###### Healthy range:\n*< ~50ms p95 per type on a healthy node.*\n\n###### Watch for:\n*ledgerdata or fetchtxndata q-wait spiking to seconds = worker threads are blocked (usually on NuDB reads - see the cause tier).*\n\n###### Keywords:\n- **Job queue / job type** *(per node)* \u2014 xrpld's worker-thread pool; every unit of background work is enqueued under a named job type.\n- **Deferred job** *(per node)* \u2014 a job held back because its type is already at its concurrency limit; the leading indicator of queue backpressure.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[core/JobQueue.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/core/detail/JobQueue.cpp)\n\n###### Function:\n`JobQueue::getJson (per-type queue timing)`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#job-queue-job-type)",
"description": "###### What this is:\n*95th-percentile time a job waits in the queue before a worker thread picks it up, for the sync-critical job types. This is the metric form of the 'ProcessLData wait: NNNNms' warnings in the debug log.*\n\n###### How it's computed:\n*histogram_quantile(0.95, rate(job_queued_us_bucket{job_type=\"<type>\"}[$__rate_interval])) for ledgerData, acceptLedger, fetchTxnData, transaction, advanceLedger, ledgerRequest. Reads the OTel-native microsecond instrument rather than the beast jobq_* pair: beast Events round every duration up to a whole millisecond, so 94-100% of their samples landed in the first bucket and every percentile was an interpolation inside it rather than a measurement.*\n\n###### Reading it:\n*Queue wait is normally tens to hundreds of microseconds. High ledgerData/fetchTxnData wait = the node cannot process inbound ledger data fast enough.*\n\n###### Healthy range:\n*< ~500us p95 per type on a healthy node; sustained milliseconds is already backpressure.*\n\n###### Watch for:\n*ledgerData or fetchTxnData q-wait spiking to seconds = worker threads are blocked (usually on NuDB reads - see the cause tier).*\n\n###### Keywords:\n- **Job queue / job type** *(per node)* \u2014 xrpld's worker-thread pool; every unit of background work is enqueued under a named job type.\n- **Deferred job** *(per node)* \u2014 a job held back because its type is already at its concurrency limit; the leading indicator of queue backpressure.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code by MetricsRegistry as an OTel-native histogram in microseconds; the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[core/JobQueue.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/core/detail/JobQueue.cpp)\n\n###### Function:\n`JobQueue::getJson (per-type queue timing)`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#job-queue-job-type)",
"type": "timeseries",
"gridPos": {
"h": 10,
@@ -1225,48 +1272,48 @@
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(jobq_ledgerdata_q_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"ledgerdata q-wait p95\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
"expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(job_queued_us_bucket{job_type=\"ledgerData\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"ledgerData q-wait p95\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(jobq_acceptledger_q_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"acceptledger q-wait p95\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
"expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(job_queued_us_bucket{job_type=\"acceptLedger\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"acceptLedger q-wait p95\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(jobq_fetchtxndata_q_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"fetchtxndata q-wait p95\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
"expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(job_queued_us_bucket{job_type=\"fetchTxnData\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"fetchTxnData q-wait p95\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(jobq_transaction_q_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"transaction q-wait p95\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
"expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(job_queued_us_bucket{job_type=\"transaction\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"transaction q-wait p95\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(jobq_advanceledger_q_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"advanceledger q-wait p95\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
"expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(job_queued_us_bucket{job_type=\"advanceLedger\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"advanceLedger q-wait p95\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(jobq_ledgerrequest_q_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"ledgerrequest q-wait p95\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
"expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(job_queued_us_bucket{job_type=\"ledgerRequest\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]))), \"series\", \"ledgerRequest q-wait p95\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
}
],
"fieldConfig": {
"defaults": {
"displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}",
"unit": "ms",
"unit": "µs",
"custom": {
"axisLabel": "p95 Wait (ms)",
"spanNulls": 1800000,

21
docker/telemetry/grafana/dashboards/ledger-operations.json
View File

@@ -21,11 +21,28 @@
"enable": true,
"hide": false,
"iconColor": "rgb(70, 70, 70)",
"name": "Annotate perf-iac runs",
"name": "Perf Runs (JMeter)",
"target": {
"limit": 100,
"matchAny": false,
"tags": ["perf-iac"],
"tags": ["perf-iac", "jmeter"],
"type": "tags"
},
"type": "tags"
},
{
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": false,
"iconColor": "rgb(15, 122, 102)",
"name": "Perf Runs (Locust)",
"target": {
"limit": 100,
"matchAny": false,
"tags": ["perf-iac", "locust"],
"type": "tags"
},
"type": "tags"

21
docker/telemetry/grafana/dashboards/ledger-sync-health.json
View File

@@ -21,11 +21,28 @@
"enable": true,
"hide": false,
"iconColor": "rgb(70, 70, 70)",
"name": "Annotate perf-iac runs",
"name": "Perf Runs (JMeter)",
"target": {
"limit": 100,
"matchAny": false,
"tags": ["perf-iac"],
"tags": ["perf-iac", "jmeter"],
"type": "tags"
},
"type": "tags"
},
{
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": false,
"iconColor": "rgb(15, 122, 102)",
"name": "Perf Runs (Locust)",
"target": {
"limit": 100,
"matchAny": false,
"tags": ["perf-iac", "locust"],
"type": "tags"
},
"type": "tags"

49
docker/telemetry/grafana/dashboards/log-derived-insights.json
View File

@@ -14,7 +14,54 @@
"uid": "log-derived-insights",
"refresh": "60s",
"annotations": {
"list": []
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
},
{
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": false,
"iconColor": "rgb(70, 70, 70)",
"name": "Perf Runs (JMeter)",
"target": {
"limit": 100,
"matchAny": false,
"tags": ["perf-iac", "jmeter"],
"type": "tags"
},
"type": "tags"
},
{
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": false,
"iconColor": "rgb(15, 122, 102)",
"name": "Perf Runs (Locust)",
"target": {
"limit": 100,
"matchAny": false,
"tags": ["perf-iac", "locust"],
"type": "tags"
},
"type": "tags"
}
]
},
"templating": {
"list": [

21
docker/telemetry/grafana/dashboards/network-traffic.json
View File

@@ -21,11 +21,28 @@
"enable": true,
"hide": false,
"iconColor": "rgb(70, 70, 70)",
"name": "Annotate perf-iac runs",
"name": "Perf Runs (JMeter)",
"target": {
"limit": 100,
"matchAny": false,
"tags": ["perf-iac"],
"tags": ["perf-iac", "jmeter"],
"type": "tags"
},
"type": "tags"
},
{
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": false,
"iconColor": "rgb(15, 122, 102)",
"name": "Perf Runs (Locust)",
"target": {
"limit": 100,
"matchAny": false,
"tags": ["perf-iac", "locust"],
"type": "tags"
},
"type": "tags"

49
docker/telemetry/grafana/dashboards/node-health.json
View File

@@ -21,11 +21,28 @@
"enable": true,
"hide": false,
"iconColor": "rgb(70, 70, 70)",
"name": "Annotate perf-iac runs",
"name": "Perf Runs (JMeter)",
"target": {
"limit": 100,
"matchAny": false,
"tags": ["perf-iac"],
"tags": ["perf-iac", "jmeter"],
"type": "tags"
},
"type": "tags"
},
{
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": false,
"iconColor": "rgb(15, 122, 102)",
"name": "Perf Runs (Locust)",
"target": {
"limit": 100,
"matchAny": false,
"tags": ["perf-iac", "locust"],
"type": "tags"
},
"type": "tags"
@@ -810,7 +827,7 @@
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "###### What this is:\n*Transitions into each operating mode, per interval.*\n\n###### How it's computed:\n*increase() over each per-mode transition counter, so each point is the number of transitions in that bucket and the series stays correct across an xrpld restart (the counters reset to 0).*\n\n###### Reading it:\n*Zero is healthy; each point is a mode change within that bucket. Brief flaps show up here even when they are too short to appear on Operating Mode (State Timeline), which can only sample state once per scrape.*\n\n###### Healthy range:\n*Few transitions once the node is stable in Full mode.*\n\n###### Watch for:\n*Frequent transitions out of Full, or into Disconnected or Syncing, indicate instability.*\n\n###### Keywords:\n- **Operating mode / server state** *(per node)* \u2014 the node's sync level: Disconnected, Connected, Syncing, Tracking, Full (and Validating/Proposing).\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`NetworkOPsImp::Stats`\n\n###### References:\n[Operating mode / server state](https://xrpl.org/docs/references/http-websocket-apis/api-conventions/xrpld-server-states) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#operating-mode-server-state)",
"description": "###### What this is:\n*Transitions into each operating mode, per interval.*\n\n###### How it's computed:\n*round(increase(...[$__interval])) over each per-mode transition counter. $__interval tiles the buckets exactly, so each bar is the transitions in that bucket and the legend Total is the true count; $__rate_interval would overlap each bucket by one scrape and inflate it (measured +5% at a 36h range, +26% zoomed in). round() removes increase()'s extrapolation, which otherwise reports fractional counts. The series stays correct across an xrpld restart (the counters reset to 0).*\n\n###### Reading it:\n*Zero is healthy; each bar is the mode changes within that bucket. Brief flaps show up here even when they are too short to appear on Operating Mode (State Timeline), which can only sample state once per scrape.*\n\n###### Healthy range:\n*Few transitions once the node is stable in Full mode.*\n\n###### Watch for:\n*Frequent transitions out of Full, or into Disconnected or Syncing, indicate instability.*\n\n###### Keywords:\n- **Operating mode / server state** *(per node)* — the node's sync level: Disconnected, Connected, Syncing, Tracking, Full (and Validating/Proposing).\n\n###### Computation boundary:\n*Result: Per node — each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[NetworkOPs.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/misc/NetworkOPs.cpp)\n\n###### Function:\n`NetworkOPsImp::Stats`\n\n###### References:\n[Operating mode / server state](https://xrpl.org/docs/references/http-websocket-apis/api-conventions/xrpld-server-states) · [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#operating-mode-server-state)",
"fieldConfig": {
"defaults": {
"color": {
@@ -824,8 +841,8 @@
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"drawStyle": "bars",
"fillOpacity": 70,
"gradientMode": "none",
"hideFrom": {
"legend": false,
@@ -834,7 +851,7 @@
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"lineWidth": 0,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
@@ -864,7 +881,8 @@
}
]
},
"unit": "short"
"unit": "short",
"decimals": 0
}
},
"gridPos": {
@@ -880,8 +898,8 @@
"multiLane": false
},
"legend": {
"calcs": [],
"displayMode": "list",
"calcs": ["sum", "max"],
"displayMode": "table",
"enableFacetedFilter": false,
"overflow": "ellipsis",
"placement": "bottom",
@@ -901,7 +919,7 @@
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "label_replace(label_join(label_replace(increase(state_accounting_full_transitions{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Full\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")",
"expr": "label_replace(label_join(label_replace(round(increase(state_accounting_full_transitions{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__interval])), \"series\", \"Full\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")",
"refId": "A"
},
{
@@ -909,7 +927,7 @@
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "label_replace(label_join(label_replace(increase(state_accounting_tracking_transitions{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Tracking\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")",
"expr": "label_replace(label_join(label_replace(round(increase(state_accounting_tracking_transitions{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__interval])), \"series\", \"Tracking\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")",
"refId": "B"
},
{
@@ -917,7 +935,7 @@
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "label_replace(label_join(label_replace(increase(state_accounting_syncing_transitions{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Syncing\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")",
"expr": "label_replace(label_join(label_replace(round(increase(state_accounting_syncing_transitions{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__interval])), \"series\", \"Syncing\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")",
"refId": "C"
},
{
@@ -925,7 +943,7 @@
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "label_replace(label_join(label_replace(increase(state_accounting_connected_transitions{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Connected\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")",
"expr": "label_replace(label_join(label_replace(round(increase(state_accounting_connected_transitions{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__interval])), \"series\", \"Connected\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")",
"refId": "D"
},
{
@@ -933,12 +951,13 @@
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "label_replace(label_join(label_replace(increase(state_accounting_disconnected_transitions{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval]), \"series\", \"Disconnected\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")",
"expr": "label_replace(label_join(label_replace(round(increase(state_accounting_disconnected_transitions{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__interval])), \"series\", \"Disconnected\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")",
"refId": "E"
}
],
"title": "Operating Mode Transitions",
"type": "timeseries"
"type": "timeseries",
"interval": "1m"
},
{
"datasource": {

21
docker/telemetry/grafana/dashboards/overlay-traffic-detail.json
View File

@@ -21,11 +21,28 @@
"enable": true,
"hide": false,
"iconColor": "rgb(70, 70, 70)",
"name": "Annotate perf-iac runs",
"name": "Perf Runs (JMeter)",
"target": {
"limit": 100,
"matchAny": false,
"tags": ["perf-iac"],
"tags": ["perf-iac", "jmeter"],
"type": "tags"
},
"type": "tags"
},
{
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": false,
"iconColor": "rgb(15, 122, 102)",
"name": "Perf Runs (Locust)",
"target": {
"limit": 100,
"matchAny": false,
"tags": ["perf-iac", "locust"],
"type": "tags"
},
"type": "tags"

21
docker/telemetry/grafana/dashboards/peer-network.json
View File

@@ -21,11 +21,28 @@
"enable": true,
"hide": false,
"iconColor": "rgb(70, 70, 70)",
"name": "Annotate perf-iac runs",
"name": "Perf Runs (JMeter)",
"target": {
"limit": 100,
"matchAny": false,
"tags": ["perf-iac"],
"tags": ["perf-iac", "jmeter"],
"type": "tags"
},
"type": "tags"
},
{
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": false,
"iconColor": "rgb(15, 122, 102)",
"name": "Perf Runs (Locust)",
"target": {
"limit": 100,
"matchAny": false,
"tags": ["perf-iac", "locust"],
"type": "tags"
},
"type": "tags"

21
docker/telemetry/grafana/dashboards/peer-quality.json
View File

@@ -21,11 +21,28 @@
"enable": true,
"hide": false,
"iconColor": "rgb(70, 70, 70)",
"name": "Annotate perf-iac runs",
"name": "Perf Runs (JMeter)",
"target": {
"limit": 100,
"matchAny": false,
"tags": ["perf-iac"],
"tags": ["perf-iac", "jmeter"],
"type": "tags"
},
"type": "tags"
},
{
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": false,
"iconColor": "rgb(15, 122, 102)",
"name": "Perf Runs (Locust)",
"target": {
"limit": 100,
"matchAny": false,
"tags": ["perf-iac", "locust"],
"type": "tags"
},
"type": "tags"

25
docker/telemetry/grafana/dashboards/rpc-pathfinding.json
View File

@@ -21,11 +21,28 @@
"enable": true,
"hide": false,
"iconColor": "rgb(70, 70, 70)",
"name": "Annotate perf-iac runs",
"name": "Perf Runs (JMeter)",
"target": {
"limit": 100,
"matchAny": false,
"tags": ["perf-iac"],
"tags": ["perf-iac", "jmeter"],
"type": "tags"
},
"type": "tags"
},
{
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": false,
"iconColor": "rgb(15, 122, 102)",
"name": "Perf Runs (Locust)",
"target": {
"limit": 100,
"matchAny": false,
"tags": ["perf-iac", "locust"],
"type": "tags"
},
"type": "tags"
@@ -137,7 +154,7 @@
},
{
"title": "RPC Response Size",
"description": "\u26a0 Instrument mismatch \u2014 values unreliable. Response size is recorded through the millisecond-scaled event histogram (rpc_size_milliseconds_bucket), so byte values saturate at the top time bucket (5000) and the percentiles are not true byte sizes. A dedicated byte-unit histogram is needed to fix this; tracked separately. Treat this panel as indicative only until then.\n\n###### What this is:\n*The 95th-percentile size of RPC response payloads in bytes.*\n\n###### How it's computed:\n*95th-percentile of response payload sizes over the dashboard rate interval, per node.*\n\n###### Reading it:\n*Smaller is cheaper; large responses cost bandwidth and memory.*\n\n###### Healthy range:\n*Workload-dependent; small for status queries, large for bulk data queries.*\n\n###### Watch for:\n*Growth in large responses, consistent with expensive queries or API misuse.*\n\n###### Keywords:\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[ServerHandler.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/ServerHandler.cpp)\n\n###### Function:\n`ServerHandler ctor`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rpc-command-method)",
"description": "\u26a0 Instrument mismatch \u2014 values unreliable. Response size is recorded through the millisecond-scaled event histogram (rpc_size_bytes_bucket), so byte values saturate at the top time bucket (5000) and the percentiles are not true byte sizes. A dedicated byte-unit histogram is needed to fix this; tracked separately. Treat this panel as indicative only until then.\n\n###### What this is:\n*The 95th-percentile size of RPC response payloads in bytes.*\n\n###### How it's computed:\n*95th-percentile of response payload sizes over the dashboard rate interval, per node.*\n\n###### Reading it:\n*Smaller is cheaper; large responses cost bandwidth and memory.*\n\n###### Healthy range:\n*Workload-dependent; small for status queries, large for bulk data queries.*\n\n###### Watch for:\n*Growth in large responses, consistent with expensive queries or API misuse.*\n\n###### Keywords:\n- **RPC command / method** *(per node)* \u2014 a named API request served by the node (e.g. account_info, ledger, submit), the unit RPC panels break down by.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[ServerHandler.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/rpc/detail/ServerHandler.cpp)\n\n###### Function:\n`ServerHandler ctor`\n\n###### References:\n[RPC command / method](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods) \u00b7 [Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#rpc-command-method)",
"type": "timeseries",
"gridPos": {
"h": 10,
@@ -158,7 +175,7 @@
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(rpc_size_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"P95 Response Size\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
"expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(rpc_size_bytes_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[5m]))), \"series\", \"P95 Response Size\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")"
}
],
"fieldConfig": {

21
docker/telemetry/grafana/dashboards/rpc-performance.json
View File

@@ -21,11 +21,28 @@
"enable": true,
"hide": false,
"iconColor": "rgb(70, 70, 70)",
"name": "Annotate perf-iac runs",
"name": "Perf Runs (JMeter)",
"target": {
"limit": 100,
"matchAny": false,
"tags": ["perf-iac"],
"tags": ["perf-iac", "jmeter"],
"type": "tags"
},
"type": "tags"
},
{
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": false,
"iconColor": "rgb(15, 122, 102)",
"name": "Perf Runs (Locust)",
"target": {
"limit": 100,
"matchAny": false,
"tags": ["perf-iac", "locust"],
"type": "tags"
},
"type": "tags"

21
docker/telemetry/grafana/dashboards/transaction-overview.json
View File

@@ -21,11 +21,28 @@
"enable": true,
"hide": false,
"iconColor": "rgb(70, 70, 70)",
"name": "Annotate perf-iac runs",
"name": "Perf Runs (JMeter)",
"target": {
"limit": 100,
"matchAny": false,
"tags": ["perf-iac"],
"tags": ["perf-iac", "jmeter"],
"type": "tags"
},
"type": "tags"
},
{
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": false,
"iconColor": "rgb(15, 122, 102)",
"name": "Perf Runs (Locust)",
"target": {
"limit": 100,
"matchAny": false,
"tags": ["perf-iac", "locust"],
"type": "tags"
},
"type": "tags"

21
docker/telemetry/grafana/dashboards/validator-health.json
View File

@@ -21,11 +21,28 @@
"enable": true,
"hide": false,
"iconColor": "rgb(70, 70, 70)",
"name": "Annotate perf-iac runs",
"name": "Perf Runs (JMeter)",
"target": {
"limit": 100,
"matchAny": false,
"tags": ["perf-iac"],
"tags": ["perf-iac", "jmeter"],
"type": "tags"
},
"type": "tags"
},
{
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": false,
"iconColor": "rgb(15, 122, 102)",
"name": "Perf Runs (Locust)",
"target": {
"limit": 100,
"matchAny": false,
"tags": ["perf-iac", "locust"],
"type": "tags"
},
"type": "tags"

15
docker/telemetry/grafana/provisioning/alerting/rules.yaml
View File

@@ -480,6 +480,15 @@ groups:
# p99 time a job waits in the queue before running. A sustained p99
# above 1s means the node is saturated and work is backing up. `le` must
# stay inside the inner sum or histogram_quantile cannot interpolate.
#
# Threshold re-validated after the microsecond ladder was re-cut. Do NOT
# tune it down against a casual reading of this p99: before that change
# the ladder's first edge was 100us with 99.3% of samples beneath it, so
# p99 reported 99.7us -- the bucket edge scaled by the quantile, not a
# latency. Measured cumulative distribution: 99.26% of samples land
# within 100us, 99.969% within 5ms, 99.990% within 100ms, and only
# 0.0022% exceed 1s. So 1s sits about four orders of magnitude above the
# healthy p99 and fires only on genuine saturation, which is the intent.
- uid: xrpld-jobqueue-latency-high
title: JobQueueLatencyHigh
condition: C
@@ -544,6 +553,12 @@ groups:
# first and explains the others. Measured p99-of-p95 is 37-49ms on
# healthy nodes and 488-566ms on nodes that are actively flapping, so
# 1000ms flags genuine degradation rather than the current baseline.
#
# Still valid after the millisecond ladder was extended: that change only
# ADDED edges above 5s (2s/3s/4s/10s/30s/60s/120s) and removed none, so
# every edge bracketing this threshold -- 25/50/100/250/500/1000ms -- is
# unchanged and the measurements above still hold. ios_latency's own mean
# is 12.9ms, far below the threshold.
- uid: xrpld-nodestore-io-latency-high
title: NodeStoreIOLatencyHigh
condition: C

2
docker/telemetry/otel-collector-config.grafanacloud.yaml
View File

@@ -139,6 +139,7 @@ processors:
- context: datapoint
statements:
- set(attributes["service_instance_id"], resource.attributes["service.instance.id"])
- set(attributes["xrpl_node_id"], resource.attributes["xrpl.node.id"])
- set(attributes["deployment_environment"], resource.attributes["deployment.environment"])
- set(attributes["xrpl_network_type"], resource.attributes["xrpl.network.type"])
@@ -159,6 +160,7 @@ connectors:
# series from distinct nodes/tiers grouped separately.
resource_metrics_key_attributes:
- service.instance.id
- xrpl.node.id
- deployment.environment
- xrpl.network.type
histogram:

1
docker/telemetry/otel-collector-config.yaml
View File

@@ -136,6 +136,7 @@ connectors:
# nodes/tiers grouped separately.
resource_metrics_key_attributes:
- service.instance.id
- xrpl.node.id
- deployment.environment
- xrpl.network.type
histogram:

17
docker/telemetry/workload/baselines/baseline-timings.json
View File

@@ -1,23 +1,8 @@
{
"_note": "job.* entries were removed on 2026-08-21. They were captured against the old microsecond ladder whose first edge was 100us, with 99.3% of job_queued_us samples beneath it, so job.acceptLedger.queued.p95 = 96.79us was 0.95/0.9926 x 100 -- arithmetic on the bucket edge, not a latency. Recapture them on a node running the re-cut ladder (floor 1us); until then the comparer reports them as \"new metric (not in baseline)\" and gates only the span metrics, which are unaffected. Removed values, for reference: job.acceptLedger.queued.p95=96.79us, job.acceptLedger.running.p95=10562.50us, job.transaction.queued.p95=478.97us, job.transaction.running.p95=494.14us.",
"captured_at": "2026-06-05T18:41:52Z",
"git_sha": "fd1c8c6060f7a15cc9e65b16f99629d9ab7ac7dc",
"metrics": {
"job.acceptLedger.queued.p95": {
"unit": "us",
"value": 96.78571428571428
},
"job.acceptLedger.running.p95": {
"unit": "us",
"value": 10562.499999999945
},
"job.transaction.queued.p95": {
"unit": "us",
"value": 478.96551724137925
},
"job.transaction.running.p95": {
"unit": "us",
"value": 494.1361256544502
},
"span.consensus.accept.p50": {
"unit": "ms",
"value": 1.059405940594059

2
docker/telemetry/workload/regression-thresholds.json
View File

@@ -1,6 +1,6 @@
{
"_description": "Per-metric regression thresholds. A metric regresses when current - baseline exceeds BOTH the percentage and absolute bounds (AND, not OR — this tolerates small-value noise). Defaults apply unless a per-metric override exists.",
"_bucket_note": "SpanMetrics latency histograms use explicit buckets [1,5,10,25,50,100,250,500,1000,2000,3000,4000,5000,10000,30000]ms. A quantile sitting near a low-end boundary can jump a full bucket (e.g. 1ms->5ms) between runs with no real change, so absolute span bounds are set to ~2 low-end bucket widths (10ms) to tolerate that quantization noise while still catching genuine multi-bucket regressions. Second-scale consensus spans now have 2s/3s/4s boundaries (previously all fell in one 1s-5s bucket); their quantiles quantize to ~1s widths there. The job_queue running bound is widened similarly — per-ledger apply work scales with TxQ burst load.",
"_bucket_note": "SpanMetrics latency histograms use explicit buckets [0.01,0.05,0.1,0.25,0.5,1,5,10,25,50,100,250,500]ms then [1,2,3,4,5,10,30]s (20 edges; docker/telemetry/otel-collector-config.yaml is the authoritative list). An earlier version of this note claimed 15 edges starting at 1ms and justified the 10ms absolute span bound as \"~2 low-end bucket widths\" — that derivation is void, because the sub-millisecond edges make the low-end bucket width 0.01ms, not 5ms. The 10ms bound is retained on its own merit: it is roughly two bucket widths in the 5-25ms band where most span quantiles actually sit, so it still absorbs single-bucket quantization jitter while catching multi-bucket regressions. Second-scale consensus spans have 2s/3s/4s boundaries, so their quantiles quantize to ~1s widths there. The job_queue running bound is widened similarly — per-ledger apply work scales with TxQ burst load. NOTE: the native job_queue histograms are microsecond-valued and their ladder was re-cut (floor 100us → 1us), so any job_queue baseline captured before that change is an interpolation artefact, not a latency.",
"defaults": {
"span": {
"p50": { "max_pct_increase": 50.0, "max_abs_increase_ms": 10.0 },

134
docker/telemetry/workload/validate_telemetry.py
View File

@@ -35,6 +35,7 @@ import asyncio
import fnmatch
import json
import logging
import re
import sys
import time
from dataclasses import dataclass, field
@@ -43,6 +44,11 @@ from typing import Any
import aiohttp
# Loki's default query window is the last hour. A validation run finishes in
# minutes, but bounding the range explicitly keeps the query reproducible when
# someone re-runs it later to investigate a result.
LOG_QUERY_WINDOW_SECONDS = 4 * 60 * 60
logger = logging.getLogger("validate_telemetry")
# ---------------------------------------------------------------------------
@@ -175,6 +181,19 @@ class ValidationReport:
# ---------------------------------------------------------------------------
def _log_query_window() -> dict[str, str]:
"""Loki query_range bounds covering a validation run.
Returns:
start/end parameters in nanoseconds since the epoch.
"""
now = time.time()
return {
"start": str(int((now - LOG_QUERY_WINDOW_SECONDS) * 1_000_000_000)),
"end": str(int(now * 1_000_000_000)),
}
async def _tempo_search(
session: aiohttp.ClientSession,
tempo_url: str,
@@ -1092,9 +1111,13 @@ async def validate_log_trace_correlation(
# Check 1: Any logs with trace_id exist.
try:
params = {
"query": '{job="xrpld"} |= "trace_id="',
# Loki's OTLP ingestion promotes service.name to the label
# `service_name`. A `job` attribute is structured metadata, which a
# stream selector cannot match — see otel-collector-config.yaml.
"query": '{service_name="xrpld"} |= "trace_id="',
"limit": 5,
"direction": "backward",
**_log_query_window(),
}
async with session.get(
f"{loki_url}/loki/api/v1/query_range", params=params
@@ -1125,56 +1148,79 @@ async def validate_log_trace_correlation(
)
)
# Check 2: Cross-reference a trace_id from Tempo to Loki.
# Check 2: Cross-reference a trace_id from a log line back to Tempo.
#
# Driven from the log side on purpose. A trace_id only reaches a log line
# when that line is emitted inside a sampled span, and at `warning` level
# most spans produce no log output at all — so picking an arbitrary trace
# from Tempo and expecting it in Loki fails even when correlation works.
# Starting from a logged trace_id tests the invariant that matters: an id
# written to a log must resolve to a trace that was actually exported.
try:
# Get a recent trace from Tempo.
traces = await _tempo_search(
session,
tempo_url,
'{resource.service.name="xrpld"}',
limit=1,
)
loki_params = {
"query": '{service_name="xrpld"} |= "trace_id="',
"limit": 5,
"direction": "backward",
**_log_query_window(),
}
async with session.get(
f"{loki_url}/loki/api/v1/query_range", params=loki_params
) as resp:
data = await resp.json()
streams = data.get("data", {}).get("result", [])
if traces:
trace_id = traces[0].get("traceID", "")
if trace_id:
# Search Loki for this trace_id.
loki_params = {
"query": f'{{job="xrpld"}} |= "{trace_id}"',
"limit": 5,
"direction": "backward",
}
async with session.get(
f"{loki_url}/loki/api/v1/query_range",
params=loki_params,
) as loki_resp:
loki_data = await loki_resp.json()
loki_streams = loki_data.get("data", {}).get("result", [])
loki_count = sum(len(s.get("values", [])) for s in loki_streams)
report.add(
CheckResult(
name="log.trace_id_cross_reference",
category="log",
passed=loki_count > 0,
message=(
f"trace_id {trace_id[:16]}... found in "
f"{loki_count} Loki entries"
if loki_count > 0
else f"trace_id {trace_id[:16]}... not found " "in Loki"
),
details={
"trace_id": trace_id,
"loki_count": loki_count,
},
)
)
else:
logged_ids = [
match.group(1)
for stream in streams
for _, line in stream.get("values", [])
if (match := re.search(r"trace_id=([0-9a-f]{32})", line))
]
if not logged_ids:
report.add(
CheckResult(
name="log.trace_id_cross_reference",
category="log",
passed=False,
message="No traces in Tempo to cross-reference",
message=(
"No logged trace_id to cross-reference. Log lines carry one only "
"when emitted inside a sampled span; raise the log level or widen "
"the workload if this persists."
),
)
)
else:
# Try every id found, not just the first: one unexported trace
# should not fail the check while correlation demonstrably works.
resolved: str | None = None
span_count = 0
unique_ids = list(dict.fromkeys(logged_ids))
for candidate in unique_ids:
try:
spans = await _tempo_get_trace(session, tempo_url, candidate)
except Exception: # noqa: BLE001 - a 404 is "not found", not an error
continue
if spans:
resolved, span_count = candidate, len(spans)
break
report.add(
CheckResult(
name="log.trace_id_cross_reference",
category="log",
passed=resolved is not None,
message=(
f"logged trace_id {resolved[:16]}... resolves to "
f"{span_count} spans in Tempo"
if resolved
else f"none of {len(unique_ids)} logged trace_id(s) resolve in "
"Tempo; the spans they name were not exported"
),
details={
"trace_id": resolved,
"span_count": span_count,
"candidates": len(unique_ids),
},
)
)
except Exception as exc: