mirror of
https://github.com/XRPLF/rippled.git
synced 2026-09-27 15:28:03 +00:00
Merge branch 'pratik/otel-phase10-workload-validation' into pratik/otel-sync-diagnostics
This commit is contained in:
Pratik Mankawde
@@ -310,20 +310,25 @@ otelcol.processor.transform "tier" {
|
||||
otelcol.connector.spanmetrics "xrpld" {
|
||||
namespace = "span"
|
||||
|
||||
// Flush every 15s, matching otel-collector-config.grafanacloud.yaml. Alloy
|
||||
// defaults to 60s, which is longer than [$__rate_interval] at short dashboard
|
||||
// ranges, so a rate() panel gets under two samples per window and renders
|
||||
// "No data". Costs ~4x span-metric datapoints.
|
||||
metrics_flush_interval = "15s"
|
||||
|
||||
histogram {
|
||||
unit = "ms"
|
||||
explicit {
|
||||
// Buckets MUST stay strictly ascending (the connector binary-searches
|
||||
// them and silently misbuckets otherwise) and MUST match the two
|
||||
// otel-collector-config*.yaml lists. Sub-MILLISECOND boundaries exist
|
||||
// because most xrpld spans sit far below 1ms (tx.preflight means
|
||||
// ~0.012ms): a 1ms floor put >99.99% of samples in bucket one and pinned
|
||||
// every p95/p99 at a constant 0.95ms.
|
||||
// otel-collector-config*.yaml lists. Sub-second boundaries cover
|
||||
// RPC/tx/ledger spans; 2s-4s resolve second-scale consensus spans
|
||||
// (consensus.round ~3.9s, consensus.establish ~1.9s) that used to pile
|
||||
// into one 1s-5s bucket; 10s/30s give the ledger.acquire catch-up tail
|
||||
// (~17% exceeds 5s) a measurable home so its p95/p99 stop reading +Inf.
|
||||
// otel-collector-config*.yaml lists. Sub-millisecond boundaries exist
|
||||
// because most xrpld spans sit far below 1ms: a 1ms floor puts nearly
|
||||
// every sample in bucket one and pins p95/p99 to that bucket's edge.
|
||||
// Sub-second boundaries cover RPC, tx and ledger spans. The 2s-4s edges
|
||||
// resolve second-scale consensus spans, and 10s/30s give the
|
||||
// ledger.acquire catch-up tail a home so its p95/p99 stop reading +Inf.
|
||||
// Each edge was chosen from a measured duration distribution for the
|
||||
// spans it covers.
|
||||
buckets = ["0.01ms", "0.05ms", "0.1ms", "0.25ms", "0.5ms", "1ms", "5ms", "10ms", "25ms", "50ms", "100ms", "250ms", "500ms", "1s", "2s", "3s", "4s", "5s", "10s", "30s"]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -81,14 +81,20 @@ services:
|
||||
"--config=/etc/otel-collector-config.yaml",
|
||||
"--config=/etc/otel-collector-filestorage.yaml",
|
||||
]
|
||||
# Published on the host loopback only. The receivers have no auth and no
|
||||
# TLS, so only processes on this host may reach them. Note this 127.0.0.1
|
||||
# is the HOST interface docker listens on; the container-side bind lives in
|
||||
# the collector config and is a separate choice. Upstream asks for a
|
||||
# specific interface rather than 0.0.0.0 on either side (CWE-1327):
|
||||
# https://opentelemetry.io/docs/security/config-best-practices/
|
||||
ports:
|
||||
- "4317:4317" # OTLP gRPC
|
||||
- "4318:4318" # OTLP HTTP (traces + native OTel metrics)
|
||||
- "8889:8889" # Prometheus metrics (span_metrics + OTLP)
|
||||
- "127.0.0.1:4317:4317" # OTLP gRPC
|
||||
- "127.0.0.1:4318:4318" # OTLP HTTP (traces + native OTel metrics)
|
||||
- "127.0.0.1:8889:8889" # Prometheus metrics (span_metrics + OTLP)
|
||||
# StatsD UDP port removed — beast::insight now uses native OTLP.
|
||||
# Uncomment if using server=statsd fallback:
|
||||
# - "8125:8125/udp"
|
||||
- "13133:13133" # Health check
|
||||
# - "127.0.0.1:8125:8125/udp"
|
||||
- "127.0.0.1:13133:13133" # Health check
|
||||
volumes:
|
||||
# Mount collector pipeline config (receivers → processors → exporters)
|
||||
- ./otel-collector-config.yaml:/etc/otel-collector-config.yaml:ro
|
||||
@@ -124,7 +130,7 @@ services:
|
||||
image: grafana/tempo:2.9.4
|
||||
command: ["-config.file=/etc/tempo.yaml"]
|
||||
ports:
|
||||
- "3200:3200" # Tempo HTTP API (health check, query)
|
||||
- "127.0.0.1:3200:3200" # Tempo HTTP API (health check, query)
|
||||
volumes:
|
||||
# Mount Tempo storage and ingestion config
|
||||
- ./tempo.yaml:/etc/tempo.yaml:ro
|
||||
@@ -151,7 +157,7 @@ services:
|
||||
# Pinned to an exact patch release for reproducible, config-stable runs.
|
||||
image: prom/prometheus:v3.13.2
|
||||
ports:
|
||||
- "9090:9090"
|
||||
- "127.0.0.1:9090:9090"
|
||||
volumes:
|
||||
- ./prometheus.yml:/etc/prometheus/prometheus.yml:ro
|
||||
- prometheus-data:/prometheus
|
||||
@@ -190,7 +196,7 @@ services:
|
||||
# `environment:` entry is interpolated from the host shell and would
|
||||
# override (and blank out) the env_file values.
|
||||
ports:
|
||||
- "3000:3000" # Grafana web UI
|
||||
- "127.0.0.1:3000:3000" # Grafana web UI
|
||||
volumes:
|
||||
# Auto-provision Tempo datasource and search filters on startup
|
||||
- ./grafana/provisioning:/etc/grafana/provisioning:ro
|
||||
|
||||
@@ -70,44 +70,61 @@
|
||||
"id": 1
|
||||
},
|
||||
{
|
||||
"title": "Current Job Latency (p99 Gauge) [$xrpl_network_type]",
|
||||
"title": "Slowest Job Types by p99 Latency [$xrpl_network_type]",
|
||||
"description": "###### What this is:\n*At-a-glance p99 of how long jobs wait in the queue and how long they run once started.*\n\n###### How it's computed:\n*99th percentile derived from the job wait-time and run-time histograms over the last 5 minutes.*\n\n###### Reading it:\n*Lower is better; green under 100ms, yellow to 1s, red beyond 1s.*\n\n###### Healthy range:\n*Wait and exec p99 under 100ms.*\n\n###### Watch for:\n*p99 wait climbing into the red, meaning worker threads are saturated and jobs are backing up.*\n\n###### Keywords:\n- **Job queue / job type** *(per node)* \u2014 xrpld's worker-thread pool; every unit of background work is enqueued under a named job type.\n- **Deferred job** *(per node)* \u2014 a job held back because its type is already at its concurrency limit; the leading indicator of queue backpressure.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordJobStarted / recordJobFinished`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#job-queue-job-type)",
|
||||
"type": "gauge",
|
||||
"type": "bargauge",
|
||||
"gridPos": {
|
||||
"h": 10,
|
||||
"w": 12,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 1
|
||||
},
|
||||
"options": {
|
||||
"displayMode": "gradient",
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": false
|
||||
},
|
||||
"maxVizHeight": 300,
|
||||
"minVizHeight": 16,
|
||||
"minVizWidth": 8,
|
||||
"namePlacement": "auto",
|
||||
"orientation": "horizontal",
|
||||
"reduceOptions": {
|
||||
"calcs": ["lastNotNull"],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"orientation": "auto",
|
||||
"showThresholdLabels": false,
|
||||
"showThresholdMarkers": true
|
||||
"showUnfilled": true,
|
||||
"sizing": "manual",
|
||||
"valueMode": "color"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus"
|
||||
},
|
||||
"expr": "label_replace(histogram_quantile(0.99, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(job_queued_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\", handler=~\"$handler\"}[5m]))), \"series\", \"p99 Wait\", \"\", \"\")"
|
||||
"instant": true,
|
||||
"range": false,
|
||||
"expr": "label_replace(histogram_quantile(0.99, sum by (le, job_type, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(job_queued_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\", handler=~\"$handler\"}[5m]))), \"series\", \"p99 Wait\", \"\", \"\")"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus"
|
||||
},
|
||||
"expr": "label_replace(histogram_quantile(0.99, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(job_running_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\", handler=~\"$handler\"}[5m]))), \"series\", \"p99 Exec\", \"\", \"\")"
|
||||
"instant": true,
|
||||
"range": false,
|
||||
"expr": "label_replace(histogram_quantile(0.99, sum by (le, job_type, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(job_running_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\", handler=~\"$handler\"}[5m]))), \"series\", \"p99 Exec\", \"\", \"\")"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
|
||||
"displayName": "${__field.labels.series} ${__field.labels.job_type} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]",
|
||||
"unit": "\u00b5s",
|
||||
"min": 0,
|
||||
"max": 1000000,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
@@ -131,7 +148,7 @@
|
||||
"id": 2,
|
||||
"repeat": "xrpl_network_type",
|
||||
"repeatDirection": "h",
|
||||
"maxPerRow": 2
|
||||
"maxPerRow": 1
|
||||
},
|
||||
{
|
||||
"title": "Throughput",
|
||||
@@ -738,7 +755,7 @@
|
||||
"label": "Node",
|
||||
"description": "Filter by rippled node (service.instance.id)",
|
||||
"type": "query",
|
||||
"query": "label_values(target_info, service_instance_id)",
|
||||
"query": "label_values(job_queued_us_count, service_instance_id)",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
|
||||
@@ -78,10 +78,13 @@ processors:
|
||||
batch:
|
||||
timeout: 1s
|
||||
send_batch_size: 100
|
||||
# Box-only (uncommitted): probabilistic 0.5% tail sampling on the
|
||||
# trace-storage branch only. spanmetrics-derived metrics see 100% of spans
|
||||
# (they run on a separate branch) so RED remain exact; Tempo/Cloud stores
|
||||
# ~1/200 of traces to keep long-running node cost bounded.
|
||||
# Probabilistic tail sampling, on the trace-storage branch only. The
|
||||
# spanmetrics branch sees every span, so span-derived RED metrics stay
|
||||
# exact while stored trace volume is cut to bound cost on a long-running
|
||||
# node. Operators who need full traces raise sampling_percentage to 100.
|
||||
# Note: a log line carries a trace id only when its span was sampled at
|
||||
# the SDK, so sampling here drops traces whose ids are already in the
|
||||
# logs, and log-to-trace links for those resolve to nothing.
|
||||
tail_sampling:
|
||||
decision_wait: 10s
|
||||
num_traces: 50000
|
||||
@@ -187,16 +190,15 @@ connectors:
|
||||
unit: ms
|
||||
explicit:
|
||||
# Buckets MUST stay strictly ascending (the connector binary-searches
|
||||
# them and silently misbuckets otherwise). Sub-MILLISECOND boundaries
|
||||
# (0.01-0.5ms) exist because most xrpld spans are far below 1ms --
|
||||
# tx.preflight means ~0.012ms, tx.preclaim ~0.15ms -- so a 1ms floor
|
||||
# put >99.99% of samples in the first bucket and pinned every p95/p99
|
||||
# at a constant 0.95ms. Sub-second boundaries cover
|
||||
# RPC/tx/ledger spans; the 2s-4s boundaries resolve second-scale
|
||||
# consensus spans (consensus.round ~3.9s, consensus.establish ~1.9s)
|
||||
# that used to pile into a single 1s-5s bucket; 10s/30s give the
|
||||
# ledger.acquire tail (~17% exceeds 5s during catch-up) a measurable
|
||||
# home so its p95/p99 stop reading as +Inf.
|
||||
# them and silently misbuckets otherwise) and MUST match the Alloy
|
||||
# list and the other collector config. Sub-millisecond boundaries
|
||||
# exist because most xrpld spans sit far below 1ms: a 1ms floor puts
|
||||
# nearly every sample in the first bucket and pins p95/p99 to that
|
||||
# bucket's edge. Sub-second boundaries cover RPC, tx and ledger spans.
|
||||
# The 2s-4s edges resolve second-scale consensus spans, and 10s/30s
|
||||
# give the ledger.acquire catch-up tail a home so its p95/p99 stop
|
||||
# reading as +Inf. Each edge was chosen from a measured duration
|
||||
# distribution for the spans it covers.
|
||||
buckets:
|
||||
[
|
||||
0.01ms,
|
||||
|
||||
@@ -168,16 +168,15 @@ connectors:
|
||||
unit: ms
|
||||
explicit:
|
||||
# Buckets MUST stay strictly ascending (the connector binary-searches
|
||||
# them and silently misbuckets otherwise). Sub-MILLISECOND boundaries
|
||||
# (0.01-0.5ms) exist because most xrpld spans are far below 1ms --
|
||||
# tx.preflight means ~0.012ms, tx.preclaim ~0.15ms -- so a 1ms floor
|
||||
# put >99.99% of samples in the first bucket and pinned every p95/p99
|
||||
# at a constant 0.95ms. Sub-second boundaries cover
|
||||
# RPC/tx/ledger spans; the 2s-4s boundaries resolve second-scale
|
||||
# consensus spans (consensus.round ~3.9s, consensus.establish ~1.9s)
|
||||
# that used to pile into a single 1s-5s bucket; 10s/30s give the
|
||||
# ledger.acquire tail (~17% exceeds 5s during catch-up) a measurable
|
||||
# home so its p95/p99 stop reading as +Inf.
|
||||
# them and silently misbuckets otherwise) and MUST match the Alloy
|
||||
# list and the other collector config. Sub-millisecond boundaries
|
||||
# exist because most xrpld spans sit far below 1ms: a 1ms floor puts
|
||||
# nearly every sample in the first bucket and pins p95/p99 to that
|
||||
# bucket's edge. Sub-second boundaries cover RPC, tx and ledger spans.
|
||||
# The 2s-4s edges resolve second-scale consensus spans, and 10s/30s
|
||||
# give the ledger.acquire catch-up tail a home so its p95/p99 stop
|
||||
# reading as +Inf. Each edge was chosen from a measured duration
|
||||
# distribution for the spans it covers.
|
||||
buckets:
|
||||
[
|
||||
0.01ms,
|
||||
|
||||
Reference in New Issue
Block a user