diff --git a/docker/telemetry/alloy/config.alloy b/docker/telemetry/alloy/config.alloy index 278f866835..bcfb3030c9 100644 --- a/docker/telemetry/alloy/config.alloy +++ b/docker/telemetry/alloy/config.alloy @@ -310,20 +310,25 @@ otelcol.processor.transform "tier" { otelcol.connector.spanmetrics "xrpld" { namespace = "span" + // Flush every 15s, matching otel-collector-config.grafanacloud.yaml. Alloy + // defaults to 60s, which is longer than [$__rate_interval] at short dashboard + // ranges, so a rate() panel gets under two samples per window and renders + // "No data". Costs ~4x span-metric datapoints. + metrics_flush_interval = "15s" + histogram { unit = "ms" explicit { // Buckets MUST stay strictly ascending (the connector binary-searches // them and silently misbuckets otherwise) and MUST match the two - // otel-collector-config*.yaml lists. Sub-MILLISECOND boundaries exist - // because most xrpld spans sit far below 1ms (tx.preflight means - // ~0.012ms): a 1ms floor put >99.99% of samples in bucket one and pinned - // every p95/p99 at a constant 0.95ms. - // otel-collector-config*.yaml lists. Sub-second boundaries cover - // RPC/tx/ledger spans; 2s-4s resolve second-scale consensus spans - // (consensus.round ~3.9s, consensus.establish ~1.9s) that used to pile - // into one 1s-5s bucket; 10s/30s give the ledger.acquire catch-up tail - // (~17% exceeds 5s) a measurable home so its p95/p99 stop reading +Inf. + // otel-collector-config*.yaml lists. Sub-millisecond boundaries exist + // because most xrpld spans sit far below 1ms: a 1ms floor puts nearly + // every sample in bucket one and pins p95/p99 to that bucket's edge. + // Sub-second boundaries cover RPC, tx and ledger spans. The 2s-4s edges + // resolve second-scale consensus spans, and 10s/30s give the + // ledger.acquire catch-up tail a home so its p95/p99 stop reading +Inf. + // Each edge was chosen from a measured duration distribution for the + // spans it covers. buckets = ["0.01ms", "0.05ms", "0.1ms", "0.25ms", "0.5ms", "1ms", "5ms", "10ms", "25ms", "50ms", "100ms", "250ms", "500ms", "1s", "2s", "3s", "4s", "5s", "10s", "30s"] } } diff --git a/docker/telemetry/docker-compose.yml b/docker/telemetry/docker-compose.yml index 228ab87fb9..1b88a138fe 100644 --- a/docker/telemetry/docker-compose.yml +++ b/docker/telemetry/docker-compose.yml @@ -81,14 +81,20 @@ services: "--config=/etc/otel-collector-config.yaml", "--config=/etc/otel-collector-filestorage.yaml", ] + # Published on the host loopback only. The receivers have no auth and no + # TLS, so only processes on this host may reach them. Note this 127.0.0.1 + # is the HOST interface docker listens on; the container-side bind lives in + # the collector config and is a separate choice. Upstream asks for a + # specific interface rather than 0.0.0.0 on either side (CWE-1327): + # https://opentelemetry.io/docs/security/config-best-practices/ ports: - - "4317:4317" # OTLP gRPC - - "4318:4318" # OTLP HTTP (traces + native OTel metrics) - - "8889:8889" # Prometheus metrics (span_metrics + OTLP) + - "127.0.0.1:4317:4317" # OTLP gRPC + - "127.0.0.1:4318:4318" # OTLP HTTP (traces + native OTel metrics) + - "127.0.0.1:8889:8889" # Prometheus metrics (span_metrics + OTLP) # StatsD UDP port removed — beast::insight now uses native OTLP. # Uncomment if using server=statsd fallback: - # - "8125:8125/udp" - - "13133:13133" # Health check + # - "127.0.0.1:8125:8125/udp" + - "127.0.0.1:13133:13133" # Health check volumes: # Mount collector pipeline config (receivers → processors → exporters) - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml:ro @@ -124,7 +130,7 @@ services: image: grafana/tempo:2.9.4 command: ["-config.file=/etc/tempo.yaml"] ports: - - "3200:3200" # Tempo HTTP API (health check, query) + - "127.0.0.1:3200:3200" # Tempo HTTP API (health check, query) volumes: # Mount Tempo storage and ingestion config - ./tempo.yaml:/etc/tempo.yaml:ro @@ -151,7 +157,7 @@ services: # Pinned to an exact patch release for reproducible, config-stable runs. image: prom/prometheus:v3.13.2 ports: - - "9090:9090" + - "127.0.0.1:9090:9090" volumes: - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro - prometheus-data:/prometheus @@ -190,7 +196,7 @@ services: # `environment:` entry is interpolated from the host shell and would # override (and blank out) the env_file values. ports: - - "3000:3000" # Grafana web UI + - "127.0.0.1:3000:3000" # Grafana web UI volumes: # Auto-provision Tempo datasource and search filters on startup - ./grafana/provisioning:/etc/grafana/provisioning:ro diff --git a/docker/telemetry/grafana/dashboards/job-queue.json b/docker/telemetry/grafana/dashboards/job-queue.json index f31a87be24..31baf79bd2 100644 --- a/docker/telemetry/grafana/dashboards/job-queue.json +++ b/docker/telemetry/grafana/dashboards/job-queue.json @@ -70,44 +70,61 @@ "id": 1 }, { - "title": "Current Job Latency (p99 Gauge) [$xrpl_network_type]", + "title": "Slowest Job Types by p99 Latency [$xrpl_network_type]", "description": "###### What this is:\n*At-a-glance p99 of how long jobs wait in the queue and how long they run once started.*\n\n###### How it's computed:\n*99th percentile derived from the job wait-time and run-time histograms over the last 5 minutes.*\n\n###### Reading it:\n*Lower is better; green under 100ms, yellow to 1s, red beyond 1s.*\n\n###### Healthy range:\n*Wait and exec p99 under 100ms.*\n\n###### Watch for:\n*p99 wait climbing into the red, meaning worker threads are saturated and jobs are backing up.*\n\n###### Keywords:\n- **Job queue / job type** *(per node)* \u2014 xrpld's worker-thread pool; every unit of background work is enqueued under a named job type.\n- **Deferred job** *(per node)* \u2014 a job held back because its type is already at its concurrency limit; the leading indicator of queue backpressure.\n\n###### Computation boundary:\n*Result: Per node \u2014 each series is one server's own value.*\n*Recorded in xrpld code as a native metric (beast::insight); the collector only forwards it; the Grafana query selects and aggregates it.*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/libxrpl/telemetry/MetricsRegistry.cpp)\n\n###### Function:\n`recordJobStarted / recordJobFinished`\n\n###### References:\n[Telemetry glossary](https://github.com/XRPLF/rippled/blob/develop/docs/telemetry-glossary.md#job-queue-job-type)", - "type": "gauge", + "type": "bargauge", "gridPos": { "h": 10, - "w": 12, + "w": 24, "x": 0, "y": 1 }, "options": { + "displayMode": "gradient", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "horizontal", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, - "orientation": "auto", - "showThresholdLabels": false, - "showThresholdMarkers": true + "showUnfilled": true, + "sizing": "manual", + "valueMode": "color" }, "targets": [ { "datasource": { "type": "prometheus" }, - "expr": "label_replace(histogram_quantile(0.99, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(job_queued_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\", handler=~\"$handler\"}[5m]))), \"series\", \"p99 Wait\", \"\", \"\")" + "instant": true, + "range": false, + "expr": "label_replace(histogram_quantile(0.99, sum by (le, job_type, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(job_queued_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\", handler=~\"$handler\"}[5m]))), \"series\", \"p99 Wait\", \"\", \"\")" }, { "datasource": { "type": "prometheus" }, - "expr": "label_replace(histogram_quantile(0.99, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(job_running_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\", handler=~\"$handler\"}[5m]))), \"series\", \"p99 Exec\", \"\", \"\")" + "instant": true, + "range": false, + "expr": "label_replace(histogram_quantile(0.99, sum by (le, job_type, service_instance_id, xrpl_branch, xrpl_node_role, xrpl_work_item) (rate(job_running_us_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", job_type=~\"$job_type\", handler=~\"$handler\"}[5m]))), \"series\", \"p99 Exec\", \"\", \"\")" } ], "fieldConfig": { "defaults": { - "displayName": "${__field.labels.series} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", + "displayName": "${__field.labels.series} ${__field.labels.job_type} [${__field.labels.service_instance_id} ${__field.labels.xrpl_branch} ${__field.labels.xrpl_node_role} ${__field.labels.xrpl_work_item}]", "unit": "\u00b5s", "min": 0, + "max": 1000000, "thresholds": { "mode": "absolute", "steps": [ @@ -131,7 +148,7 @@ "id": 2, "repeat": "xrpl_network_type", "repeatDirection": "h", - "maxPerRow": 2 + "maxPerRow": 1 }, { "title": "Throughput", @@ -738,7 +755,7 @@ "label": "Node", "description": "Filter by rippled node (service.instance.id)", "type": "query", - "query": "label_values(target_info, service_instance_id)", + "query": "label_values(job_queued_us_count, service_instance_id)", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" diff --git a/docker/telemetry/otel-collector-config.grafanacloud.yaml b/docker/telemetry/otel-collector-config.grafanacloud.yaml index 99c1bbbaa3..13b7b144b9 100644 --- a/docker/telemetry/otel-collector-config.grafanacloud.yaml +++ b/docker/telemetry/otel-collector-config.grafanacloud.yaml @@ -78,10 +78,13 @@ processors: batch: timeout: 1s send_batch_size: 100 - # Box-only (uncommitted): probabilistic 0.5% tail sampling on the - # trace-storage branch only. spanmetrics-derived metrics see 100% of spans - # (they run on a separate branch) so RED remain exact; Tempo/Cloud stores - # ~1/200 of traces to keep long-running node cost bounded. + # Probabilistic tail sampling, on the trace-storage branch only. The + # spanmetrics branch sees every span, so span-derived RED metrics stay + # exact while stored trace volume is cut to bound cost on a long-running + # node. Operators who need full traces raise sampling_percentage to 100. + # Note: a log line carries a trace id only when its span was sampled at + # the SDK, so sampling here drops traces whose ids are already in the + # logs, and log-to-trace links for those resolve to nothing. tail_sampling: decision_wait: 10s num_traces: 50000 @@ -187,16 +190,15 @@ connectors: unit: ms explicit: # Buckets MUST stay strictly ascending (the connector binary-searches - # them and silently misbuckets otherwise). Sub-MILLISECOND boundaries - # (0.01-0.5ms) exist because most xrpld spans are far below 1ms -- - # tx.preflight means ~0.012ms, tx.preclaim ~0.15ms -- so a 1ms floor - # put >99.99% of samples in the first bucket and pinned every p95/p99 - # at a constant 0.95ms. Sub-second boundaries cover - # RPC/tx/ledger spans; the 2s-4s boundaries resolve second-scale - # consensus spans (consensus.round ~3.9s, consensus.establish ~1.9s) - # that used to pile into a single 1s-5s bucket; 10s/30s give the - # ledger.acquire tail (~17% exceeds 5s during catch-up) a measurable - # home so its p95/p99 stop reading as +Inf. + # them and silently misbuckets otherwise) and MUST match the Alloy + # list and the other collector config. Sub-millisecond boundaries + # exist because most xrpld spans sit far below 1ms: a 1ms floor puts + # nearly every sample in the first bucket and pins p95/p99 to that + # bucket's edge. Sub-second boundaries cover RPC, tx and ledger spans. + # The 2s-4s edges resolve second-scale consensus spans, and 10s/30s + # give the ledger.acquire catch-up tail a home so its p95/p99 stop + # reading as +Inf. Each edge was chosen from a measured duration + # distribution for the spans it covers. buckets: [ 0.01ms, diff --git a/docker/telemetry/otel-collector-config.yaml b/docker/telemetry/otel-collector-config.yaml index cb6b580aff..d029e655fb 100644 --- a/docker/telemetry/otel-collector-config.yaml +++ b/docker/telemetry/otel-collector-config.yaml @@ -168,16 +168,15 @@ connectors: unit: ms explicit: # Buckets MUST stay strictly ascending (the connector binary-searches - # them and silently misbuckets otherwise). Sub-MILLISECOND boundaries - # (0.01-0.5ms) exist because most xrpld spans are far below 1ms -- - # tx.preflight means ~0.012ms, tx.preclaim ~0.15ms -- so a 1ms floor - # put >99.99% of samples in the first bucket and pinned every p95/p99 - # at a constant 0.95ms. Sub-second boundaries cover - # RPC/tx/ledger spans; the 2s-4s boundaries resolve second-scale - # consensus spans (consensus.round ~3.9s, consensus.establish ~1.9s) - # that used to pile into a single 1s-5s bucket; 10s/30s give the - # ledger.acquire tail (~17% exceeds 5s during catch-up) a measurable - # home so its p95/p99 stop reading as +Inf. + # them and silently misbuckets otherwise) and MUST match the Alloy + # list and the other collector config. Sub-millisecond boundaries + # exist because most xrpld spans sit far below 1ms: a 1ms floor puts + # nearly every sample in the first bucket and pins p95/p99 to that + # bucket's edge. Sub-second boundaries cover RPC, tx and ledger spans. + # The 2s-4s edges resolve second-scale consensus spans, and 10s/30s + # give the ledger.acquire catch-up tail a home so its p95/p99 stop + # reading as +Inf. Each edge was chosen from a measured duration + # distribution for the spans it covers. buckets: [ 0.01ms,