From f52673b6c1588a515ef7eb2ae0f80a7cbee8d2be Mon Sep 17 00:00:00 2001 From: Pratik Mankawde <3397372+pratikmankawde@users.noreply.github.com> Date: Wed, 22 Jul 2026 12:23:21 +0100 Subject: [PATCH] feat(telemetry): correct ledger-close-time derivation on dashboards MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The "ledger close time" was mis-derived and partly un-queryable: - Build vs Close Duration derived close time from the consensus.ledger_close span, which only wraps the onClose() prologue (~0.8ms live) — not the close. Repoint the close series to consensus.round (full round, live P95 ~4.8s). - The network close-time value (close_time) lived only as a span attribute, un-queryable in Prometheus and unfit as a spanmetrics label (monotonic timestamp -> unbounded cardinality). Expose it as last_close_time on the existing server_info observable gauge (native gauge, no new instrument). - Add a "Ledger Close Interval & Age" panel to ledger-operations and node-health: interval = 1/rate(ledgers_closed_total) (counter-based, scrape-independent); age = time() - (last_close_time + epoch_offset). A gauge delta is deliberately NOT used for the interval — a timestamp gauge's delta aliases to the scrape period, not the close cadence (verified live). Guardrail comments in both collector configs record why close_time must never become a spanmetrics dimension. Docs (09-reference) and the operator runbook updated to match. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../09-data-collection-reference.md | 21 ++-- .../grafana/dashboards/ledger-operations.json | 52 ++++++++- .../grafana/dashboards/node-health.json | 107 ++++++++++++++++++ .../otel-collector-config.grafanacloud.yaml | 6 + docker/telemetry/otel-collector-config.yaml | 6 + docs/telemetry-runbook.md | 66 +++++------ src/xrpld/telemetry/MetricsRegistry.cpp | 16 +++ 7 files changed, 230 insertions(+), 44 deletions(-) diff --git a/OpenTelemetryPlan/09-data-collection-reference.md b/OpenTelemetryPlan/09-data-collection-reference.md index d2da48fd84..8a7cf88780 100644 --- a/OpenTelemetryPlan/09-data-collection-reference.md +++ b/OpenTelemetryPlan/09-data-collection-reference.md @@ -845,16 +845,17 @@ async callbacks for new categories. #### Server Info (via OTel MetricsRegistry) -| Prometheus Metric | Type | Labels | Description | -| --------------------------------------------------- | ----- | -------- | -------------------------------------------- | -| `server_info{metric="server_state"}` | Gauge | `metric` | Operating mode (0=DISCONNECTED .. 4=FULL) | -| `server_info{metric="uptime"}` | Gauge | `metric` | Seconds since server start | -| `server_info{metric="peers"}` | Gauge | `metric` | Total connected peers | -| `server_info{metric="validated_ledger_seq"}` | Gauge | `metric` | Validated ledger sequence number | -| `server_info{metric="ledger_current_index"}` | Gauge | `metric` | Current open ledger sequence | -| `server_info{metric="peer_disconnects_resources"}` | Gauge | `metric` | Cumulative resource-related peer disconnects | -| `server_info{metric="last_close_proposers"}` | Gauge | `metric` | Proposers in last closed round | -| `server_info{metric="last_close_converge_time_ms"}` | Gauge | `metric` | Last close convergence time (milliseconds) | +| Prometheus Metric | Type | Labels | Description | +| --------------------------------------------------- | ----- | -------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `server_info{metric="server_state"}` | Gauge | `metric` | Operating mode (0=DISCONNECTED .. 4=FULL) | +| `server_info{metric="uptime"}` | Gauge | `metric` | Seconds since server start | +| `server_info{metric="peers"}` | Gauge | `metric` | Total connected peers | +| `server_info{metric="validated_ledger_seq"}` | Gauge | `metric` | Validated ledger sequence number | +| `server_info{metric="ledger_current_index"}` | Gauge | `metric` | Current open ledger sequence | +| `server_info{metric="peer_disconnects_resources"}` | Gauge | `metric` | Cumulative resource-related peer disconnects | +| `server_info{metric="last_close_proposers"}` | Gauge | `metric` | Proposers in last closed round | +| `server_info{metric="last_close_converge_time_ms"}` | Gauge | `metric` | Last close convergence time (milliseconds) | +| `server_info{metric="last_close_time"}` | Gauge | `metric` | Network close time of last closed ledger (NetClock secs since Ripple epoch). Query `time() - (value + 946684800)` for last-close age (staleness). Use `1/rate(ledgers_closed_total)` — not a gauge delta — for the close interval | #### Build Info (via OTel MetricsRegistry) diff --git a/docker/telemetry/grafana/dashboards/ledger-operations.json b/docker/telemetry/grafana/dashboards/ledger-operations.json index 926e54af63..c5ea53e679 100644 --- a/docker/telemetry/grafana/dashboards/ledger-operations.json +++ b/docker/telemetry/grafana/dashboards/ledger-operations.json @@ -302,7 +302,7 @@ }, { "title": "Build vs Close Duration", - "description": "###### What this is:\n*Ledger build time compared with the total consensus ledger-close time, both at the 95th percentile.*\n\n###### How it's computed:\n*Two 95th-percentile duration series over 5 minutes: ledger construction and the full consensus close, per node.*\n\n###### Reading it:\n*Build should sit below close; the gap is consensus overhead outside construction.*\n\n###### Healthy range:\n*Close a bit above build; both workload-dependent and under the round interval.*\n\n###### Watch for:\n*A widening gap (consensus-pipeline overhead growing) or build time approaching close time.*\n\n###### Source:\n[BuildLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/BuildLedger.cpp) \u00b7 [RCLConsensus.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/RCLConsensus.cpp)\n\n###### Function:\n`buildLedgerImpl ; RCLConsensus::Adaptor::onClose`", + "description": "###### What this is:\n*Ledger build time compared with the full consensus round duration, both at the 95th percentile.*\n\n###### How it's computed:\n*Two 95th-percentile duration series over 5 minutes: ledger construction (ledger.build span) and the whole consensus round from open to accept (consensus.round span), per node.*\n\n###### Reading it:\n*Build should sit well below the round; the gap is consensus wait time (proposing, converging, validating) outside construction.*\n\n###### Healthy range:\n*Round tracks the network close interval (~3-5s on mainnet); build is a fraction of it (tens to hundreds of ms).*\n\n###### Watch for:\n*Build time approaching the round duration \u2014 construction is dominating the close and leaving little slack.*\n\n###### Note:\n*The close series uses consensus.round, not consensus.ledger_close: the latter span only wraps the onClose() prologue (sub-millisecond) and is not the ledger close time.*\n\n###### Source:\n[BuildLedger.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/ledger/detail/BuildLedger.cpp) \u00b7 [RCLConsensus.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/RCLConsensus.cpp)\n\n###### Function:\n`buildLedgerImpl ; RCLConsensus::Adaptor::onClose (round span)`", "type": "timeseries", "gridPos": { "h": 8, @@ -330,7 +330,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"consensus.ledger_close\"}[5m]))), \"series\", \"P95 consensus.ledger_close\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + "expr": "label_replace(label_join(label_replace(histogram_quantile(0.95, sum by (le, service_instance_id, xrpl_branch, xrpl_node_role) (rate(span_duration_milliseconds_bucket{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\", span_name=\"consensus.round\"}[5m]))), \"series\", \"P95 Close (consensus.round)\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" } ], "fieldConfig": { @@ -347,6 +347,54 @@ }, "overrides": [] } + }, + { + "title": "Ledger Close Interval & Age", + "description": "###### What this is:\n*The wall-clock time between consecutive ledger closes — the network close cadence.*\n\n###### How it's computed:\n*Close Interval: 1 / rate(ledgers_closed_total), the average seconds between closes from the monotonic close counter (scrape-independent, unlike a gauge delta which would alias to the scrape period). Last-Close Age: time() minus the last-close network time (server_info last_close_time gauge + Ripple-epoch offset), i.e. seconds since the last ledger closed.*\n\n###### Reading it:\n*A steady line near the network's target close interval.*\n\n###### Healthy range:\n*About 3-5s on mainnet; workload-dependent on test networks.*\n\n###### Watch for:\n*A rising interval (consensus slowing or the node lagging) or a flat line at zero (ledgers no longer closing).*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp) · [RCLConsensus.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/RCLConsensus.cpp)\n\n###### Function:\n`registerServerInfoGauge (last_close_time) ; ledgers_closed_total`", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "options": { + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(time() - (server_info{metric=\"last_close_time\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"} + 946684800), \"series\", \"Last-Close Age (s)\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(1 / sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(ledgers_closed_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])), \"series\", \"Close Interval\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")" + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "unit": "s", + "custom": { + "axisLabel": "Seconds", + "spanNulls": 1800000, + "insertNulls": false, + "showPoints": "auto", + "pointSize": 3 + } + }, + "overrides": [] + } } ], "schemaVersion": 39, diff --git a/docker/telemetry/grafana/dashboards/node-health.json b/docker/telemetry/grafana/dashboards/node-health.json index 5b2b33c05e..eb1744b2a6 100644 --- a/docker/telemetry/grafana/dashboards/node-health.json +++ b/docker/telemetry/grafana/dashboards/node-health.json @@ -3693,6 +3693,113 @@ "title": "Last Close \u2014 Proposers", "type": "timeseries" }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "###### What this is:\n*The wall-clock time between consecutive ledger closes \u2014 the network close cadence.*\n\n###### How it's computed:\n*Close Interval: 1 / rate(ledgers_closed_total), the average seconds between closes from the monotonic close counter (scrape-independent, unlike a gauge delta which would alias to the scrape period). Last-Close Age: time() minus the last-close network time (server_info last_close_time gauge + Ripple-epoch offset), i.e. seconds since the last ledger closed.*\n\n###### Reading it:\n*A steady line near the network's target close interval.*\n\n###### Healthy range:\n*About 3-5s on mainnet; workload-dependent on test networks.*\n\n###### Watch for:\n*A rising interval (consensus slowing or the node lagging) or a flat line at zero (ledgers no longer closing).*\n\n###### Source:\n[MetricsRegistry.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/telemetry/MetricsRegistry.cpp) \u00b7 [RCLConsensus.cpp](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/app/consensus/RCLConsensus.cpp)\n\n###### Function:\n`registerServerInfoGauge (last_close_time) ; ledgers_closed_total`", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Seconds", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.series} ${__field.labels.xrpl_ident}", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "s" + } + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 236 + }, + "id": 128, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "enableFacetedFilter": false, + "overflow": "ellipsis", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "13.2.0-28926505616", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(time() - (server_info{metric=\"last_close_time\", service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"} + 946684800), \"series\", \"Last-Close Age (s)\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "label_replace(label_join(label_replace(1 / sum by (service_instance_id, xrpl_branch, xrpl_node_role) (rate(ledgers_closed_total{service_instance_id=~\"$node\", deployment_environment=~\"$deployment_environment\", xrpl_network_type=~\"$xrpl_network_type\", service_name=~\"$service_name\", xrpl_work_item=~\"$xrpl_work_item\", xrpl_branch=~\"$xrpl_branch\", xrpl_node_role=~\"$xrpl_node_role\"}[$__rate_interval])), \"series\", \"Close Interval\", \"\", \"\"), \"xrpl_ident\", \", \", \"service_instance_id\", \"xrpl_branch\", \"xrpl_work_item\"), \"xrpl_ident\", \"[$1]\", \"xrpl_ident\", \"(?:, )*(.*[^, ])(?:, )*\")", + "refId": "B" + } + ], + "title": "Ledger Close Interval & Age", + "type": "timeseries" + }, { "datasource": { "type": "prometheus", diff --git a/docker/telemetry/otel-collector-config.grafanacloud.yaml b/docker/telemetry/otel-collector-config.grafanacloud.yaml index 50efeb5d57..a78bdf751f 100644 --- a/docker/telemetry/otel-collector-config.grafanacloud.yaml +++ b/docker/telemetry/otel-collector-config.grafanacloud.yaml @@ -150,6 +150,12 @@ connectors: - name: command - name: rpc_status - name: consensus_mode + # Only the boolean close_time_correct is promoted, never the close_time + # value itself: close_time is a monotonic NetClock timestamp, so a + # dimension on it would mint a new metric series every ledger (unbounded + # cardinality). The close-time value is exposed as the server_info + # last_close_time gauge instead (for last-close age); the close interval + # comes from the ledgers_closed_total counter. - name: close_time_correct - name: consensus_state - name: local diff --git a/docker/telemetry/otel-collector-config.yaml b/docker/telemetry/otel-collector-config.yaml index 6200318b56..a2d1967f66 100644 --- a/docker/telemetry/otel-collector-config.yaml +++ b/docker/telemetry/otel-collector-config.yaml @@ -129,6 +129,12 @@ connectors: - name: command - name: rpc_status - name: consensus_mode + # Only the boolean close_time_correct is promoted, never the close_time + # value itself: close_time is a monotonic NetClock timestamp, so a + # dimension on it would mint a new metric series every ledger (unbounded + # cardinality). The close-time value is exposed as the server_info + # last_close_time gauge instead (for last-close age); the close interval + # comes from the ledgers_closed_total counter. - name: close_time_correct - name: consensus_state - name: local diff --git a/docs/telemetry-runbook.md b/docs/telemetry-runbook.md index 6897d8a18b..594fe9a931 100644 --- a/docs/telemetry-runbook.md +++ b/docs/telemetry-runbook.md @@ -761,28 +761,29 @@ The `OTelCollector` implementation exports metrics via OTLP/HTTP to the same OTe These gauges are exported via the OTel Metrics SDK `PeriodicMetricReader` (10s interval), NOT through beast::insight. -| Prometheus Metric | Source | Description | -| --------------------------------------------------- | ------------------- | ------------------------------------------------ | -| `server_info{metric="server_state"}` | MetricsRegistry.cpp | Operating mode (0=DISCONNECTED .. 4=FULL) | -| `server_info{metric="uptime"}` | MetricsRegistry.cpp | Seconds since server start | -| `server_info{metric="peers"}` | MetricsRegistry.cpp | Total connected peers | -| `server_info{metric="validated_ledger_seq"}` | MetricsRegistry.cpp | Validated ledger sequence number | -| `server_info{metric="ledger_current_index"}` | MetricsRegistry.cpp | Current open ledger sequence | -| `server_info{metric="peer_disconnects_resources"}` | MetricsRegistry.cpp | Cumulative resource-related peer disconnects | -| `server_info{metric="last_close_proposers"}` | MetricsRegistry.cpp | Proposers in last closed round | -| `server_info{metric="last_close_converge_time_ms"}` | MetricsRegistry.cpp | Last close convergence time (ms) | -| `build_info{version=""}` | MetricsRegistry.cpp | Info-style metric (always 1) | -| `complete_ledgers{bound="start\|end",index=""}` | MetricsRegistry.cpp | Complete ledger range start/end pairs | -| `db_metrics{metric="db_kb_total"}` | MetricsRegistry.cpp | Total database size (KB) | -| `db_metrics{metric="db_kb_ledger"}` | MetricsRegistry.cpp | Ledger database size (KB) | -| `db_metrics{metric="db_kb_transaction"}` | MetricsRegistry.cpp | Transaction database size (KB) | -| `db_metrics{metric="historical_perminute"}` | MetricsRegistry.cpp | Historical ledger fetches per minute | -| `cache_metrics{metric="AL_size"}` | MetricsRegistry.cpp | AcceptedLedger cache size | -| `nodestore_state{metric="node_reads_duration_us"}` | MetricsRegistry.cpp | Cumulative read time (microseconds) | -| `nodestore_state{metric="read_request_bundle"}` | MetricsRegistry.cpp | Read request bundle count | -| `nodestore_state{metric="read_threads_running"}` | MetricsRegistry.cpp | Active read threads | -| `nodestore_state{metric="read_threads_total"}` | MetricsRegistry.cpp | Total read threads configured | -| `rpc_in_flight_requests` | PerfLogImp.cpp | RPC requests currently executing (UpDownCounter) | +| Prometheus Metric | Source | Description | +| --------------------------------------------------- | ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `server_info{metric="server_state"}` | MetricsRegistry.cpp | Operating mode (0=DISCONNECTED .. 4=FULL) | +| `server_info{metric="uptime"}` | MetricsRegistry.cpp | Seconds since server start | +| `server_info{metric="peers"}` | MetricsRegistry.cpp | Total connected peers | +| `server_info{metric="validated_ledger_seq"}` | MetricsRegistry.cpp | Validated ledger sequence number | +| `server_info{metric="ledger_current_index"}` | MetricsRegistry.cpp | Current open ledger sequence | +| `server_info{metric="peer_disconnects_resources"}` | MetricsRegistry.cpp | Cumulative resource-related peer disconnects | +| `server_info{metric="last_close_proposers"}` | MetricsRegistry.cpp | Proposers in last closed round | +| `server_info{metric="last_close_converge_time_ms"}` | MetricsRegistry.cpp | Last close convergence time (ms) | +| `server_info{metric="last_close_time"}` | MetricsRegistry.cpp | Network close time of last closed ledger (NetClock secs since Ripple epoch). Age = `time() - (value + 946684800)`; close interval = `1/rate(ledgers_closed_total)`, not a gauge delta | +| `build_info{version=""}` | MetricsRegistry.cpp | Info-style metric (always 1) | +| `complete_ledgers{bound="start\|end",index=""}` | MetricsRegistry.cpp | Complete ledger range start/end pairs | +| `db_metrics{metric="db_kb_total"}` | MetricsRegistry.cpp | Total database size (KB) | +| `db_metrics{metric="db_kb_ledger"}` | MetricsRegistry.cpp | Ledger database size (KB) | +| `db_metrics{metric="db_kb_transaction"}` | MetricsRegistry.cpp | Transaction database size (KB) | +| `db_metrics{metric="historical_perminute"}` | MetricsRegistry.cpp | Historical ledger fetches per minute | +| `cache_metrics{metric="AL_size"}` | MetricsRegistry.cpp | AcceptedLedger cache size | +| `nodestore_state{metric="node_reads_duration_us"}` | MetricsRegistry.cpp | Cumulative read time (microseconds) | +| `nodestore_state{metric="read_request_bundle"}` | MetricsRegistry.cpp | Read request bundle count | +| `nodestore_state{metric="read_threads_running"}` | MetricsRegistry.cpp | Active read threads | +| `nodestore_state{metric="read_threads_total"}` | MetricsRegistry.cpp | Total read threads configured | +| `rpc_in_flight_requests` | PerfLogImp.cpp | RPC requests currently executing (UpDownCounter) | #### Counters @@ -990,16 +991,17 @@ Ten dashboards are pre-provisioned in `docker/telemetry/grafana/dashboards/`: ### Ledger Operations (`ledger-operations`) -| Panel | Type | PromQL | Labels Used | -| ----------------------- | ---------- | ---------------------------------------------- | ----------- | -| Ledger Build Rate | stat | `ledger.build` call rate | — | -| Ledger Build Duration | timeseries | p95/p50 of `ledger.build` | — | -| Ledger Validation Rate | stat | `ledger.validate` call rate | — | -| Build Duration Heatmap | heatmap | `ledger.build` histogram buckets | `le` | -| TX Apply Duration | timeseries | p95/p50 of `tx.apply` | — | -| TX Apply Rate | timeseries | `tx.apply` call rate | — | -| Ledger Store Rate | stat | `ledger.store` call rate | — | -| Build vs Close Duration | timeseries | p95 `ledger.build` vs `consensus.ledger_close` | — | +| Panel | Type | PromQL | Labels Used | +| --------------------------- | ---------- | ---------------------------------------------------------------------------------------------------------------------------- | ----------- | +| Ledger Build Rate | stat | `ledger.build` call rate | — | +| Ledger Build Duration | timeseries | p95/p50 of `ledger.build` | — | +| Ledger Validation Rate | stat | `ledger.validate` call rate | — | +| Build Duration Heatmap | heatmap | `ledger.build` histogram buckets | `le` | +| TX Apply Duration | timeseries | p95/p50 of `tx.apply` | — | +| TX Apply Rate | timeseries | `tx.apply` call rate | — | +| Ledger Store Rate | stat | `ledger.store` call rate | — | +| Build vs Close Duration | timeseries | p95 `ledger.build` vs `consensus.round` (full round, not `consensus.ledger_close` which is only the sub-ms onClose prologue) | — | +| Ledger Close Interval & Age | timeseries | Interval: `1/rate(ledgers_closed_total)`; Age: `time() - (server_info{metric="last_close_time"} + 946684800)` | — | ### Peer Network (`peer-network`) diff --git a/src/xrpld/telemetry/MetricsRegistry.cpp b/src/xrpld/telemetry/MetricsRegistry.cpp index 4f31e89043..c8918b5d82 100644 --- a/src/xrpld/telemetry/MetricsRegistry.cpp +++ b/src/xrpld/telemetry/MetricsRegistry.cpp @@ -848,6 +848,22 @@ MetricsRegistry::registerServerInfoGauge() "last_close_converge_time_ms", static_cast(consensusInfo["previous_mseconds"].asUInt())); } + + // Network close time of the last closed ledger, as NetClock + // seconds since the Ripple epoch (2000-01-01). Unlike a span + // timestamp, a gauge value survives as a queryable time series, + // so dashboards can show last-close age (staleness) via + // now - value. The close interval comes from the + // ledgers_closed_total counter, not a delta of this gauge + // (a timestamp gauge's delta aliases to the scrape period). + // Skip until a ledger has closed. + if (auto const closed = app.getLedgerMaster().getClosedLedger()) + { + observe( + "last_close_time", + static_cast( + closed->header().closeTime.time_since_epoch().count())); + } } catch (...) // NOLINT(bugprone-empty-catch) {